From a74f586f28f864825b9a9a47601d96f36adac62b Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 5 Dec 2017 14:13:12 -0300 Subject: [PATCH 001/246] add resource limitation to delivery and mocks --- k8s/account.yaml | 7 +++++++ k8s/delivery.yaml | 7 +++++++ k8s/dronescheduler.yaml | 7 +++++++ k8s/mockdeliveryscheduler.yaml | 7 +++++++ k8s/thirdparty.yaml | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/k8s/account.yaml b/k8s/account.yaml index 3d0857bc..ba122ceb 100644 --- a/k8s/account.yaml +++ b/k8s/account.yaml @@ -66,3 +66,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index 8503f5fe..9d1f02c2 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -108,3 +108,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: 1 + memory: 2G + limits: + cpu: 1 + memory: 3G diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index dca8cab8..9efec81b 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -66,3 +66,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G diff --git a/k8s/mockdeliveryscheduler.yaml b/k8s/mockdeliveryscheduler.yaml index 80136bb6..376922a8 100644 --- a/k8s/mockdeliveryscheduler.yaml +++ b/k8s/mockdeliveryscheduler.yaml @@ -66,3 +66,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G diff --git a/k8s/thirdparty.yaml b/k8s/thirdparty.yaml index 83a9b1fa..a57e1e0e 100644 --- a/k8s/thirdparty.yaml +++ b/k8s/thirdparty.yaml @@ -66,3 +66,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G From 5889a2362c0e38af390e83199e40edf0dfd9b0f8 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 29 Dec 2017 11:55:47 -0300 Subject: [PATCH 002/246] remove secret creation for redis secondary key --- deployment.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deployment.md b/deployment.md index 8ea496a5..41970c1a 100644 --- a/deployment.md +++ b/deployment.md @@ -140,8 +140,7 @@ kubectl --namespace bc-shipping create --save-config=true secret generic deliver --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ - --from-literal=EH_ConnectionString= \ - --from-literal=Redis_SecondaryKey= + --from-literal=EH_ConnectionString= ``` Deploy the Delivery service: @@ -362,4 +361,4 @@ You can send delivery requests to the ingestion service using the swagger ui. ```bash export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace bc-shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` \ No newline at end of file +``` From ab2f0553aa4a38026bd84e2893d4bb9feda9d0fc Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 29 Dec 2017 12:24:51 -0300 Subject: [PATCH 003/246] remove dup config for RedisCache and EventHub sender svcs --- .../Services/Constants.cs | 1 - .../Fabrikam.DroneDelivery.DeliveryService/Startup.cs | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs b/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs index ca944bb6..95a834c9 100644 --- a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs +++ b/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs @@ -9,6 +9,5 @@ public class Constants { public const int PartitionKeyLength = 4; public const int RedisCacheDBId_Delivery = 0; - public const int RedisCacheDBId_DeliveryStatus = 1; } } diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index dd90b9cf..8a6b40fc 100644 --- a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -85,15 +85,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF c.SwaggerEndpoint("/swagger/v1/swagger.json", "Fabrikam DroneDelivery DeliveryService API V1"); }); - //TODO look into creating a factory of DocDBRepos/RedisCache/EventHubMessenger DocumentDBRepository.Configure(Configuration["DOCDB_ENDPOINT"], Configuration["DOCDB_KEY"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["REDIS_CONNSTR"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["REDIS_CONNSTR"], loggerFactory); - - EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); - EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); } } From 01ffe87125cdbc0a21c63ed60feeb600561546c0 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 13 Dec 2017 11:56:55 -0300 Subject: [PATCH 004/246] restore l5d integration restore scheduler integration add information why is required to default to bc-shipping namespace the dtab rules give deploy instructions using Y servicemesh.yml --- deployment.md | 16 +++++++++++++++- k8s/scheduler.yaml | 19 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/deployment.md b/deployment.md index 41970c1a..cb827355 100644 --- a/deployment.md +++ b/deployment.md @@ -349,7 +349,21 @@ Deploy Elasticsearch. For more information, see https://github.com/kubernetes/ex Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd -Deploy linkerd. For more information, see https://linkerd.io/getting-started/k8s/ +#### Deploy linkerd + +For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) + +> Note: +> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. +> Since Drone Delivery microservices are getting deployed into the bc-shipping custom namespace, this config needs to be modified. This will consist of a small change in the dtab rules. + +Deploy linkerd defaulting the namespace to bc-shipping instead: + +```bash +wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ +sed -i "s#/default#/bc-shipping#g" servicemesh.yml && \ +kubectl apply -f servicemesh.yml +``` Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy diff --git a/k8s/scheduler.yaml b/k8s/scheduler.yaml index 762f3bc4..f99c0a41 100644 --- a/k8s/scheduler.yaml +++ b/k8s/scheduler.yaml @@ -45,10 +45,10 @@ spec: value: http://package/api/packages - name: SERVICE_URI_THIRDPARTY value: http://thirdparty/api/ThirdPartyDeliveries - ## set this env to read from period of time - ## otherwise is from last known checkpoint + # set this env to read from period of time + # otherwise is from last known checkpoint - name: CHECKP_TIME_MINUTES - value: "0" + value: "0" - name: IOTHUB_EVENTHUB_NAME valueFrom: secretKeyRef: @@ -82,7 +82,18 @@ spec: name: scheduler-secrets key: queueconstring - name: IOTHUB_EVENTHUB_PARTITIONS - value: "32" + value: "32" + # Integration with l5d Daemon Sets + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: http_proxy + value: $(NODE_NAME):4140 - name: SERVICEMESH_HEADERS value: l5d-ctx-deadline,l5d-ctx-trace - name: SERVICEMESH_CORRELATION_HEADER From dcd55e24198291908c0ea6e2cec62776cf40efa9 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 3 Jan 2018 16:06:24 -0300 Subject: [PATCH 005/246] move account, dronescheduler and thirdparty services to their own namespaces and add new label co address https://github.com/mspnp/microservices-reference-implementation/issues/39 --- deployment.md | 53 ++++++++++++++++++++++++----------------- k8s/account.yaml | 9 ++++--- k8s/delivery.yaml | 3 +++ k8s/dronescheduler.yaml | 9 ++++--- k8s/ingestion.yaml | 9 ++++++- k8s/package.yml | 3 +++ k8s/scheduler.yaml | 2 ++ k8s/thirdparty.yaml | 11 +++++---- 8 files changed, 66 insertions(+), 33 deletions(-) diff --git a/deployment.md b/deployment.md index cb827355..6228feec 100644 --- a/deployment.md +++ b/deployment.md @@ -47,7 +47,10 @@ sudo az acs kubernetes install-cli az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create the Shipping BC namespace -kubectl create namespace bc-shipping +kubectl create namespace shipping && \ +kubectl create namespace accounts && \ +kubectl create namespace dronemgmt && \ +kubectl create namespace 3rdparty ``` Create an Azure Container Registry instance. @@ -112,7 +115,7 @@ az cosmosdb collection create \ Build the Delivery service ```bash -export DELIVERY_PATH=./microservices-reference-implementation/src/bc-shipping/delivery +export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up ``` @@ -136,7 +139,7 @@ export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) -kubectl --namespace bc-shipping create --save-config=true secret generic delivery-storageconf \ +kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ @@ -155,7 +158,7 @@ sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./micros sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" # Deploy the service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml ``` ## Deploy the Package service @@ -170,7 +173,7 @@ az cosmosdb create --name $COSMOSDB_NAME --kind MongoDB --resource-group $RESOUR Build the Package service ```bash -export PACKAGE_PATH=microservices-reference-implementation/src/bc-shipping/package +export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package # Build the app docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up @@ -191,10 +194,10 @@ sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-ref # Create secret export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") -kubectl -n bc-shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} +kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} # Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/package.yml +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/package.yml ``` ## Deploy the Ingestion service @@ -219,7 +222,7 @@ Note: you could also create this from [the Azure Portal](https://docs.microsoft. Build the Ingestion service ```bash -export INGESTION_PATH=./microservices-reference-implementation/src/bc-shipping/ingestion +export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion # Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ @@ -244,13 +247,13 @@ export EH_ACCESS_KEY_NAME=[YOUR_SHARED_ACCESS_POLICY_NAME_HERE] export EH_ACCESS_KEY_VALUE=[YOUR_SHARED_ACCESS_POLICY_VALUE_HERE] # Create secret -kubectl -n bc-shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ +kubectl -n shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ --from-literal=eventhub_name=${INGESTION_EH_NAME} \ --from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ --from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} # Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml ``` ## Deploy the Scheduler service @@ -265,7 +268,7 @@ az storage account create --resource-group $RESOURCE_GROUP --name $SCHEDULER_STO Build the Scheduler service ```bash -export SCHEDULER_PATH=./microservices-reference-implementation/src/bc-shipping/scheduler +export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler # Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ @@ -291,14 +294,14 @@ export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" # Create secrets -kubectl -n bc-shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ +kubectl -n shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ --from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ --from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ --from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ --from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} # Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml ``` ## Deploy mock services @@ -306,7 +309,7 @@ kubectl --namespace bc-shipping apply -f ./microservices-reference-implementatio Build the mock services ```bash -export MOCKS_PATH=microservices-reference-implementation/src/bc-shipping/delivery +export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up ``` @@ -334,15 +337,15 @@ sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-refe sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml # Deploy the service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/account.yaml && \ -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml +kubectl --namespace accounts apply -f ./microservices-reference-implementation/k8s/account.yaml && \ +kubectl --namespace dronemgmt apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ +kubectl --namespace 3rdparty apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml ``` ## Verify all services are running: ```bash -kubectl get all -n bc-shipping +kubectl get all --all-namespaces -l co=fabrikam ``` Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch @@ -355,13 +358,19 @@ For more information, see [https://linkerd.io/getting-started/k8s/](https://link > Note: > the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. -> Since Drone Delivery microservices are getting deployed into the bc-shipping custom namespace, this config needs to be modified. This will consist of a small change in the dtab rules. +> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified. This will consist of a small change in the dtab rules. -Deploy linkerd defaulting the namespace to bc-shipping instead: +Deploy linkerd defaulting the namespace to shipping instead: ```bash wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ -sed -i "s#/default#/bc-shipping#g" servicemesh.yml && \ +sed -i "s#/default#/shipping#g" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ kubectl apply -f servicemesh.yml ``` @@ -373,6 +382,6 @@ however for convenience, we exposed the Ingestion service with a public IP addre You can send delivery requests to the ingestion service using the swagger ui. ```bash -export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace bc-shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST ``` diff --git a/k8s/account.yaml b/k8s/account.yaml index 6350a5cb..07f37193 100644 --- a/k8s/account.yaml +++ b/k8s/account.yaml @@ -12,7 +12,8 @@ metadata: name: account labels: app: account - bc: shipping + bc: accounts + co: fabrikam spec: ports: - name: http @@ -29,7 +30,8 @@ metadata: labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: account version: 0.1.0 - bc: shipping + bc: accounts + co: fabrikam spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -40,7 +42,8 @@ spec: labels: app: account version: 0.1.0 - bc: shipping + bc: accounts + co: fabrikam annotations: team: accountservice spec: diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index 01a1e2d1..17999b5f 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -13,6 +13,7 @@ metadata: labels: app: delivery bc: shipping + co: fabrikam spec: ports: - name: http @@ -30,6 +31,7 @@ metadata: app: delivery version: 0.1.0 bc: shipping + co: fabrikam spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -41,6 +43,7 @@ spec: app: delivery version: 0.1.0 bc: shipping + co: fabrikam annotations: team: deliveryservice spec: diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index 9a5b4c31..5bcfb5e5 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -12,7 +12,8 @@ metadata: name: dronescheduler labels: app: dronescheduler - bc: shipping + bc: dronemgmt + co: fabrikam spec: ports: - name: http @@ -29,7 +30,8 @@ metadata: labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: dronescheduler version: 0.1.0 - bc: shipping + bc: dronemgmt + co: fabrikam spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -40,7 +42,8 @@ spec: labels: app: dronescheduler version: 0.1.0 - bc: shipping + bc: dronemgmt + co: fabrikam annotations: team: droneschedulerservice spec: diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index c96672f3..825a046e 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -13,6 +13,7 @@ metadata: labels: app: ingestion bc: shipping + co: fabrikam spec: ports: - port: 80 @@ -26,6 +27,11 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: ingestion + labels: + app: ingestion + version: 0.1.0 + bc: shipping + co: fabrikam spec: replicas: 1 template: @@ -34,6 +40,7 @@ spec: app: ingestion version: 0.1.0 bc: shipping + co: fabrikam spec: containers: - name: ingestion @@ -85,4 +92,4 @@ spec: fieldRef: fieldPath: status.podIP - name: http_proxy - value: $(NODE_NAME):4140 \ No newline at end of file + value: $(NODE_NAME):4140 diff --git a/k8s/package.yml b/k8s/package.yml index 3abb9e29..b7577cca 100644 --- a/k8s/package.yml +++ b/k8s/package.yml @@ -11,6 +11,7 @@ metadata: app: package version: 0.1.0 bc: shipping + co: fabrikam spec: replicas: 1 selector: @@ -22,6 +23,7 @@ spec: app: package version: 0.1.0 bc: shipping + co: fabrikam spec: containers: - name: package @@ -49,6 +51,7 @@ metadata: labels: app: package bc: shipping + co: fabrikam spec: selector: app: package diff --git a/k8s/scheduler.yaml b/k8s/scheduler.yaml index f99c0a41..906595ac 100644 --- a/k8s/scheduler.yaml +++ b/k8s/scheduler.yaml @@ -13,6 +13,7 @@ metadata: labels: app: scheduler bc: shipping + co: fabrikam spec: serviceName: scheduler replicas: 32 @@ -25,6 +26,7 @@ spec: app: scheduler version: 0.1.0 bc: shipping + co: fabrikam spec: containers: - image: diff --git a/k8s/thirdparty.yaml b/k8s/thirdparty.yaml index f5563b03..6e2a7aa6 100644 --- a/k8s/thirdparty.yaml +++ b/k8s/thirdparty.yaml @@ -12,7 +12,8 @@ metadata: name: thirdparty labels: app: thirdparty - bc: shipping + bc: 3rdparty + co: fabrikam spec: ports: - name: http @@ -29,18 +30,20 @@ metadata: labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: thirdparty version: 0.1.0 - bc: shipping + bc: 3rdparty + co: fabrikam spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: thirdparty + app: thirdparty template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template metadata: labels: app: thirdparty - bc: shipping + bc: 3rdparty version: 0.1.0 + co: fabrikam annotations: team: thirdpartyservice spec: From 24140e11f3c3d82952843fbb45fcb65de40fd868 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 5 Jan 2018 13:03:07 -0300 Subject: [PATCH 006/246] move from bc-shipping to shipping folder --- .../CorrelationLogEventEnricher.cs | 0 .../Fabrikam.DroneDelivery.Common/DeliveryStage.cs | 0 .../Fabrikam.DroneDelivery.Common.csproj | 0 .../Fabrikam.DroneDelivery.Common/Location.cs | 0 .../Fabrikam.DroneDelivery.Common/UserAccount.cs | 0 .../Common/DocumentClientExceptionExtensions.cs | 0 .../DeliveriesControllerFixture.cs | 0 ...rikam.DroneDelivery.DeliveryService.Tests.csproj | 0 .../GlobalInternalErrorHandlerMiddlewareFixture.cs | 0 .../GlobalLoggerMiddlewareFixture.cs | 0 .../.dockerignore | 0 .../Controllers/DeliveriesController.cs | 0 .../Dockerfile | 0 .../Fabrikam.DroneDelivery.DeliveryService.csproj | 0 .../Middlewares/Builder/GlobalHandlersBuilder.cs | 0 .../GlobalInternalErrorHandlerMiddleware.cs | 0 .../Middlewares/GlobalLoggerMiddleware.cs | 0 .../Models/BaseCache.cs | 0 .../Models/BaseDocument.cs | 0 .../Models/BaseMessage.cs | 0 .../Models/Confirmation.cs | 0 .../Models/ConfirmationType.cs | 0 .../Models/DateTimeStamp.cs | 0 .../Models/Delivery.cs | 0 .../Models/DeliveryHistory.cs | 0 .../Models/DeliveryStatus.cs | 0 .../Models/DeliveryTrackingEvent.cs | 0 .../Models/DuplicateResourceException.cs | 0 .../Models/Extensions.cs | 0 .../Models/InternalConfirmation.cs | 0 .../Models/InternalDelivery.cs | 0 .../Models/InternalNotifyMeRequest.cs | 0 .../Models/NotifyMeRequest.cs | 0 .../Models/RescheduledDelivery.cs | 0 .../Program.cs | 0 .../Services/Constants.cs | 0 .../Services/DeliveryHistoryService.cs | 0 .../Services/DeliveryRepository.cs | 0 .../Services/DeliveryTrackingEventRepository.cs | 0 .../Services/DocumentDBRepository.cs | 0 .../Services/EventHubSender.cs | 0 .../Services/IDeliveryHistoryService.cs | 0 .../Services/IDeliveryRepository.cs | 0 .../Services/IDeliveryTrackingEventRepository.cs | 0 .../Services/INotificationService.cs | 0 .../Services/INotifyMeRequestRepository.cs | 0 .../Services/NoOpNotificationService.cs | 0 .../Services/NotifyMeRequestRepository.cs | 0 .../Services/RedisCache.cs | 0 .../Startup.cs | 0 .../appsettings.Development.json | 0 .../appsettings.json | 0 .../delivery/Fabrikam.DroneDelivery.sln | 0 .../delivery/MockAccountService/.dockerignore | 0 .../Controllers/AccountController.cs | 0 .../delivery/MockAccountService/Dockerfile | 0 .../MockAccountService/MockAccountService.csproj | 0 .../delivery/MockAccountService/Program.cs | 0 .../delivery/MockAccountService/Startup.cs | 0 .../MockAccountService/appsettings.Development.json | 0 .../delivery/MockAccountService/appsettings.json | 0 .../delivery/MockDeliveryScheduler/.dockerignore | 0 .../Controllers/HomeController.cs | 0 .../delivery/MockDeliveryScheduler/Dockerfile | 0 .../MockDeliveryScheduler.csproj | 0 .../delivery/MockDeliveryScheduler/Program.cs | 0 .../delivery/MockDeliveryScheduler/Startup.cs | 0 .../MockDeliveryScheduler/Views/Home/Index.cshtml | 0 .../MockDeliveryScheduler/Views/Shared/Error.cshtml | 0 .../Views/Shared/_Layout.cshtml | 0 .../Views/Shared/_ValidationScriptsPartial.cshtml | 0 .../MockDeliveryScheduler/Views/_ViewImports.cshtml | 0 .../MockDeliveryScheduler/Views/_ViewStart.cshtml | 0 .../appsettings.Development.json | 0 .../delivery/MockDeliveryScheduler/appsettings.json | 0 .../MockDeliveryScheduler/wwwroot/favicon.ico | Bin .../delivery/MockDroneScheduler/.dockerignore | 0 .../Controllers/DroneDeliveriesController.cs | 0 .../delivery/MockDroneScheduler/Dockerfile | 0 .../MockDroneScheduler/MockDroneScheduler.csproj | 0 .../MockDroneScheduler/Models/DroneDelivery.cs | 0 .../MockDroneScheduler/Models/PackageDetail.cs | 0 .../MockDroneScheduler/Models/PackageSize.cs | 0 .../delivery/MockDroneScheduler/Program.cs | 0 .../delivery/MockDroneScheduler/Startup.cs | 0 .../MockDroneScheduler/appsettings.Development.json | 0 .../delivery/MockDroneScheduler/appsettings.json | 0 .../delivery/MockThirdPartyService/.dockerignore | 0 .../Controllers/ThirdPartyDeliveriesController.cs | 0 .../delivery/MockThirdPartyService/Dockerfile | 0 .../MockThirdPartyService.csproj | 0 .../delivery/MockThirdPartyService/Program.cs | 0 .../delivery/MockThirdPartyService/Startup.cs | 0 .../appsettings.Development.json | 0 .../delivery/MockThirdPartyService/appsettings.json | 0 .../delivery/docker-compose.ci.build.yml | 0 .../delivery/docker-compose.dcproj | 0 .../delivery/docker-compose.override.yml | 0 .../delivery/docker-compose.yml | 0 .../deliveryhistory/Model.csx | 0 .../deliveryhistory/function.json | 0 .../deliveryhistory/historydocument.csx | 0 .../deliveryhistory/project.json | 0 .../deliveryhistory/run.csx | 0 src/{bc-shipping => shipping}/ingestion/.gitignore | 0 .../ingestion/.mvn/wrapper/maven-wrapper.jar | Bin .../ingestion/.mvn/wrapper/maven-wrapper.properties | 0 src/{bc-shipping => shipping}/ingestion/Dockerfile | 0 .../ingestion/Dockerfilemaven | 0 .../ingestion/mvn-entrypoint.sh | 0 src/{bc-shipping => shipping}/ingestion/mvnw | 0 src/{bc-shipping => shipping}/ingestion/mvnw.cmd | 0 src/{bc-shipping => shipping}/ingestion/pom.xml | 0 .../ingestion/settings-docker.xml | 0 .../ingestion/IngestionApplication.java | 0 .../configuration/ApplicationProperties.java | 0 .../ingestion/configuration/AsyncConfiguration.java | 0 .../configuration/AsyncExceptionHandler.java | 0 .../ingestion/controller/IngestionController.java | 0 .../ingestion/models/ConfirmationRequired.java | 0 .../ingestion/models/ContainerSize.java | 0 .../dronedelivery/ingestion/models/Delivery.java | 0 .../ingestion/models/DeliveryBase.java | 0 .../ingestion/models/ExternalDelivery.java | 0 .../models/ExternalRescheduledDelivery.java | 0 .../dronedelivery/ingestion/models/PackageInfo.java | 0 .../ingestion/models/RescheduledDelivery.java | 0 .../dronedelivery/ingestion/service/Ingestion.java | 0 .../ingestion/service/IngestionImpl.java | 0 .../dronedelivery/ingestion/util/ClientPool.java | 0 .../ingestion/util/ClientPoolImpl.java | 0 .../ingestion/src/main/resources/DeliveryBean.xml | 0 .../src/main/resources/application.properties | 0 .../ingestion/src/main/resources/log4j2.xml | 0 .../ingestion/IngestionControllerTest.java | 0 .../ingestion/IngestionPerformanceTest.java | 0 src/{bc-shipping => shipping}/package/.gitignore | 0 src/{bc-shipping => shipping}/package/app/api.json | 0 .../package/app/controllers/index.ts | 0 .../package/app/controllers/package-controllers.ts | 0 .../package/app/initializer.ts | 0 src/{bc-shipping => shipping}/package/app/main.ts | 0 .../package/app/models/api-models.ts | 0 .../package/app/models/package.ts | 0 .../package/app/models/repository.ts | 0 src/{bc-shipping => shipping}/package/app/server.ts | 0 .../package/app/util/logging.ts | 0 .../package/app/util/mongo-err.ts | 0 .../package/app/util/settings.ts | 0 .../package/build/create-secrets.sh | 0 .../package/build/dev.dockerfile | 0 .../package/build/docker-compose.ci.build.yml | 0 .../package/build/docker-compose.yml | 0 .../package/build/prod.dockerfile | 0 src/{bc-shipping => shipping}/package/down.sh | 0 src/{bc-shipping => shipping}/package/gulpfile.js | 0 .../package/package-service.md | 0 src/{bc-shipping => shipping}/package/package.json | 0 .../package/test/models/test-data-access.ts | 0 src/{bc-shipping => shipping}/package/tsconfig.json | 0 src/{bc-shipping => shipping}/package/up.sh | 0 src/{bc-shipping => shipping}/scheduler/.classpath | 0 src/{bc-shipping => shipping}/scheduler/.project | 0 src/{bc-shipping => shipping}/scheduler/Dockerfile | 0 .../scheduler/Dockerfilemaven | 0 src/{bc-shipping => shipping}/scheduler/Readme.txt | 0 .../scheduler/conf/Config.properties | 0 .../scheduler/conf/application.conf | 0 .../scheduler/conf/log4j2.xml | 0 .../scheduler/mvn-entrypoint.sh | 0 src/{bc-shipping => shipping}/scheduler/pom.xml | 0 .../scheduler/settings-docker.xml | 0 .../deliveryscheduler/akkareader/AkkaDelivery.java | 0 .../deliveryscheduler/akkareader/Main.java | 0 .../akkareader/ReactiveStreamingApp.java | 0 .../scheduler/DeliveryRequestEventProcessor.java | 0 .../scheduler/SchedulerSettings.java | 0 .../scheduler/ServiceFailureMetadata.java | 0 .../deliveryscheduler/scheduler/ServiceName.java | 0 .../StorageQueue/StorageQueueClientFactory.java | 0 .../scheduler/StorageQueue/StorageQueueTest.java | 0 .../scheduler/models/invoker/ConfirmationType.java | 0 .../scheduler/models/invoker/DeliverySchedule.java | 0 .../scheduler/models/invoker/DroneDelivery.java | 0 .../scheduler/models/invoker/Location.java | 0 .../scheduler/models/invoker/PackageDetail.java | 0 .../scheduler/models/invoker/PackageGen.java | 0 .../scheduler/models/invoker/PackageSize.java | 0 .../scheduler/models/invoker/UserAccount.java | 0 .../models/receiver/ConfirmationRequired.java | 0 .../scheduler/models/receiver/ContainerSize.java | 0 .../scheduler/models/receiver/Delivery.java | 0 .../scheduler/models/receiver/PackageInfo.java | 0 .../services/AccountServiceCallerImpl.java | 0 .../services/BackendServiceCallFailedException.java | 0 .../services/DeliveryServiceCallerImpl.java | 0 .../services/DroneSchedulerServiceCallerImpl.java | 0 .../services/PackageServiceCallerImpl.java | 0 .../scheduler/services/ServiceCaller.java | 0 .../scheduler/services/ServiceCallerImpl.java | 0 .../services/ServiceCallerResponseErrorHandler.java | 0 .../services/ThirdPartyServiceCallerImpl.java | 0 .../services/tests/PostDeliveryRequests.java | 0 .../services/tests/TestBackendServices.java | 0 .../tests/ValidateSerializationOptions.java | 0 .../scheduler/utils/ConfigReader.java | 0 .../scheduler/utils/CustomDateTimeDeserializer.java | 0 .../utils/IdleConnectionMonitorThread.java | 0 .../scheduler/utils/KeepAliveStrategy.java | 0 .../scheduler/utils/LocationRandomizer.java | 0 .../scheduler/utils/ModelsConverter.java | 0 211 files changed, 0 insertions(+), 0 deletions(-) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.Common/DeliveryStage.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.Common/Location.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Common/DocumentClientExceptionExtensions.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalInternalErrorHandlerMiddlewareFixture.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/.dockerignore (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/Builder/GlobalHandlersBuilder.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalInternalErrorHandlerMiddleware.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalLoggerMiddleware.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseCache.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseDocument.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseMessage.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Confirmation.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/ConfirmationType.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DateTimeStamp.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryHistory.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryStatus.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryTrackingEvent.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DuplicateResourceException.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Extensions.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalConfirmation.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalDelivery.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalNotifyMeRequest.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/NotifyMeRequest.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/RescheduledDelivery.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryTrackingEventRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryTrackingEventRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotificationService.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotifyMeRequestRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NoOpNotificationService.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NotifyMeRequestRepository.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json (100%) rename src/{bc-shipping => shipping}/delivery/Fabrikam.DroneDelivery.sln (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/.dockerignore (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/Controllers/AccountController.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/Dockerfile (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/MockAccountService.csproj (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/Program.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/Startup.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/appsettings.Development.json (100%) rename src/{bc-shipping => shipping}/delivery/MockAccountService/appsettings.json (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/.dockerignore (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Controllers/HomeController.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Dockerfile (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Program.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Startup.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/appsettings.Development.json (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/appsettings.json (100%) rename src/{bc-shipping => shipping}/delivery/MockDeliveryScheduler/wwwroot/favicon.ico (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/.dockerignore (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Dockerfile (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/MockDroneScheduler.csproj (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Models/DroneDelivery.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Models/PackageDetail.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Models/PackageSize.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Program.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/Startup.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/appsettings.Development.json (100%) rename src/{bc-shipping => shipping}/delivery/MockDroneScheduler/appsettings.json (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/.dockerignore (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/Dockerfile (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/MockThirdPartyService.csproj (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/Program.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/Startup.cs (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/appsettings.Development.json (100%) rename src/{bc-shipping => shipping}/delivery/MockThirdPartyService/appsettings.json (100%) rename src/{bc-shipping => shipping}/delivery/docker-compose.ci.build.yml (100%) rename src/{bc-shipping => shipping}/delivery/docker-compose.dcproj (100%) rename src/{bc-shipping => shipping}/delivery/docker-compose.override.yml (100%) rename src/{bc-shipping => shipping}/delivery/docker-compose.yml (100%) rename src/{bc-shipping => shipping}/deliveryhistory/Model.csx (100%) rename src/{bc-shipping => shipping}/deliveryhistory/function.json (100%) rename src/{bc-shipping => shipping}/deliveryhistory/historydocument.csx (100%) rename src/{bc-shipping => shipping}/deliveryhistory/project.json (100%) rename src/{bc-shipping => shipping}/deliveryhistory/run.csx (100%) rename src/{bc-shipping => shipping}/ingestion/.gitignore (100%) rename src/{bc-shipping => shipping}/ingestion/.mvn/wrapper/maven-wrapper.jar (100%) rename src/{bc-shipping => shipping}/ingestion/.mvn/wrapper/maven-wrapper.properties (100%) rename src/{bc-shipping => shipping}/ingestion/Dockerfile (100%) rename src/{bc-shipping => shipping}/ingestion/Dockerfilemaven (100%) rename src/{bc-shipping => shipping}/ingestion/mvn-entrypoint.sh (100%) rename src/{bc-shipping => shipping}/ingestion/mvnw (100%) rename src/{bc-shipping => shipping}/ingestion/mvnw.cmd (100%) rename src/{bc-shipping => shipping}/ingestion/pom.xml (100%) rename src/{bc-shipping => shipping}/ingestion/settings-docker.xml (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/IngestionApplication.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncConfiguration.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ConfirmationRequired.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ContainerSize.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/Delivery.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/DeliveryBase.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalDelivery.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalRescheduledDelivery.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/PackageInfo.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/RescheduledDelivery.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/Ingestion.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/resources/DeliveryBean.xml (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/resources/application.properties (100%) rename src/{bc-shipping => shipping}/ingestion/src/main/resources/log4j2.xml (100%) rename src/{bc-shipping => shipping}/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java (100%) rename src/{bc-shipping => shipping}/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java (100%) rename src/{bc-shipping => shipping}/package/.gitignore (100%) rename src/{bc-shipping => shipping}/package/app/api.json (100%) rename src/{bc-shipping => shipping}/package/app/controllers/index.ts (100%) rename src/{bc-shipping => shipping}/package/app/controllers/package-controllers.ts (100%) rename src/{bc-shipping => shipping}/package/app/initializer.ts (100%) rename src/{bc-shipping => shipping}/package/app/main.ts (100%) rename src/{bc-shipping => shipping}/package/app/models/api-models.ts (100%) rename src/{bc-shipping => shipping}/package/app/models/package.ts (100%) rename src/{bc-shipping => shipping}/package/app/models/repository.ts (100%) rename src/{bc-shipping => shipping}/package/app/server.ts (100%) rename src/{bc-shipping => shipping}/package/app/util/logging.ts (100%) rename src/{bc-shipping => shipping}/package/app/util/mongo-err.ts (100%) rename src/{bc-shipping => shipping}/package/app/util/settings.ts (100%) rename src/{bc-shipping => shipping}/package/build/create-secrets.sh (100%) rename src/{bc-shipping => shipping}/package/build/dev.dockerfile (100%) rename src/{bc-shipping => shipping}/package/build/docker-compose.ci.build.yml (100%) rename src/{bc-shipping => shipping}/package/build/docker-compose.yml (100%) rename src/{bc-shipping => shipping}/package/build/prod.dockerfile (100%) rename src/{bc-shipping => shipping}/package/down.sh (100%) rename src/{bc-shipping => shipping}/package/gulpfile.js (100%) rename src/{bc-shipping => shipping}/package/package-service.md (100%) rename src/{bc-shipping => shipping}/package/package.json (100%) rename src/{bc-shipping => shipping}/package/test/models/test-data-access.ts (100%) rename src/{bc-shipping => shipping}/package/tsconfig.json (100%) rename src/{bc-shipping => shipping}/package/up.sh (100%) rename src/{bc-shipping => shipping}/scheduler/.classpath (100%) rename src/{bc-shipping => shipping}/scheduler/.project (100%) rename src/{bc-shipping => shipping}/scheduler/Dockerfile (100%) rename src/{bc-shipping => shipping}/scheduler/Dockerfilemaven (100%) rename src/{bc-shipping => shipping}/scheduler/Readme.txt (100%) rename src/{bc-shipping => shipping}/scheduler/conf/Config.properties (100%) rename src/{bc-shipping => shipping}/scheduler/conf/application.conf (100%) rename src/{bc-shipping => shipping}/scheduler/conf/log4j2.xml (100%) rename src/{bc-shipping => shipping}/scheduler/mvn-entrypoint.sh (100%) rename src/{bc-shipping => shipping}/scheduler/pom.xml (100%) rename src/{bc-shipping => shipping}/scheduler/settings-docker.xml (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java (100%) rename src/{bc-shipping => shipping}/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java (100%) diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/DeliveryStage.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/DeliveryStage.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/DeliveryStage.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.Common/DeliveryStage.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj rename to src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Common/DocumentClientExceptionExtensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Common/DocumentClientExceptionExtensions.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Common/DocumentClientExceptionExtensions.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Common/DocumentClientExceptionExtensions.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalInternalErrorHandlerMiddlewareFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalInternalErrorHandlerMiddlewareFixture.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalInternalErrorHandlerMiddlewareFixture.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalInternalErrorHandlerMiddlewareFixture.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/.dockerignore b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/.dockerignore similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/.dockerignore rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/.dockerignore diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/Builder/GlobalHandlersBuilder.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/Builder/GlobalHandlersBuilder.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/Builder/GlobalHandlersBuilder.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/Builder/GlobalHandlersBuilder.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalInternalErrorHandlerMiddleware.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalInternalErrorHandlerMiddleware.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalInternalErrorHandlerMiddleware.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalInternalErrorHandlerMiddleware.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalLoggerMiddleware.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalLoggerMiddleware.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalLoggerMiddleware.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Middlewares/GlobalLoggerMiddleware.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseCache.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseCache.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseCache.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseCache.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseDocument.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseDocument.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseDocument.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseDocument.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseMessage.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseMessage.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseMessage.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/BaseMessage.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Confirmation.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Confirmation.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Confirmation.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Confirmation.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/ConfirmationType.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/ConfirmationType.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/ConfirmationType.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/ConfirmationType.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DateTimeStamp.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DateTimeStamp.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DateTimeStamp.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DateTimeStamp.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryHistory.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryHistory.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryHistory.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryHistory.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryStatus.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryStatus.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryStatus.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryStatus.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryTrackingEvent.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryTrackingEvent.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryTrackingEvent.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveryTrackingEvent.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DuplicateResourceException.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DuplicateResourceException.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DuplicateResourceException.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DuplicateResourceException.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Extensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Extensions.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Extensions.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Extensions.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalConfirmation.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalConfirmation.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalConfirmation.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalConfirmation.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalDelivery.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalDelivery.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalDelivery.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalDelivery.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalNotifyMeRequest.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalNotifyMeRequest.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalNotifyMeRequest.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/InternalNotifyMeRequest.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/NotifyMeRequest.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/NotifyMeRequest.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/NotifyMeRequest.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/NotifyMeRequest.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/RescheduledDelivery.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/RescheduledDelivery.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/RescheduledDelivery.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/RescheduledDelivery.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryTrackingEventRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryTrackingEventRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryTrackingEventRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryTrackingEventRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryTrackingEventRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryTrackingEventRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryTrackingEventRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryTrackingEventRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotificationService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotificationService.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotificationService.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotificationService.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotifyMeRequestRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotifyMeRequestRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotifyMeRequestRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/INotifyMeRequestRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NoOpNotificationService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NoOpNotificationService.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NoOpNotificationService.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NoOpNotificationService.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NotifyMeRequestRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NotifyMeRequestRepository.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NotifyMeRequestRepository.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/NotifyMeRequestRepository.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json rename to src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json diff --git a/src/bc-shipping/delivery/Fabrikam.DroneDelivery.sln b/src/shipping/delivery/Fabrikam.DroneDelivery.sln similarity index 100% rename from src/bc-shipping/delivery/Fabrikam.DroneDelivery.sln rename to src/shipping/delivery/Fabrikam.DroneDelivery.sln diff --git a/src/bc-shipping/delivery/MockAccountService/.dockerignore b/src/shipping/delivery/MockAccountService/.dockerignore similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/.dockerignore rename to src/shipping/delivery/MockAccountService/.dockerignore diff --git a/src/bc-shipping/delivery/MockAccountService/Controllers/AccountController.cs b/src/shipping/delivery/MockAccountService/Controllers/AccountController.cs similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/Controllers/AccountController.cs rename to src/shipping/delivery/MockAccountService/Controllers/AccountController.cs diff --git a/src/bc-shipping/delivery/MockAccountService/Dockerfile b/src/shipping/delivery/MockAccountService/Dockerfile similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/Dockerfile rename to src/shipping/delivery/MockAccountService/Dockerfile diff --git a/src/bc-shipping/delivery/MockAccountService/MockAccountService.csproj b/src/shipping/delivery/MockAccountService/MockAccountService.csproj similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/MockAccountService.csproj rename to src/shipping/delivery/MockAccountService/MockAccountService.csproj diff --git a/src/bc-shipping/delivery/MockAccountService/Program.cs b/src/shipping/delivery/MockAccountService/Program.cs similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/Program.cs rename to src/shipping/delivery/MockAccountService/Program.cs diff --git a/src/bc-shipping/delivery/MockAccountService/Startup.cs b/src/shipping/delivery/MockAccountService/Startup.cs similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/Startup.cs rename to src/shipping/delivery/MockAccountService/Startup.cs diff --git a/src/bc-shipping/delivery/MockAccountService/appsettings.Development.json b/src/shipping/delivery/MockAccountService/appsettings.Development.json similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/appsettings.Development.json rename to src/shipping/delivery/MockAccountService/appsettings.Development.json diff --git a/src/bc-shipping/delivery/MockAccountService/appsettings.json b/src/shipping/delivery/MockAccountService/appsettings.json similarity index 100% rename from src/bc-shipping/delivery/MockAccountService/appsettings.json rename to src/shipping/delivery/MockAccountService/appsettings.json diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/.dockerignore b/src/shipping/delivery/MockDeliveryScheduler/.dockerignore similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/.dockerignore rename to src/shipping/delivery/MockDeliveryScheduler/.dockerignore diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs b/src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs rename to src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Dockerfile b/src/shipping/delivery/MockDeliveryScheduler/Dockerfile similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Dockerfile rename to src/shipping/delivery/MockDeliveryScheduler/Dockerfile diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj b/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj rename to src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Program.cs b/src/shipping/delivery/MockDeliveryScheduler/Program.cs similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Program.cs rename to src/shipping/delivery/MockDeliveryScheduler/Program.cs diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Startup.cs b/src/shipping/delivery/MockDeliveryScheduler/Startup.cs similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Startup.cs rename to src/shipping/delivery/MockDeliveryScheduler/Startup.cs diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml rename to src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/appsettings.Development.json b/src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/appsettings.Development.json rename to src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/appsettings.json b/src/shipping/delivery/MockDeliveryScheduler/appsettings.json similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/appsettings.json rename to src/shipping/delivery/MockDeliveryScheduler/appsettings.json diff --git a/src/bc-shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico b/src/shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico similarity index 100% rename from src/bc-shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico rename to src/shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico diff --git a/src/bc-shipping/delivery/MockDroneScheduler/.dockerignore b/src/shipping/delivery/MockDroneScheduler/.dockerignore similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/.dockerignore rename to src/shipping/delivery/MockDroneScheduler/.dockerignore diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs b/src/shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs rename to src/shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Dockerfile b/src/shipping/delivery/MockDroneScheduler/Dockerfile similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Dockerfile rename to src/shipping/delivery/MockDroneScheduler/Dockerfile diff --git a/src/bc-shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj b/src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj rename to src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs b/src/shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs rename to src/shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs b/src/shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs rename to src/shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Models/PackageSize.cs b/src/shipping/delivery/MockDroneScheduler/Models/PackageSize.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Models/PackageSize.cs rename to src/shipping/delivery/MockDroneScheduler/Models/PackageSize.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Program.cs b/src/shipping/delivery/MockDroneScheduler/Program.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Program.cs rename to src/shipping/delivery/MockDroneScheduler/Program.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/Startup.cs b/src/shipping/delivery/MockDroneScheduler/Startup.cs similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/Startup.cs rename to src/shipping/delivery/MockDroneScheduler/Startup.cs diff --git a/src/bc-shipping/delivery/MockDroneScheduler/appsettings.Development.json b/src/shipping/delivery/MockDroneScheduler/appsettings.Development.json similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/appsettings.Development.json rename to src/shipping/delivery/MockDroneScheduler/appsettings.Development.json diff --git a/src/bc-shipping/delivery/MockDroneScheduler/appsettings.json b/src/shipping/delivery/MockDroneScheduler/appsettings.json similarity index 100% rename from src/bc-shipping/delivery/MockDroneScheduler/appsettings.json rename to src/shipping/delivery/MockDroneScheduler/appsettings.json diff --git a/src/bc-shipping/delivery/MockThirdPartyService/.dockerignore b/src/shipping/delivery/MockThirdPartyService/.dockerignore similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/.dockerignore rename to src/shipping/delivery/MockThirdPartyService/.dockerignore diff --git a/src/bc-shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs b/src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs rename to src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs diff --git a/src/bc-shipping/delivery/MockThirdPartyService/Dockerfile b/src/shipping/delivery/MockThirdPartyService/Dockerfile similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/Dockerfile rename to src/shipping/delivery/MockThirdPartyService/Dockerfile diff --git a/src/bc-shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj b/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj rename to src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj diff --git a/src/bc-shipping/delivery/MockThirdPartyService/Program.cs b/src/shipping/delivery/MockThirdPartyService/Program.cs similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/Program.cs rename to src/shipping/delivery/MockThirdPartyService/Program.cs diff --git a/src/bc-shipping/delivery/MockThirdPartyService/Startup.cs b/src/shipping/delivery/MockThirdPartyService/Startup.cs similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/Startup.cs rename to src/shipping/delivery/MockThirdPartyService/Startup.cs diff --git a/src/bc-shipping/delivery/MockThirdPartyService/appsettings.Development.json b/src/shipping/delivery/MockThirdPartyService/appsettings.Development.json similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/appsettings.Development.json rename to src/shipping/delivery/MockThirdPartyService/appsettings.Development.json diff --git a/src/bc-shipping/delivery/MockThirdPartyService/appsettings.json b/src/shipping/delivery/MockThirdPartyService/appsettings.json similarity index 100% rename from src/bc-shipping/delivery/MockThirdPartyService/appsettings.json rename to src/shipping/delivery/MockThirdPartyService/appsettings.json diff --git a/src/bc-shipping/delivery/docker-compose.ci.build.yml b/src/shipping/delivery/docker-compose.ci.build.yml similarity index 100% rename from src/bc-shipping/delivery/docker-compose.ci.build.yml rename to src/shipping/delivery/docker-compose.ci.build.yml diff --git a/src/bc-shipping/delivery/docker-compose.dcproj b/src/shipping/delivery/docker-compose.dcproj similarity index 100% rename from src/bc-shipping/delivery/docker-compose.dcproj rename to src/shipping/delivery/docker-compose.dcproj diff --git a/src/bc-shipping/delivery/docker-compose.override.yml b/src/shipping/delivery/docker-compose.override.yml similarity index 100% rename from src/bc-shipping/delivery/docker-compose.override.yml rename to src/shipping/delivery/docker-compose.override.yml diff --git a/src/bc-shipping/delivery/docker-compose.yml b/src/shipping/delivery/docker-compose.yml similarity index 100% rename from src/bc-shipping/delivery/docker-compose.yml rename to src/shipping/delivery/docker-compose.yml diff --git a/src/bc-shipping/deliveryhistory/Model.csx b/src/shipping/deliveryhistory/Model.csx similarity index 100% rename from src/bc-shipping/deliveryhistory/Model.csx rename to src/shipping/deliveryhistory/Model.csx diff --git a/src/bc-shipping/deliveryhistory/function.json b/src/shipping/deliveryhistory/function.json similarity index 100% rename from src/bc-shipping/deliveryhistory/function.json rename to src/shipping/deliveryhistory/function.json diff --git a/src/bc-shipping/deliveryhistory/historydocument.csx b/src/shipping/deliveryhistory/historydocument.csx similarity index 100% rename from src/bc-shipping/deliveryhistory/historydocument.csx rename to src/shipping/deliveryhistory/historydocument.csx diff --git a/src/bc-shipping/deliveryhistory/project.json b/src/shipping/deliveryhistory/project.json similarity index 100% rename from src/bc-shipping/deliveryhistory/project.json rename to src/shipping/deliveryhistory/project.json diff --git a/src/bc-shipping/deliveryhistory/run.csx b/src/shipping/deliveryhistory/run.csx similarity index 100% rename from src/bc-shipping/deliveryhistory/run.csx rename to src/shipping/deliveryhistory/run.csx diff --git a/src/bc-shipping/ingestion/.gitignore b/src/shipping/ingestion/.gitignore similarity index 100% rename from src/bc-shipping/ingestion/.gitignore rename to src/shipping/ingestion/.gitignore diff --git a/src/bc-shipping/ingestion/.mvn/wrapper/maven-wrapper.jar b/src/shipping/ingestion/.mvn/wrapper/maven-wrapper.jar similarity index 100% rename from src/bc-shipping/ingestion/.mvn/wrapper/maven-wrapper.jar rename to src/shipping/ingestion/.mvn/wrapper/maven-wrapper.jar diff --git a/src/bc-shipping/ingestion/.mvn/wrapper/maven-wrapper.properties b/src/shipping/ingestion/.mvn/wrapper/maven-wrapper.properties similarity index 100% rename from src/bc-shipping/ingestion/.mvn/wrapper/maven-wrapper.properties rename to src/shipping/ingestion/.mvn/wrapper/maven-wrapper.properties diff --git a/src/bc-shipping/ingestion/Dockerfile b/src/shipping/ingestion/Dockerfile similarity index 100% rename from src/bc-shipping/ingestion/Dockerfile rename to src/shipping/ingestion/Dockerfile diff --git a/src/bc-shipping/ingestion/Dockerfilemaven b/src/shipping/ingestion/Dockerfilemaven similarity index 100% rename from src/bc-shipping/ingestion/Dockerfilemaven rename to src/shipping/ingestion/Dockerfilemaven diff --git a/src/bc-shipping/ingestion/mvn-entrypoint.sh b/src/shipping/ingestion/mvn-entrypoint.sh similarity index 100% rename from src/bc-shipping/ingestion/mvn-entrypoint.sh rename to src/shipping/ingestion/mvn-entrypoint.sh diff --git a/src/bc-shipping/ingestion/mvnw b/src/shipping/ingestion/mvnw similarity index 100% rename from src/bc-shipping/ingestion/mvnw rename to src/shipping/ingestion/mvnw diff --git a/src/bc-shipping/ingestion/mvnw.cmd b/src/shipping/ingestion/mvnw.cmd similarity index 100% rename from src/bc-shipping/ingestion/mvnw.cmd rename to src/shipping/ingestion/mvnw.cmd diff --git a/src/bc-shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml similarity index 100% rename from src/bc-shipping/ingestion/pom.xml rename to src/shipping/ingestion/pom.xml diff --git a/src/bc-shipping/ingestion/settings-docker.xml b/src/shipping/ingestion/settings-docker.xml similarity index 100% rename from src/bc-shipping/ingestion/settings-docker.xml rename to src/shipping/ingestion/settings-docker.xml diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/IngestionApplication.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/IngestionApplication.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/IngestionApplication.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/IngestionApplication.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncConfiguration.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncConfiguration.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncConfiguration.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncConfiguration.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ConfirmationRequired.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ConfirmationRequired.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ConfirmationRequired.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ConfirmationRequired.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ContainerSize.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ContainerSize.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ContainerSize.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ContainerSize.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/Delivery.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/Delivery.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/Delivery.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/Delivery.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/DeliveryBase.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/DeliveryBase.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/DeliveryBase.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/DeliveryBase.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalDelivery.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalDelivery.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalDelivery.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalDelivery.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalRescheduledDelivery.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalRescheduledDelivery.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalRescheduledDelivery.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/ExternalRescheduledDelivery.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/PackageInfo.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/PackageInfo.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/PackageInfo.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/PackageInfo.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/RescheduledDelivery.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/RescheduledDelivery.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/RescheduledDelivery.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/models/RescheduledDelivery.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/Ingestion.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/Ingestion.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/Ingestion.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/Ingestion.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java diff --git a/src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java similarity index 100% rename from src/bc-shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java rename to src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java diff --git a/src/bc-shipping/ingestion/src/main/resources/DeliveryBean.xml b/src/shipping/ingestion/src/main/resources/DeliveryBean.xml similarity index 100% rename from src/bc-shipping/ingestion/src/main/resources/DeliveryBean.xml rename to src/shipping/ingestion/src/main/resources/DeliveryBean.xml diff --git a/src/bc-shipping/ingestion/src/main/resources/application.properties b/src/shipping/ingestion/src/main/resources/application.properties similarity index 100% rename from src/bc-shipping/ingestion/src/main/resources/application.properties rename to src/shipping/ingestion/src/main/resources/application.properties diff --git a/src/bc-shipping/ingestion/src/main/resources/log4j2.xml b/src/shipping/ingestion/src/main/resources/log4j2.xml similarity index 100% rename from src/bc-shipping/ingestion/src/main/resources/log4j2.xml rename to src/shipping/ingestion/src/main/resources/log4j2.xml diff --git a/src/bc-shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java similarity index 100% rename from src/bc-shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java rename to src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java diff --git a/src/bc-shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java similarity index 100% rename from src/bc-shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java rename to src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java diff --git a/src/bc-shipping/package/.gitignore b/src/shipping/package/.gitignore similarity index 100% rename from src/bc-shipping/package/.gitignore rename to src/shipping/package/.gitignore diff --git a/src/bc-shipping/package/app/api.json b/src/shipping/package/app/api.json similarity index 100% rename from src/bc-shipping/package/app/api.json rename to src/shipping/package/app/api.json diff --git a/src/bc-shipping/package/app/controllers/index.ts b/src/shipping/package/app/controllers/index.ts similarity index 100% rename from src/bc-shipping/package/app/controllers/index.ts rename to src/shipping/package/app/controllers/index.ts diff --git a/src/bc-shipping/package/app/controllers/package-controllers.ts b/src/shipping/package/app/controllers/package-controllers.ts similarity index 100% rename from src/bc-shipping/package/app/controllers/package-controllers.ts rename to src/shipping/package/app/controllers/package-controllers.ts diff --git a/src/bc-shipping/package/app/initializer.ts b/src/shipping/package/app/initializer.ts similarity index 100% rename from src/bc-shipping/package/app/initializer.ts rename to src/shipping/package/app/initializer.ts diff --git a/src/bc-shipping/package/app/main.ts b/src/shipping/package/app/main.ts similarity index 100% rename from src/bc-shipping/package/app/main.ts rename to src/shipping/package/app/main.ts diff --git a/src/bc-shipping/package/app/models/api-models.ts b/src/shipping/package/app/models/api-models.ts similarity index 100% rename from src/bc-shipping/package/app/models/api-models.ts rename to src/shipping/package/app/models/api-models.ts diff --git a/src/bc-shipping/package/app/models/package.ts b/src/shipping/package/app/models/package.ts similarity index 100% rename from src/bc-shipping/package/app/models/package.ts rename to src/shipping/package/app/models/package.ts diff --git a/src/bc-shipping/package/app/models/repository.ts b/src/shipping/package/app/models/repository.ts similarity index 100% rename from src/bc-shipping/package/app/models/repository.ts rename to src/shipping/package/app/models/repository.ts diff --git a/src/bc-shipping/package/app/server.ts b/src/shipping/package/app/server.ts similarity index 100% rename from src/bc-shipping/package/app/server.ts rename to src/shipping/package/app/server.ts diff --git a/src/bc-shipping/package/app/util/logging.ts b/src/shipping/package/app/util/logging.ts similarity index 100% rename from src/bc-shipping/package/app/util/logging.ts rename to src/shipping/package/app/util/logging.ts diff --git a/src/bc-shipping/package/app/util/mongo-err.ts b/src/shipping/package/app/util/mongo-err.ts similarity index 100% rename from src/bc-shipping/package/app/util/mongo-err.ts rename to src/shipping/package/app/util/mongo-err.ts diff --git a/src/bc-shipping/package/app/util/settings.ts b/src/shipping/package/app/util/settings.ts similarity index 100% rename from src/bc-shipping/package/app/util/settings.ts rename to src/shipping/package/app/util/settings.ts diff --git a/src/bc-shipping/package/build/create-secrets.sh b/src/shipping/package/build/create-secrets.sh similarity index 100% rename from src/bc-shipping/package/build/create-secrets.sh rename to src/shipping/package/build/create-secrets.sh diff --git a/src/bc-shipping/package/build/dev.dockerfile b/src/shipping/package/build/dev.dockerfile similarity index 100% rename from src/bc-shipping/package/build/dev.dockerfile rename to src/shipping/package/build/dev.dockerfile diff --git a/src/bc-shipping/package/build/docker-compose.ci.build.yml b/src/shipping/package/build/docker-compose.ci.build.yml similarity index 100% rename from src/bc-shipping/package/build/docker-compose.ci.build.yml rename to src/shipping/package/build/docker-compose.ci.build.yml diff --git a/src/bc-shipping/package/build/docker-compose.yml b/src/shipping/package/build/docker-compose.yml similarity index 100% rename from src/bc-shipping/package/build/docker-compose.yml rename to src/shipping/package/build/docker-compose.yml diff --git a/src/bc-shipping/package/build/prod.dockerfile b/src/shipping/package/build/prod.dockerfile similarity index 100% rename from src/bc-shipping/package/build/prod.dockerfile rename to src/shipping/package/build/prod.dockerfile diff --git a/src/bc-shipping/package/down.sh b/src/shipping/package/down.sh similarity index 100% rename from src/bc-shipping/package/down.sh rename to src/shipping/package/down.sh diff --git a/src/bc-shipping/package/gulpfile.js b/src/shipping/package/gulpfile.js similarity index 100% rename from src/bc-shipping/package/gulpfile.js rename to src/shipping/package/gulpfile.js diff --git a/src/bc-shipping/package/package-service.md b/src/shipping/package/package-service.md similarity index 100% rename from src/bc-shipping/package/package-service.md rename to src/shipping/package/package-service.md diff --git a/src/bc-shipping/package/package.json b/src/shipping/package/package.json similarity index 100% rename from src/bc-shipping/package/package.json rename to src/shipping/package/package.json diff --git a/src/bc-shipping/package/test/models/test-data-access.ts b/src/shipping/package/test/models/test-data-access.ts similarity index 100% rename from src/bc-shipping/package/test/models/test-data-access.ts rename to src/shipping/package/test/models/test-data-access.ts diff --git a/src/bc-shipping/package/tsconfig.json b/src/shipping/package/tsconfig.json similarity index 100% rename from src/bc-shipping/package/tsconfig.json rename to src/shipping/package/tsconfig.json diff --git a/src/bc-shipping/package/up.sh b/src/shipping/package/up.sh similarity index 100% rename from src/bc-shipping/package/up.sh rename to src/shipping/package/up.sh diff --git a/src/bc-shipping/scheduler/.classpath b/src/shipping/scheduler/.classpath similarity index 100% rename from src/bc-shipping/scheduler/.classpath rename to src/shipping/scheduler/.classpath diff --git a/src/bc-shipping/scheduler/.project b/src/shipping/scheduler/.project similarity index 100% rename from src/bc-shipping/scheduler/.project rename to src/shipping/scheduler/.project diff --git a/src/bc-shipping/scheduler/Dockerfile b/src/shipping/scheduler/Dockerfile similarity index 100% rename from src/bc-shipping/scheduler/Dockerfile rename to src/shipping/scheduler/Dockerfile diff --git a/src/bc-shipping/scheduler/Dockerfilemaven b/src/shipping/scheduler/Dockerfilemaven similarity index 100% rename from src/bc-shipping/scheduler/Dockerfilemaven rename to src/shipping/scheduler/Dockerfilemaven diff --git a/src/bc-shipping/scheduler/Readme.txt b/src/shipping/scheduler/Readme.txt similarity index 100% rename from src/bc-shipping/scheduler/Readme.txt rename to src/shipping/scheduler/Readme.txt diff --git a/src/bc-shipping/scheduler/conf/Config.properties b/src/shipping/scheduler/conf/Config.properties similarity index 100% rename from src/bc-shipping/scheduler/conf/Config.properties rename to src/shipping/scheduler/conf/Config.properties diff --git a/src/bc-shipping/scheduler/conf/application.conf b/src/shipping/scheduler/conf/application.conf similarity index 100% rename from src/bc-shipping/scheduler/conf/application.conf rename to src/shipping/scheduler/conf/application.conf diff --git a/src/bc-shipping/scheduler/conf/log4j2.xml b/src/shipping/scheduler/conf/log4j2.xml similarity index 100% rename from src/bc-shipping/scheduler/conf/log4j2.xml rename to src/shipping/scheduler/conf/log4j2.xml diff --git a/src/bc-shipping/scheduler/mvn-entrypoint.sh b/src/shipping/scheduler/mvn-entrypoint.sh similarity index 100% rename from src/bc-shipping/scheduler/mvn-entrypoint.sh rename to src/shipping/scheduler/mvn-entrypoint.sh diff --git a/src/bc-shipping/scheduler/pom.xml b/src/shipping/scheduler/pom.xml similarity index 100% rename from src/bc-shipping/scheduler/pom.xml rename to src/shipping/scheduler/pom.xml diff --git a/src/bc-shipping/scheduler/settings-docker.xml b/src/shipping/scheduler/settings-docker.xml similarity index 100% rename from src/bc-shipping/scheduler/settings-docker.xml rename to src/shipping/scheduler/settings-docker.xml diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java diff --git a/src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java similarity index 100% rename from src/bc-shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java From cc692512566cd0e79f8c90ba318164f0318c8334 Mon Sep 17 00:00:00 2001 From: Francis Cheung Date: Thu, 11 Jan 2018 10:36:29 -0800 Subject: [PATCH 007/246] Added instructions for es & fluentd (#85) --- deployment.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/deployment.md b/deployment.md index 6228feec..52ea604c 100644 --- a/deployment.md +++ b/deployment.md @@ -350,8 +350,28 @@ kubectl get all --all-namespaces -l co=fabrikam Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch +```bash +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml +``` + Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd +```bash +# The example elasticsearch yaml files deploy a service named "elasticsearch" +wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml + +# Commenting out X-Pack credentials for demo purposes. +# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads +sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ +kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml +``` + #### Deploy linkerd For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) From 99e51a6e1bef2a2ea7e9f39948e0a9bd0ec336dc Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Fri, 19 Jan 2018 12:18:49 -0800 Subject: [PATCH 008/246] Mention that Bash in deployment instructions --- deployment.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment.md b/deployment.md index 52ea604c..13328bb6 100644 --- a/deployment.md +++ b/deployment.md @@ -15,6 +15,8 @@ Clone or download this repo locally. git clone https://github.com/mspnp/microservices-reference-implementation.git ``` +The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. + ## Create the Kubernetes cluster Set environment variables. From 3405991f979c2a00765a48b3c594a98593bd5378 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 9 Jan 2018 16:20:59 -0300 Subject: [PATCH 009/246] add ARM templates for ACS --- README.md | 2 +- azuredeploy.json | 142 +++++++++++++ deploymentF5.md | 388 ++++++++++++++++++++++++++++++++++++ service-principal-creds.png | Bin 0 -> 14259 bytes 4 files changed, 531 insertions(+), 1 deletion(-) create mode 100644 azuredeploy.json create mode 100644 deploymentF5.md create mode 100644 service-principal-creds.png diff --git a/README.md b/README.md index 8a08259f..585fb5e4 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The Drone Delivery application is a sample application that consists of several ## Deployment -To deploy the solution, follow the steps listed [here](./deployment.md). +To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure you are going to create or just go for a [quick start using the ARM tempplates](./deploymentF5.md). diff --git a/azuredeploy.json b/azuredeploy.json new file mode 100644 index 00000000..a250a25b --- /dev/null +++ b/azuredeploy.json @@ -0,0 +1,142 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "dnsNamePrefix": { + "type": "string", + "metadata": { + "description": "Sets the Domain name prefix for the cluster. The concatenation of the domain name and the regionalized DNS zone make up the fully qualified domain name associated with the public IP address." + } + }, + "agentCount": { + "type": "int", + "defaultValue": 3, + "metadata": { + "description": "The number of agents for the cluster. This value can be from 1 to 100 (note, for Kubernetes clusters you will also get 1 or 2 public agents in addition to these seleted masters)" + }, + "minValue":1, + "maxValue":100 + }, + "agentVMSize": { + "type": "string", + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The size of the Virtual Machine." + } + }, + "adminUsername": { + "type": "string", + "metadata": { + "description": "User name for the Linux Virtual Machines." + }, + "defaultValue" : "azureuser" + }, + "orchestratorType": { + "type": "string", + "defaultValue": "Kubernetes", + "allowedValues": [ + "Kubernetes", + "DCOS", + "Swarm" + ], + "metadata": { + "description": "The type of orchestrator used to manage the applications on the cluster." + } + }, + "masterCount": { + "type": "int", + "defaultValue": 1, + "allowedValues": [ + 1 + ], + "metadata": { + "description": "The number of Kubernetes masters for the cluster." + } + }, + "sshRSAPublicKey": { + "type": "string", + "metadata": { + "description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example 'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm'" + } + }, + "servicePrincipalClientId": { + "metadata": { + "description": "Client ID (used by cloudprovider)" + }, + "type": "securestring" + }, + "servicePrincipalClientSecret": { + "metadata": { + "description": "The Service Principal Client Secret." + }, + "type": "securestring" + } + }, + "variables": { + "agentsEndpointDNSNamePrefix":"[concat(parameters('dnsNamePrefix'),'agents')]", + "mastersEndpointDNSNamePrefix":"[concat(parameters('dnsNamePrefix'),'mgmt')]", + "useServicePrincipalDictionary": { + "DCOS": 0, + "Swarm": 0, + "Kubernetes": 1 + }, + "useServicePrincipal": "[variables('useServicePrincipalDictionary')[parameters('orchestratorType')]]", + "servicePrincipalFields": [ + null, + { + "ClientId": "[parameters('servicePrincipalClientId')]", + "Secret": "[parameters('servicePrincipalClientSecret')]" + } + ] + }, + "resources": [ + { + "apiVersion": "2016-09-30", + "type": "Microsoft.ContainerService/containerServices", + "location": "[resourceGroup().location]", + "name":"[concat('containerservice-',resourceGroup().name)]", + "properties": { + "orchestratorProfile": { + "orchestratorType": "[parameters('orchestratorType')]" + }, + "masterProfile": { + "count": "[parameters('masterCount')]", + "dnsPrefix": "[variables('mastersEndpointDNSNamePrefix')]" + }, + "agentPoolProfiles": [ + { + "name": "agentpools", + "count": "[parameters('agentCount')]", + "vmSize": "[parameters('agentVMSize')]", + "dnsPrefix": "[variables('agentsEndpointDNSNamePrefix')]" + } + ], + "linuxProfile": { + "adminUsername": "[parameters('adminUsername')]", + "ssh": { + "publicKeys": [ + { + "keyData": "[parameters('sshRSAPublicKey')]" + } + ] + } + }, + "servicePrincipalProfile": "[variables('servicePrincipalFields')[variables('useServicePrincipal')]]" + } + } + ], + "outputs": { + "masterFQDN": { + "type": "string", + "value": "[reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).masterProfile.fqdn]" + }, + "sshMaster0": { + "type": "string", + "value": "[concat('ssh ', parameters('adminUsername'), '@', reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).masterProfile.fqdn, ' -A -p 22')]" + }, + "agentFQDN": { + "type": "string", + "value": "[reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).agentPoolProfiles[0].fqdn]" + } + } +} diff --git a/deploymentF5.md b/deploymentF5.md new file mode 100644 index 00000000..34211f61 --- /dev/null +++ b/deploymentF5.md @@ -0,0 +1,388 @@ +# Deploying the Reference Implementation + + + +## Prerequisites + +- Azure suscription +- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Docker](https://docs.docker.com/) +- [Docker Compose](https://docs.docker.com/compose/install/) + +Clone or download this repo locally. + +```bash +git clone https://github.com/mspnp/microservices-reference-implementation.git +``` + +## Create the Kubernetes cluster + +Set environment variables. + +```bash +export LOCATION=[YOUR_LOCATION_HERE] + +export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] + +export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ +export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" +``` + +Infrastructure Prerequisites + +```bash +# Log in to Azure +az login + +# Create a resource group for ACS +az group create --name $RESOURCE_GROUP --location $LOCATION + +# Create an Azure Service Principal for RBAC +az ad sp create-for-rbac --role="Contributor" +``` + +Obtain a SSH rsa public key + +the SSH rsa public key is typically generated by ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could providate the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). + +Create the ACS cluster +* using Azure CLI 2.0 + + ```bash + az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json + ``` + > Note: + > 1. when sshRSAPublicKey parameter prompts you for the string, do not enclose within quotes, or they will be treated as part of the public key. + > 2. Highlighted are the servicePrincipalClientId (appId) and the servicePrincipalClientSecret (password) that you use as service principal parameters for cluster deployment. Please note that this is a secure string and you will not see the input in your screen. + + ![](./service-principal-creds.png) + +* from Azure Portal + + [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Ffeature%2F48_AddARMTemplates%2Fazuredeploy.json) + > Note: + > 1. when deployed using the Preview Portal, you should paste in the contents of your ssh-rsa public key file as a string. + > 2. paste the $RESOURCE_GROUP value and choose use existing resource group + +# Install kubectl +sudo az acs kubernetes install-cli + +# Get the Kubernetes cluster credentials +az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME + +# Create the Shipping BC namespace +kubectl create namespace bc-shipping +``` + +Create an Azure Container Registry instance. + +> Note: Azure Container Registory is not required. If you prefer, you can store the Docker images for this solution in another container registry. + +```bash +export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] + +# Create the ACR instance +az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic + +# Log in to ACR +az acr login --name $ACR_NAME + +# Get the ACR login server name +export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer") + +# Strip quotes +export ACR_SERVER=("${ACR_SERVER[@]//\"/}") +``` + +## Deploy the Delivery service + +Provision Azure resources + +```bash +export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ +export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ +export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ +export COLLECTION_NAME="${DATABASE_NAME}-col" + +# Create Azure Redis Cache +az redis create --location $LOCATION \ + --name $REDIS_NAME \ + --resource-group $RESOURCE_GROUP \ + --sku Premium \ + --vm-size P4 + +# Create Cosmos DB account with DocumentDB API +az cosmosdb create \ + --name $COSMOSDB_NAME \ + --kind GlobalDocumentDB \ + --resource-group $RESOURCE_GROUP \ + --max-interval 10 \ + --max-staleness-prefix 200 + +# Create a Cosmos DB database +az cosmosdb database create \ + --name $COSMOSDB_NAME \ + --db-name=$DATABASE_NAME \ + --resource-group $RESOURCE_GROUP + +# Create a Cosmos DB collection +az cosmosdb collection create \ + --collection-name $COLLECTION_NAME \ + --name $COSMOSDB_NAME \ + --db-name $DATABASE_NAME \ + --resource-group $RESOURCE_GROUP +``` + +Build the Delivery service + +```bash +export DELIVERY_PATH=./microservices-reference-implementation/src/bc-shipping/delivery +docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/Fabrikam.DroneDelivery.DeliveryService/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 + +``` + +Create Kubernetes secrets + +```bash +export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] + +export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ +export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) + +kubectl --namespace bc-shipping create --save-config=true secret generic delivery-storageconf \ + --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ + --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ + --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ + --from-literal=EH_ConnectionString= +``` + +Deploy the Delivery service: + +```bash +# Update the image tag in the deployment YAML +sed -i "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" ./microservices-reference-implementation/k8s/delivery.yaml + +## Update config values in the deployment YAML +sed -i "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ +sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ +sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" + +# Deploy the service +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml +``` + +## Deploy the Package service + +Provision Azure resources + +```bash +export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" +az cosmosdb create --name $COSMOSDB_NAME --kind MongoDB --resource-group $RESOURCE_GROUP +``` + +Build the Package service + +```bash +export PACKAGE_PATH=microservices-reference-implementation/src/bc-shipping/package + +# Build the app +docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up + +# Build the docker image +sudo docker build -f $PACKAGE_PATH/build/prod.dockerfile -t $ACR_SERVER/package-service:0.1.0 $PACKAGE_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/package-service:0.1.0 +``` + +Deploy the Package service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml + +# Create secret +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") +kubectl -n bc-shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} + +# Deploy service +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/package.yml +``` + +## Deploy the Ingestion service +Provision Azure resources + +```bash +export INGESTION_EH_NS=[INGESTION_EVENT_HUB_NAMESPACE_HERE] +export INGESTION_EH_NAME=[INGESTION_EVENT_HUB_NAME_HERE] +export INGESTION_EH_CONSUMERGROUP_NAME=[INGESTION_EVENT_HUB_CONSUMERGROUP_NAME_HERE] + +wget https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/201-event-hubs-create-event-hub-and-consumer-group/azuredeploy.json && \ +sed -i 's#"partitionCount": "4"#"partitionCount": "32"#g' azuredeploy.json && \ +az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ +'{ \ + "namespaceName": {"value": "'${INGESTION_EH_NS}'"}, \ + "eventHubName": {"value": "'${INGESTION_EH_NAME}'"}, \ + "consumerGroupName": {"value": "'${INGESTION_EH_CONSUMERGROUP_NAME}'"} \ +}' +``` +Note: you could also create this from [the Azure Portal](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create) + +Build the Ingestion service + +```bash +export INGESTION_PATH=./microservices-reference-implementation/src/bc-shipping/ingestion + +# Build the app +docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ +docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk + +# Build the docker image +docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/ingestion:0.1.0 +``` + +Deploy the Ingestion service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" ./microservices-reference-implementation/k8s/ingestion.yaml + +# Get the EventHub shared access policy name and key from the Azure Portal +export EH_ACCESS_KEY_NAME=[YOUR_SHARED_ACCESS_POLICY_NAME_HERE] +export EH_ACCESS_KEY_VALUE=[YOUR_SHARED_ACCESS_POLICY_VALUE_HERE] + +# Create secret +kubectl -n bc-shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ +--from-literal=eventhub_name=${INGESTION_EH_NAME} \ +--from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ +--from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} + +# Deploy service +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml +``` + +## Deploy the Scheduler service + +Provision Azure resources +```bash +export SCHEDULER_STORAGE_ACCOUNT_NAME=[SCHEDULER_STORAGE_ACCOUNT_NAME_HERE] + +az storage account create --resource-group $RESOURCE_GROUP --name $SCHEDULER_STORAGE_ACCOUNT_NAME --sku Standard_LRS +``` + +Build the Scheduler service + +```bash +export SCHEDULER_PATH=./microservices-reference-implementation/src/bc-shipping/scheduler + +# Build the app +docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ +docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk + +# Build the docker image +docker build -f $SCHEDULER_PATH/Dockerfile -t $ACR_SERVER/scheduler:0.1.0 $SCHEDULER_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/scheduler:0.1.0 +``` + +Deploy the Scheduler service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" ./microservices-reference-implementation/k8s/scheduler.yaml + +# Get the following values from the Azure Portal +export EH_CONNECTION_STRING="[YOUR_EVENT_HUB_CONNECTION_STRING_HERE]" +export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] +export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" + +# Create secrets +kubectl -n bc-shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ +--from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ +--from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ +--from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ +--from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} + +# Deploy service +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml +``` + +## Deploy mock services + +Build the mock services + +```bash +export MOCKS_PATH=microservices-reference-implementation/src/bc-shipping/delivery +docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ +docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ +docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/account:0.1.0 && \ +docker push $ACR_SERVER/dronescheduler:0.1.0 && \ +docker push $ACR_SERVER/thirdparty:0.1.0 +``` + +Deploy the mock services: + +```bash +# Update the image tag in the deployment YAML +sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ +sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ +sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml + +# Deploy the service +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/account.yaml && \ +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ +kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml +``` + +## Verify all services are running: + +```bash +kubectl get all -n bc-shipping +``` + +Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch + +Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd + +Deploy linkerd. For more information, see https://linkerd.io/getting-started/k8s/ + +Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy + +It is recommended to put an API Gateway in front of all APIs you want exposed to the public, +however for convenience, we exposed the Ingestion service with a public IP address. + +You can send delivery requests to the ingestion service using the swagger ui. + +```bash +export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace bc-shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST +``` diff --git a/service-principal-creds.png b/service-principal-creds.png new file mode 100644 index 0000000000000000000000000000000000000000..e3161993c2257694c9c2be9d44b69621d6742cdc GIT binary patch literal 14259 zcmaibbyOV9w=F?J1`qCT!QCaeyE}usy9N(VaCdii3GPmCcX!tZ`M%%1>)pH7dw+CK z&zVy))m_!K&)z3kR$2rW3KI$h1O!%0R8SrSx@N?`H;}u-*Fyz(!Qf9s~rg z=i~SkM+t}ZZiI9YlM;g50HB}}QJr<_|6>wTb>MS!FtN4+DJmZkefNTKcsB_e+Uwhy z+Ble6TY(?}2#DV8P#<;yYa3TPQ)3eckW%;&j(02EU#p!V@WU0~!PL?ag!v90@cs+n z;}->MBL`=oogs*p&D)1N^xw)&?e#5y@2++r4Zmw6-_2h>%ql=TD^n|Dkl$FlLGL!m zf66=B=^KLFy!jpd$4$W6($dh%0pxZhzT@5Y`NK`o)WO0Kr0{f#3;4wcd3DJf9iyL$s3DN1o=E*R>Ab=Y18$~zml{70kPhB5O7d$=ldbWE% z9aVI^Li&8iNBE-r`|WY%E5a9W(9R6MLPc;;pYN3nf_a4R7T-7m3X~70QW&|ScLyI` z!=K_p?`6O#1r#aX9r0B$#!D25c_*hCVY=6;PTlj!0q<_>ZI>MxSNb|kBlU`8NL;%b zdv)2wANAc0)woeeFNinDp`wI*G;Tz=`)`ZsT1D61*E&SFAJyK2sW!|1I;Mcr$mlql ztH8kf=tQf2d~zYYF%0OD=c%avOcc5$SJBF_2&w82FTxo1`9qc<)i z9WOU?I&W7kL0IFKZnPf4l2K@U~ac;pL3nBiQzLNQ3?}j`556JJb$E zAFZD!Hn1oWxcxh963P@n27!P|+OZ^H6rjVZ#|sYcqUNG8>5YW}NO}rXQVGofqGNO= zrdVMb+xGCA`>bPxeH((+PgL>#iuIJ+5idOF^^_sjYUNr_k)DV2igUkd+{029<9)mu zwal4+>7HGSeF-Nrh@UaX18U=K92IWZbjnTkCGFY=O{2!&F43J0ziEDsh};TGIdiEs zD~f0L`DtI$>6)0}3{&~X6NmA)G}^k=&B|XkCuBcAr+U7i z*u^Yw-AnPpbea=9FY-1N`2Qf6V#250mp(v+_gM2=95lTJ%^VF=ib7(1EKTXZ)51Bc^9Q5}^ZQA6J)55OY=KqsZSdV$bGM^vm4r(#n zu-(Nz#Eo|`LXhR>IrNM68pNG#%%?y`4L;hk@n=5K@qrwsd%q%Tz)5KKnP+x7z+U#3 zoI~0KU#iVXl&1Ny@h?vncgscP12P4f^VMI^whUj%aEOAG$)^*QzRh==cqK`nNuhn2%7lvI6ig~a;}*OE zXnL#oQ1r);5fzz2@tK4fKAlQMw~?6zhucK^>+RG>5d$@+lTG~FKc0qo?AKzO)&c|$ z(@*ShaUfM!Roz%E`vp39&A$rdo^z^6CHn?!YO(gNN4T_yB5~l2=@>+g`()S|VPJ9u zc@B*ivc>IAVn*Ow#v?TVO#%Vmk39}?`>K$}V(G+9XZGSFn*?INhuMz|$pF&mVA|q1 ziIFMgPrg^g=gs;@W^lppjY%-NVr6KOl;xR>B3ddV`ol?+H@Hna2scgk>N)EMo)YeB z6==Nh*Dv}=2PnSAm;VW`Zeg=vUhk8qjd`5So-2`)jBTnZ-BLP%=e-mJxO&?pwPm+B z=!xBL>5xq?Ady(d4*0YFfW2bh&Vf-7lb?WZ`0_Z{?zQaO^YnaCS=NL@7`9q{7Rm-Tv-1latsfjNZGjS8u>FkK1dj(o3`^ zH#lxaWrd)edTlM?p=Rc<7ZkN7%w1utnOZI@iJD;Ckr5|3^8~si`iItE34%E_i@+Kw zA|mpp?26qWCml8$oN*@4@|H*+;`q^+zn)_ulb=pOWO3E_`8-CnW#0`%gGfeIN!Ay` zDuH&#i@=GP^4gngyS2%5cF4}fruteo$)f6eMG!_aT1%P4){w`bVTLfrabE~TN_RWH z1s+XHfK!ere3T}I+u}F}{!$=(kAUJ*;4oDY35OQ^H96j1;~@utR56#2ZOSrT^_25G zO><~ndQ~{o?Yq1fc$xzZB6foNIk(7AqrmDc9X#0=4o*$th;Mh9Dm*u5i@iggSU0H zS$X?A2Dity+2@r(h_O%n<;7~&Tb=S^Wz0Vr*uCMf699FdGy=Qe-T-zUs0gu zcIZGWSg;d)Wh=%yEr*)vd$qFl7u%IUCe*M=+sOt*AFDFR;)x_NhI~VDFA;sM4;5m7 zq-BXbw+){JnAfF!b*G>(TXMCuFXoGNCj!&C-k1B+NOh|xYHtwmvpR$9hQ zaneIpr!7I>l+ti#va@_ICauNcjPu_nUfUNN7dbNc-oK~Ls)sh_m59!ICc(^^B;*kF z8)_7jslVHq64yyJiXo+#P^j5LJ}+aabv-bYt)f1v>{GM9HrIRp*vbEa%M^6hm4xu3 zjm5%5xP;dzPL}S)5J}2$MjmcFe@3=l!PV2sayrJ8k9E{`g>2~U^W!SL10sb07;!V^Ocy{oVHb?QoaZq-R?{Q;o-O{*)5McsuA! z@!Mxqot_9ZfCkgg8q3U4vo2bnr)iP>+N1@CDlt{6BW(1~kXzn(g6ht_WNjPu?Lirw1cqaZ(@oEelEMjw|KB zBOOzD2bqB8>;eXu*))K+CBhb|TCua;TEF5$9(Ob=)m^~HR;gUkw%NKL;hX8cafuYW zM1psJuyyTBWBtxgjferwR6<$t$jJ7i8E?Lv*i<1H=JnAy&oA>4ihwC9tiE92M z^(#8(w;#a!1octHGY6nl7Z2+5&oTJN7mZ85hmj@+%v7@Sb1bjp2sIyWA1$KwdRgA# zvGRE*CJ*yF#hcBrrg&h0z3}ZIoM+^$@43}v%bpBnjj;k=0YutFJbrMp;}s)e2uO9+}i^v>Y|>yot!sl z@aM%ey6Pucm;o#d!;R@ipKoI9B?P^6J-KC=8p*bR73eU&h*w9N{04+mhIJBO-nT@z z9wZN2#qGrRD-Ss~*IQl_@!lTzGnAXG6K9@ns?H$9a2JZbFX7BX<0gk^COosJCCEl& zmGG3|3ox#qx(Oo>_@m}ITC_4SmgD`s za;v}T+OyyCWmMlEWn{Jv3qs2Ra!iP8tt2CRxBBIfr^+8djC2D+94S+P_JsCCse()XDZUHU&tB0x)in`_lc?neZff(i zEfsgkMiYbyqxR+(zsS+i1HP17C2fzyKgr0l3-h)?So>?*n%$^W{7k2sUitEin#=1g z=;~V_P#4+3Y3D4JbQfkpS|2`#t&FMSs>c0WJ6(Gn%}uCUTwq$|X3t{l?uBnQLO+pptFHC2K*mVGw3K@`~c_fwOX_@&xa0<766&G|2xacf|OKba@R*3)3C z#aRU%Mz_pf5sfvQ-^N+%OC$!wJLZFBP*Z->;Sv$M9!D*|o?uilDGy|Q6Z#8?;)(Q; zRa>m(xuss?p&qXU1^O3~H3aYf z5A?Y;80_|e{l26Rm+bp@Ct)N@%zbt3&WzPOiAdQP=|-!wmMeRxY0WRb&nV*anitd6 z3%vm-Mw5UMc^jP8QX)L-z52I4oOUQ+Sh=8$q2H@uoa^qXO1y-v|1KnNfAt>UW>d6- zaduJtMmfPLQSOQOh$~I1v@qQQ4xf~Fk`~Vp&^&(+eRHnS0MikvSp4fNnei}wqb$rv)Bx7EJ*8gOR)8KEeLzdlGl>``*A{f1R zsOIp27}w`ZrtZaFxm6f{lmQN=hZG^5!a8&oN3QTz zNay_tY%iiHPYE6`?#ZgaxP8hOrOM3omrk$p!P9p$cErzGB6g_cU7WEkbS~fLi`9)D z#)G@P-p_=f*y7|Msc1Q1!o}LP3yQ&FqnO>--2{`M7e(7f4UCB_v1dw zem0&H#dX_P$u0~LXPaK1)&{r4;KOxN?3LQnBr7Zw42Lrhxy7H0(JUOSNIktVef4+q z$lZdYD3tRXOZG)|Md*d7H?*<`Dw2ri(^O$_=z0&r0{3_$Ax!IuiB@iVboO}7?-?(5 zqq*aT65U#S{0$fegJMv5Ii^-K0Ooo00h`g4i1&_Rrf0YEUkt(l+7~>X{HB zc~Xtmm**si33cwJ6T>#L>`AH)UPPZR!vf9W&-&fVqu&_XD55h=I7V(OnVBXL^``>3 z?dT*or#ld@Ue_<1+@D8a-KCT1=d-11>n0767qi*8}vj{t>L6Bk{_Yr}!&IwyZ8){r5-8tTk zC_9qynZ*8FBWLBe;J^XXnUFmzO>r}($i4L9qE|T6WMAt#JC>c$cknfK+0z!)Q@O!k zi!-)RW&=|}ICD!VPVa4X2x(mt7%8ii2oRp=R(}dHnLSRrZqI9kLtc=t&va|UKSyoq zp4it(vk1>eT#tb)WXLb$=}cQTvJc*u_D|z zt2cRNG9ptGpmEaeXQk26m5zRfLVHn~FmPv`hEn>vxK8f*v%z@wI0SQ7&#Yh_XkNl3 z!EXqkb+TZZ_F1H?s;wxEtSrUd;$|$k*eG=1t7^RxqTqswyf}nhEdHY*e1E#IDySPo z@eR*W`Ff$yw-a|;**&w7=IJq=Y5St%5h$&N_bo4MY<|Kuw!6s&(8RovR|~1bM1N_u zTRU8jiFfnz350VaH?Tp6fMW_kDVV;1>4vkLi;-_&GGoR=VLN9jKeih`tN0u9bsja% z1A)Xvc&UEWY64~%S32AENimxu1e0_+N{(=nq(7GA`2rkr4A<~wLrcBh3Vl`Oa(Cw) zA0MPW^j+|UeaFTWRcPR^qW#=H7M=W&tMXI7{+ank=~hn|LGzz9IfSNkFORtPB4}xt}1e1Ap_Or4khQWqBQ81sykom9A&dB#7 z29^7u0=>M=HYb{Az=Rk1CCu*UczNat4j3?#o{VJSdu<>{$O{{BEGZTpB2F%LvOCrA z{XI9if6Y%to4npkR^o)gKiwQSgQQ@*ZM=c5NBye2U^hN@$n08)_nt#kto>wJ#9=`<6Y3K8d_G=xb8J%1SlCbit&m*z`U9225YB%H1UKk@-QI z4Vn7ZV!#OtNQg zmCL9K9oWEek0YqLl=!4En?yX+`&)z+BpTXsJw>da)ZO~3Tl<`AH!lg}{@59av2x4% z1EuV>TKCs)st4An(sS%L!@VT{W`CZC79dNLjA~@hm)R?7b&z%aF4sC~&Tg*JM0SONj}ma?&`m(&1LVIjt$m zd0b9QSI8rXuP%WCcknbd^yy3OiFmhMI1LzcHVi&`<~I3w@Vndb0Qb(TeJXLrbT71e zR@9^Vw@A(`w&tIKpZ*ZNXyN@0q3MKg5SITZI)pl!s|PLE7T%+)IM@$aA0T>LD61 z`&(;6PIU^js2N;M0Z)N=)G@j(<~GeU2=V?pyuD0D=DGP2H)sqx{cU_~F%0e^ZvJ_* z$6>1HT=CL17_Ev8ZNz5P{=oOhnRj>f42deAOx}R$U}$?nc44A<^Hl>>=i~()1`(Yy zUGF<}AmHrKOzFC|y~#wsAffgoS#A;$?2y8$V76S)U5*R`8o|vE;fJ7=%nG|G8J<#i zsrl#SnEeoWkLtu^r{aF8qrV>*uJTP69Q!x`3Y~yaJDg=Yi@iNwhc)*s1?(wMf z^sT~3F1aZibBP5S+zvgb5I!i&%m7*?RTz#O9nv91CDz(7mc;z08O5L}7ymh1KlwgJ zw`7g!uTYXIVrJjqE|Z-J6LOPB?obT$sts1~X#vqsFhrHt6qQ-4NjD&5@WBktPRw!J ziXjMGRJz#G9o^G&EO84}RZq78o~4dQ21buld%$*MdExW4B8Ow$0ZVegD`oqY(ku zCdKeLOn3YQabbgAy;fh)ZfFJioO8DtR)EBU0q-2lDuZ7-oaLdgUElNK(t3tQ5loOk zvy^@G#mNofdmXxA?4b;+2-aTre~FDV$PI6 zl{zrS!e(l=<}k2{uX5vWErD~=^zDZutD}ZN+iJnr)qy$Ap%Vb;RUUpas>E@xJ^!UbLXk&qsiNnk{$t*;XW-AdiynU(>F$GD=}Vo=%~BE2|!8$DhDtN$xF*C zP5M60Md`(;O~AQw=jPrqzVGr==uUTPH|j37HY_IvjRcPV?p&G^zc$&4_-Ukjl{Z`# z(m#dYx8Q)MpBbjQngo;?Y7L+(9LD-BK|EzE%;R-4OKhxjS=wFqnFy*uEzy0z@?QSl zV{S>Abb>Wl_EUe_r<*$q{D)mUj6h5RP#}D#Obs43Bb7UzCUCGwL=#_8CS-f z&dUwr$tgp~g7=r|UeJ@-DcTb5?cRm5tZ|D!rXfyNmB6*}!)=DjX{GMM)+tlzj(&`zDUzu%vfB4l!MdVVm=gB}}L*>A|Q zdRad8u|-NE%FEqXk9W&)83a2Ea<*zm=rv>tUtH51%pXT22_3rL>l%JP#zpr1Av$2~ z2Z|VPCTHwKj|-s5SFSDA+C=tBUV}rN$a@SxJ6pC+j{Jr4E|H#}I^HJ#`nuH#L8!}&jFeV_bV5!KSk-=!dgLu;=_b-k-FK-$V3U~ zH)Vm>d4}1_xYq-KcfbI=uVZU|sk;}7-hJg|{QO{D+WH3Ym%is_sM9zj@+{ldp#v>Cjg4_B%|@SL%bl4BB2K`wF|x#0WB4!d2D7I&I3>7B>+! z=}G&`n;7j0kj70G$sW8D&Z$X&O|FiCe8 zn#Fnu^tmJKg`b9!-LFV~IaO}?w#9{uJVr6e51nH7V;2**%%+u<3zt}}ZXFJXZp8EI zsvoYoy}zLE;Jd17)*1VEmAs9B0+!sFCX4Qu37j_kw}Eyd=_`&?X}E!zx~0tL zLjEFMRCY_!p{ESpLGfNmQG1C?>>%+U^8+OZL|#t9XfwvsARwrAA6ftp9il78i98O8 znhts=A`Q%g=Y`A`rA>3UjG%7B@)sr@iSE;RhXx5F5xlc<*r3&s)5Wbx#5ji{K|5!g z)5&i-L3`;tVi6)#shw=0H7!_Vyp6Ad%iA~+(j5`yPlaE9aS8%@*o<}IVA3Ql&y2`Y zzX4;X3ddKRoPqOT9RKiLft**x9!~0o8F(pWInr^*57eFok@)MFY+1cqV8 zd5Ms4-L>tbo^9JPNv3lAZxKeZVi|xeQRn>~(maqdv8{3p-D_WBzF};OEOAQ|9w7G3 zv3Z%w&4>ZseCKwM}e-?m8A^p4Of{0P_DEHKWwI|nBAhaL4G zOw7S7&X!&u%2BeI5D|OX)UgTglb z%jRJuYM||8Pp5V9!`+IM z%{6fV-M?6lOCMeHed4V8x+xtS1*t&Kne~&|TzP?|3?lYH_h7>G2RHVfE>`iN4`#zu zZxdR|GR5j9DROktA%)p9R4P{;-B4f)IISb^JJlzm&tkUtVzYYeP+7asMauH4Y~e&F+06m|5$B# z>{IHxaF~Xqq?9e3QuiI)GB{nb!b9@3IFvE*d)#lNKlAZcgZ*|M1tJe@{YrCUbXkP3 zu@zI6tJe-54m3c-a2v#& zHo}<@uw#iyV`P~@=n_c=d;K_W^v*_d2X=f?dH6lFUAi(?G~yuID2oCa--4Oow>CbC zcL_;#oy4{LDaEs|j`d5*>3(NlMd;Od!xosHOk+qlMh$4VrFy5?X~c8mrw&ZzVx$NZ z^J_Qe>OZ8YEnsKs^Mq-1s+F!UTt%Ig0~QwURWi(^hiNqN@XERg21SDEy-XmtoJ?jk zzN?PSe^QIK^8}bvHmkX3DIhBynnwEuX5FsKX-yr<2Bd%gi@N1bK-y5-j8kwlw&B3Y zeZEkxNiwB&(!qojAy^$MmIszW`JG8YOLqi{^f_b`dzQjR(8v#P7*Y_{FX=fr*0V}NTsEAR@1KA?+W;RnDS?cc7514o#yT-e zIuvXj4YtThB-{>k+cBdHsl@k2Uo%xty=R7JKy>O5o@(fmbmDihl%kkE-4}Y;0|W}V z4W^v-RBn!>jpz)#hg?5Z4uL`UgQnR~N>E+-7|@AkBV>Dfi?BOO)-$+ZwUda>I}nQd zry^#(D@zik$7vjRbPt?N2)Cj$r*WMal3I;3qe2zdJ3P#G)Tcy;3MXHDjpAVP#ZX{th)2-=KWM@!GI^WEuwA=1pB5c5Vh zM7>79dpPj0ZjzmLYVD@_D5}!QOmiC80%=|iT?kvg&0o)7QFmMn35}}Y|HT4d&8MMW zwrF~t%Qj%`oYC&q(;fGxATUmWm&k>@vzyPXJggQj z9HSJqi}Wn+3H$xMeDgfVEGFaF8X&}E)s7Z&16*lziQTwBT3#C3#;ebwg}s2acM1gQ z_C0{Q31^M(qmF&XXN3?!Id4VVXzQ_yf4JeMIZ}&%EM&GEZiC(HG~65RHx4PsHCI&S zYw%jP*YD0Zf5tW%c&$E%T!9I`63y;hl_}X|<@{S8c>BBG9(jjbeU4q+k+(`l+9h}L zaDq+7W{340o^j3h`Qk;w|NXVuG6;Iox3j&x6A@^Al3uKh{q!WWow3$46xcx(UGyW4 z8%U^)b&}l4%q2m#Z0=bwd3rHJu27&=u!_OCxIa8Yo4m`vJ|mKGJP`RIx!tM z$;E)bHQD82X=x=LAVxEjJ68X^d{KcWIYS$I%qR7z+EF2w?zLwK zsbQ9CF)(IHFubibt~{Wv!_pVnPpnxSI5(TGT}}<>&XDTUd`fA7MBt$Emrwm9@ZHiR^$8#GDRxkUQE*uV!m)7C5@*;!0v zB(hwncNu|*e#{DVmv&E}u%KFwoTN=Og$<>pJ@Q@TF7}?bJKsMmhy~0-4kA2T6;nrw zJzD4xa!wzP)l4pjzUT}4q8Xs&?Pumvm)2+^3L}>p;@Ts3-N6Ve>zU8-Jb$<`V8QyE zL4PEi`!~ZW1;<|bhdm`*+gMh*u!?>Ad_Cn^1+}NtoO_JKS!K8X@XcV zk&~8o_Q?kKYPWNk}Nv06ZToGNl(pju{MYo zNtbhLj3{4-hx0gn5GNbC8g)5~ss>>XN?nP_79^Gd0Y9C@a&ZM!ENa6GF%qu8drv^K zY>?mn(2uUSR@PjRrIYa5YIV-C{7m+VA-7Wa5)k}YA^J7V{sc6F{eowpVieYW(%>aU ziZTDvvqPdqj1?+(E60J!?ceYF%P7bN2Xh;r3S& zPQxdeN5#H7kED*fgd|xz*G7uJ(F1SLn48~kz4oz!Zx$Qk@W0&jW~4miXg?&60R~~9 z=66!X`b20o|ArP&?Fz{yUaTPEhUOVGKhVf?j#{z&S2N(pHI?7_|DxRrAJCCK}iDX1RuWMfUD`L54+Rx2C zqn)a1hu;ynMjFL209=_v4>WSAf21a0X>`l1wD*YY7{}jRI<2_)ZR{-(I^2`dbbhAB z;#p!{5Icnrr29pD-NDz;c;)KE>#;jKx5$reBT}y0`8g7iPP~oNx^?F`CeNW`rjKRX zd@Hu^&W5328W~iX6T@?^XWkyM=_0^Ml2;1EP#~8%JLK9fzpi&;R7;YczwGqg={_X7 z*P}v=-YN2{C=yFYx^0AW$)p~gd%D&L?>w!>1zlojXre1dBvwe6hry}jt$Ir9X|~8) zBImvArkw(az@=V?kr!uY1~S%ej~|OjCi!cH+N>V~1(lG6NK#D+yjRH>PWV%Tu8+m5Vh zeK6lDw)v&Jb{r~XuIijp8-gaE8PaaIZ}6D<{obx67MHK?$fZ^TZ(n*jfI80>x#4wI zeMa!T7RNEK>TYy^K4&~9HQ2xELeh?inaqd%dzuog7=edR%Y_!rzG6w6GUr7Ed1qCJ*_0VrM4-E`5{wdHLmy1 z%a$|EtG3YZk~e_$7!E#DGc-h#IchA>&)OyYC0FZ-Rl|ta$+70#nIa`q*?s}G5TpD% z6#`z%&>900wJo%YTQuv2z>eegRr6$OuOJf5C$URoKM(ejJsz)&keFA?U|Rj9)3ZKf z(>}BS?$|p|0*#aJ1^g7=5_5;b3lQ^vIXI$`+RLG(`)vwSY9aI8{2c|TD-12@P)p6W z^rXcmwSRSmyU9i*PD=2XS^>{~d%6z{dnm3Kf}T4r@IrFrD@oC5a?~7$kq;9$+&s+E zkcZx}}I6)vKwRri4uQlI1UfN=0!ird{o zc0Njd=$_2ojqNg*$Z5mwX^>WBG{x<;KUvQU(9FHWAKa>mI)^CdZ+SDhQHHllQ`cC1MKO&E1GrR1Pq16IJ2bKXY`# z4Ei-3Xi8oL@QwHk(RQB;+S4kFKFyN4Rptt=<9$}~SE%-U!8-j%an#Q1h-gz@mu+~m zjlh|0evC&ar|iDcovuN-Jojf9R4wjW7)Ms(NcY&VmX%#lqPQ!cITOF>Gh2*%$?JS--PU5U-BFg6KS&2Y`@NUeN+l zd2~o_l>wnYqFSZ|Hy(!NiWIXDu_XZ#0Ek;26DeuQ%PaeO3sQ}O`HBxmYY=%ivKGLI z{wW{EPhw)|>oDg4o=L7HVuVrfey!8ouf6XQ`Z3r{5|bj2>N2n3%)j#Yo_|ZZsfQmC zc)vr1I~%V?0`Br1t>6CyBPI2a&O}q!G^9xVqpBTAom$i-C~;M zl^;{W6EVsF)W8NtyFxT9=V_1GZG^1H1;w{y^a}m9%<;^N!Ey=HV8q_ecUuW_uAlmi z$Mok&$P`V>ae|2{gxkpF=v~dnBzR+O?{%Hx@J$(})J@_FHdt>aumYZaVVUjD=@&n&@q zPfNH!qijltc0p)B3}}YN-bOmSaY|_40b@XXH9;_5!jIF4nm)n&)DYmt{=7`PloaIT=(o>#*xXdFQ8Y5A#fHY(=`br!iF6>#&Ujc}D1r1#unF zE>b6_p)-vY#Vei*o^9`t`yP&0az?B=p#R7;5VeP~dJAK_Sqc(9zNz=O0&wlWg?#5= z{|Nb16a+PJJr*sfY@#HjG0Zzeu;6(LP(qo%k9`SyA$-@{g zjOe^4?r}U%TAymb<@C@RM5|!S6F>8*hc^0c=;={g{=&2Kees{Z!|NI%zhh2990wja zy?rxUNeu*8Gv{^Xn8HZUPUx=ozQVeyMx{;8%XOUFn%5bV&YD75^P8$B{zhUzLbBjm zJl<%yDsfQKohC)Ib=p&L=R+8Z^-iB*}bndXt+Nstt zUN3eJzUL#<%2FgY3@mL`hDl+h-2kSqz2eNC_nGn3q-<^by6`0A=Cx&30~#r?y$JwNa%bH<89{Vu(>#bm|I0GQV=OaGZ`pZBj+d(I-nJ=b}(R z4bYMzVQ)Lw;G%+A-QXw~1fGpOf1jj-Q5ZE}AvQwS$D@No8-i)aP=l9TR=KDZ6p$B1 z3C(q)j=gHoj}s44KEhqvD^0qSr?A_K?O|*`x%x}UX>ytUY{&UG=|&Yp?~L|izxe;i zJ<68<|Bzi(nt2S%Y*t-{AK6Txoy5aR6z_=_vG{|@hC~0_z{b&E4`m0}rs|VvkUrY` zlJB~jDqeEeI*R>Huzf#|X*{+s*6RU>aYb=3Y_>ThloZ?ds6#o7R-Be8eLCY591 zN72_wE(NkuL;#x!N*6Y=Myr-o*oWB*a{Rbxz@xI(^e5Ec^eILJq??b;@IUEO(D5oE zKR^6ddB+PC4Y+`4^7yAu4nJ#pH3l)`l6@rxoIF`Lehfe{JOsX2gDVFZ2$lO literal 0 HcmV?d00001 From ae1b3f576e6dd6aebe3d99129fd825a493f44faa Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 9 Jan 2018 17:18:13 -0300 Subject: [PATCH 010/246] add ARM template for ACR --- azuredeploy.json | 71 +++++++++++++++++++++++++++++++++++++++++++++++- deploymentF5.md | 14 ++++------ 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index a250a25b..edbf7fa6 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -70,6 +70,33 @@ "description": "The Service Principal Client Secret." }, "type": "securestring" + }, + "acrName": { + "type": "string", + "minLength": 5, + "maxLength": 50, + "metadata": { + "description": "Name of your Azure Container Registry" + } + }, + "acrAdminUserEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Enable admin user that have push / pull permission to the registry." + } + }, + "acrStorageType": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Standard_LRS", + "Standard_ZRS", + "Standard_GRS" + ], + "metadata": { + "description": "Type of the storage account that will store container registry datas." + } } }, "variables": { @@ -87,7 +114,9 @@ "ClientId": "[parameters('servicePrincipalClientId')]", "Secret": "[parameters('servicePrincipalClientSecret')]" } - ] + ], + "acrStorageName": "[concat('acr', uniqueString(resourceGroup().id))]", + "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]" }, "resources": [ { @@ -123,6 +152,42 @@ }, "servicePrincipalProfile": "[variables('servicePrincipalFields')[variables('useServicePrincipal')]]" } + }, + { + "name": "[variables('acrStorageName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "comments": "This storage account is used by Container Registry for storing its datas.", + "dependsOn": [], + "tags": { + "displayName": "ACR Image's storage", + "container.registry": "[parameters('acrName')]" + }, + "properties": { + "accountType": "[parameters('acrStorageType')]" + } + }, + { + "name": "[parameters('acrName')]", + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2016-06-27-preview", + "location": "[resourceGroup().location]", + "comments": "Container registry for storing docker images", + "dependsOn": [ + "[variables('acrStorageId')]" + ], + "tags": { + "displayName": "Container Registry", + "container.registry": "[parameters('acrName')]" + }, + "properties": { + "adminUserEnabled": "[parameters('acrAdminUserEnabled')]", + "storageAccount": { + "accessKey": "[listKeys(variables('acrStorageId'),'2015-06-15').key1]", + "name": "[variables('acrStorageName')]" + } + } } ], "outputs": { @@ -137,6 +202,10 @@ "agentFQDN": { "type": "string", "value": "[reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).agentPoolProfiles[0].fqdn]" + }, + "acrLoginServer": { + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',parameters('acrName')),'2016-06-27-preview').loginServer]", + "type": "string" } } } diff --git a/deploymentF5.md b/deploymentF5.md index 34211f61..2d03cf45 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -23,6 +23,7 @@ Set environment variables. export LOCATION=[YOUR_LOCATION_HERE] export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] +export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" @@ -53,7 +54,7 @@ Create the ACS cluster ``` > Note: > 1. when sshRSAPublicKey parameter prompts you for the string, do not enclose within quotes, or they will be treated as part of the public key. - > 2. Highlighted are the servicePrincipalClientId (appId) and the servicePrincipalClientSecret (password) that you use as service principal parameters for cluster deployment. Please note that this is a secure string and you will not see the input in your screen. + > 2. Highlighted are the servicePrincipalClientId (appId) and the servicePrincipalClientSecret (password) that you use as service principal parameters for cluster deployment. Please note that these params are secure strings, so when you enter or paste the value it is not going to be displayed in your screen. ![](./service-principal-creds.png) @@ -63,7 +64,10 @@ Create the ACS cluster > Note: > 1. when deployed using the Preview Portal, you should paste in the contents of your ssh-rsa public key file as a string. > 2. paste the $RESOURCE_GROUP value and choose use existing resource group + > 3. paste the $ACR_NAME value in the acraname parameter +Download kubectl and create a k8s namespace +```bash # Install kubectl sudo az acs kubernetes install-cli @@ -74,15 +78,9 @@ az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUST kubectl create namespace bc-shipping ``` -Create an Azure Container Registry instance. - -> Note: Azure Container Registory is not required. If you prefer, you can store the Docker images for this solution in another container registry. +Obtain your Azure Container Registry instance details. ```bash -export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] - -# Create the ACR instance -az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic # Log in to ACR az acr login --name $ACR_NAME From ed84401b4832d030fa862b3a2b9ab75a59c2710c Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 10 Jan 2018 18:05:15 -0300 Subject: [PATCH 011/246] add ARM template for Redis Cache (delivery) --- README.md | 2 +- azuredeploy.json | 132 ++++++++++++++++++++++++++++++++++++++++++++++- deploymentF5.md | 20 +++---- 3 files changed, 142 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 585fb5e4..a6f86af7 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The Drone Delivery application is a sample application that consists of several ## Deployment -To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure you are going to create or just go for a [quick start using the ARM tempplates](./deploymentF5.md). +To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure you are going to create or just go for a [quick start using the ARM templates](./deploymentF5.md). diff --git a/azuredeploy.json b/azuredeploy.json index edbf7fa6..5eb8c27f 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -97,6 +97,77 @@ "metadata": { "description": "Type of the storage account that will store container registry datas." } + }, + "deliveryRedisStorageType": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Standard_LRS", + "Standard_ZRS", + "Standard_GRS" + ], + "metadata": { + "description": "Type of the storage account that will store Redis Cache." + } + }, + "delivery_redisCacheName": { + "type": "string", + "metadata": { + "description": "The name of the Azure Redis Cache to create." + } + }, + "delivery_redisCacheSKU": { + "type": "string", + "allowedValues": [ + "Basic", + "Standard", + "Premium" + ], + "defaultValue": "Premium", + "metadata": { + "description": "The pricing tier of the new Azure Redis Cache." + } + }, + "delivery_redisCacheFamily": { + "type": "string", + "defaultValue": "P", + "metadata": { + "description": "The family for the sku." + }, + "allowedValues": [ + "C", + "P" + ] + }, + "delivery_redisCacheCapacity": { + "type": "int", + "allowedValues": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "defaultValue": 1, + "metadata": { + "description": "The size of the new Azure Redis Cache instance. " + } + }, + "delivery_redisEnableNonSslPort": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "A boolean value that indicates whether to allow access via non-SSL ports." + } + }, + "delivery_redisDiagnosticsEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "A value that indicates whether diagnostics should be saved to the specified storage account." + } } }, "variables": { @@ -116,7 +187,9 @@ } ], "acrStorageName": "[concat('acr', uniqueString(resourceGroup().id))]", - "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]" + "deliveryRedisStorageName": "[concat('red', uniqueString(resourceGroup().id))]", + "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]" }, "resources": [ { @@ -188,6 +261,63 @@ "name": "[variables('acrStorageName')]" } } + }, + { + "name": "[variables('deliveryRedisStorageName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "comments": "This storage account is used by Delivery Redis", + "dependsOn": [], + "tags": { + "displayName": "Delivery Redis storage" + }, + "properties": { + "accountType": "[parameters('deliveryRedisStorageType')]" + } + }, + { + "apiVersion": "2015-08-01", + "name": "[parameters('delivery_redisCacheName')]", + "type": "Microsoft.Cache/Redis", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('deliveryRedisStorageId')]" + ], + "properties": { + "redisEnableNonSslPort": "[parameters('delivery_redisEnableNonSslPort')]", + "sku": { + "capacity": "[parameters('delivery_redisCacheCapacity')]", + "family": "[parameters('delivery_redisCacheFamily')]", + "name": "[parameters('delivery_redisCacheSKU')]" + }, + "vm-size": "P4" + }, + "resources": [ + { + "apiVersion": "2017-05-01-preview", + "type": "Microsoft.Cache/redis/providers/diagnosticsettings", + "name": "[concat(parameters('delivery_redisCacheName'), '/Microsoft.Insights/', parameters('delivery_redisCacheName'))]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[concat('Microsoft.Cache/Redis/', parameters('delivery_redisCacheName'))]" + ], + "properties": { + "storageAccountId": "[variables('deliveryRedisStorageId')]", + "logs": [], + "metrics": [ + { + "timeGrain": "AllMetrics", + "enabled": "[parameters('delivery_redisDiagnosticsEnabled')]", + "retentionPolicy": { + "days": 90, + "enabled": "[parameters('delivery_redisDiagnosticsEnabled')]" + } + } + ] + } + } + ] } ], "outputs": { diff --git a/deploymentF5.md b/deploymentF5.md index 2d03cf45..11472e7d 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -26,7 +26,8 @@ export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ -export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" +export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ +export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" ``` Infrastructure Prerequisites @@ -50,7 +51,13 @@ Create the ACS cluster * using Azure CLI 2.0 ```bash - az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json + az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ + '{ \ + "acrName": {"value": "'${ACR_NAME}'"}, \ + "dnsNamePrefix": {"value": "'${UNIQUE_APP_NAME_PREFIX}'"}, \ + "delivery_redisCacheName": {"value": "'${REDIS_NAME}'"} + }' + ``` > Note: > 1. when sshRSAPublicKey parameter prompts you for the string, do not enclose within quotes, or they will be treated as part of the public key. @@ -97,18 +104,11 @@ export ACR_SERVER=("${ACR_SERVER[@]//\"/}") Provision Azure resources ```bash -export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ + export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" -# Create Azure Redis Cache -az redis create --location $LOCATION \ - --name $REDIS_NAME \ - --resource-group $RESOURCE_GROUP \ - --sku Premium \ - --vm-size P4 - # Create Cosmos DB account with DocumentDB API az cosmosdb create \ --name $COSMOSDB_NAME \ From 333f40638df864aa72853493e79d3e924b6fdcda Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 12:59:21 -0300 Subject: [PATCH 012/246] add ARM template for CosmosDb --- azuredeploy.json | 36 ++++++++++++++++++++++++------------ deploymentF5.md | 16 +++------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 5eb8c27f..2f211ad3 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -110,12 +110,6 @@ "description": "Type of the storage account that will store Redis Cache." } }, - "delivery_redisCacheName": { - "type": "string", - "metadata": { - "description": "The name of the Azure Redis Cache to create." - } - }, "delivery_redisCacheSKU": { "type": "string", "allowedValues": [ @@ -187,9 +181,11 @@ } ], "acrStorageName": "[concat('acr', uniqueString(resourceGroup().id))]", - "deliveryRedisStorageName": "[concat('red', uniqueString(resourceGroup().id))]", "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", - "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]" + "deliveryRedisStorageName": "[concat('red', uniqueString(resourceGroup().id))]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-cosmosdb')]", + "deliveryRedisName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-redis')]" }, "resources": [ { @@ -278,12 +274,12 @@ }, { "apiVersion": "2015-08-01", - "name": "[parameters('delivery_redisCacheName')]", + "name": "[variables('deliveryRedisName')]", "type": "Microsoft.Cache/Redis", "location": "[resourceGroup().location]", "dependsOn": [ "[variables('deliveryRedisStorageId')]" - ], + ], "properties": { "redisEnableNonSslPort": "[parameters('delivery_redisEnableNonSslPort')]", "sku": { @@ -297,10 +293,10 @@ { "apiVersion": "2017-05-01-preview", "type": "Microsoft.Cache/redis/providers/diagnosticsettings", - "name": "[concat(parameters('delivery_redisCacheName'), '/Microsoft.Insights/', parameters('delivery_redisCacheName'))]", + "name": "[concat(variables('deliveryRedisName'), '/Microsoft.Insights/', variables('deliveryRedisName'))]", "location": "[resourceGroup().location]", "dependsOn": [ - "[concat('Microsoft.Cache/Redis/', parameters('delivery_redisCacheName'))]" + "[concat('Microsoft.Cache/Redis/', variables('deliveryRedisName'))]" ], "properties": { "storageAccountId": "[variables('deliveryRedisStorageId')]", @@ -318,6 +314,22 @@ } } ] + }, + { + "apiVersion": "2015-04-08", + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "[variables('deliveryCosmosDbName')]", + "location": "[resourceGroup().location]", + "properties": { + "name": "[variables('deliveryCosmosDbName')]", + "databaseAccountOfferType": "['Standard']", + "locations": [ + { + "locationName": "[resourceGroup().location]", + "failoverPriority": 0 + } + ] + } } ], "outputs": { diff --git a/deploymentF5.md b/deploymentF5.md index 11472e7d..c44f617d 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -27,7 +27,8 @@ export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ -export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" +export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ +export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" ``` Infrastructure Prerequisites @@ -54,8 +55,7 @@ Create the ACS cluster az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ '{ \ "acrName": {"value": "'${ACR_NAME}'"}, \ - "dnsNamePrefix": {"value": "'${UNIQUE_APP_NAME_PREFIX}'"}, \ - "delivery_redisCacheName": {"value": "'${REDIS_NAME}'"} + "dnsNamePrefix": {"value": "'${UNIQUE_APP_NAME_PREFIX}'"} }' ``` @@ -104,19 +104,9 @@ export ACR_SERVER=("${ACR_SERVER[@]//\"/}") Provision Azure resources ```bash - -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" -# Create Cosmos DB account with DocumentDB API -az cosmosdb create \ - --name $COSMOSDB_NAME \ - --kind GlobalDocumentDB \ - --resource-group $RESOURCE_GROUP \ - --max-interval 10 \ - --max-staleness-prefix 200 - # Create a Cosmos DB database az cosmosdb database create \ --name $COSMOSDB_NAME \ From 37c359babe379a0f385e774a15a9e8f9bd09a196 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 13:12:30 -0300 Subject: [PATCH 013/246] add ARM template for MongoDb --- azuredeploy.json | 14 +++++++++++++- deploymentF5.md | 12 +++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 2f211ad3..248f8d94 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -185,7 +185,8 @@ "deliveryRedisStorageName": "[concat('red', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", "deliveryCosmosDbName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-cosmosdb')]", - "deliveryRedisName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-redis')]" + "deliveryRedisName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-redis')]", + "packageMongoDbName": "[concat(parameters('dnsNamePrefix'),'-package-service-cosmosdb')]" }, "resources": [ { @@ -330,6 +331,17 @@ } ] } + }, + { + "apiVersion": "2015-04-08", + "type": "Microsoft.DocumentDB/databaseAccounts", + "kind": "MongoDB", + "name": "[variables('packageMongoDbName')]", + "location": "[resourceGroup().location]", + "properties": { + "databaseAccountOfferType": "Standard", + "name": "[variables('packageMongoDbName')]" + } } ], "outputs": { diff --git a/deploymentF5.md b/deploymentF5.md index c44f617d..b290c468 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -28,7 +28,8 @@ export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" +export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ +export MONGODB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" ``` Infrastructure Prerequisites @@ -172,13 +173,6 @@ kubectl --namespace bc-shipping apply -f ./microservices-reference-implementatio ## Deploy the Package service -Provision Azure resources - -```bash -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" -az cosmosdb create --name $COSMOSDB_NAME --kind MongoDB --resource-group $RESOURCE_GROUP -``` - Build the Package service ```bash @@ -202,7 +196,7 @@ Deploy the Package service sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml # Create secret -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $MONGODB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") kubectl -n bc-shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} # Deploy service From a174a9c0d923a7cf9305b44e1760cdaf52c83fe6 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 13:41:29 -0300 Subject: [PATCH 014/246] add ARM template for Ingestion Event Hub --- azuredeploy.json | 45 ++++++++++++++++++++++++++++++++++++++++++++- deploymentF5.md | 22 ++++------------------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 248f8d94..d7eb4295 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -186,7 +186,10 @@ "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", "deliveryCosmosDbName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-cosmosdb')]", "deliveryRedisName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-redis')]", - "packageMongoDbName": "[concat(parameters('dnsNamePrefix'),'-package-service-cosmosdb')]" + "packageMongoDbName": "[concat(parameters('dnsNamePrefix'),'-package-service-cosmosdb')]", + "ingestionEHNamespace": "[concat(parameters('dnsNamePrefix'),'ingehns')]", + "ingestionEHName": "[concat(parameters('dnsNamePrefix'),'ingeh')]", + "ingestionEHConsumerGroupName": "[concat(parameters('dnsNamePrefix'),'ingehcg')]" }, "resources": [ { @@ -342,6 +345,46 @@ "databaseAccountOfferType": "Standard", "name": "[variables('packageMongoDbName')]" } + }, + { + "apiVersion":"2017-04-01", + "name":"[variables('ingestionEHNamespace')]", + "type":"Microsoft.EventHub/Namespaces", + "location":"[resourceGroup().location]", + "sku":{ + "name":"Standard" + }, + "properties": { + "isAutoInflateEnabled": "true", + "maximumThroughputUnits": "20" + }, + "resources":[ + { + "apiVersion":"2017-04-01", + "name":"[variables('ingestionEHName')]", + "type":"EventHubs", + "dependsOn":[ + "[concat('Microsoft.EventHub/namespaces/', variables('ingestionEHNamespace'))]" + ], + "properties":{ + "messageRetentionInDays": "7", + "partitionCount": "4" + }, + "resources":[ + { + "apiVersion":"2017-04-01", + "name":"[variables('ingestionEHConsumerGroupName')]", + "type":"ConsumerGroups", + "dependsOn":[ + "[variables('ingestionEHName')]" + ], + "properties":{ + "userMetadata": "Consumer Group for ingestion Event Hub" + } + } + ] + } + ] } ], "outputs": { diff --git a/deploymentF5.md b/deploymentF5.md index b290c468..452946f3 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -29,7 +29,10 @@ export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ -export MONGODB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" +export MONGODB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" && \ +export INGESTION_EH_NS="${UNIQUE_APP_NAME_PREFIX}ingehns" && \ +export INGESTION_EH_NAME="${UNIQUE_APP_NAME_PREFIX}ingeh" && \ +export INGESTION_EH_CONSUMERGROUP_NAME="${UNIQUE_APP_NAME_PREFIX}ingehcg" ``` Infrastructure Prerequisites @@ -204,23 +207,6 @@ kubectl --namespace bc-shipping apply -f ./microservices-reference-implementatio ``` ## Deploy the Ingestion service -Provision Azure resources - -```bash -export INGESTION_EH_NS=[INGESTION_EVENT_HUB_NAMESPACE_HERE] -export INGESTION_EH_NAME=[INGESTION_EVENT_HUB_NAME_HERE] -export INGESTION_EH_CONSUMERGROUP_NAME=[INGESTION_EVENT_HUB_CONSUMERGROUP_NAME_HERE] - -wget https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/201-event-hubs-create-event-hub-and-consumer-group/azuredeploy.json && \ -sed -i 's#"partitionCount": "4"#"partitionCount": "32"#g' azuredeploy.json && \ -az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ -'{ \ - "namespaceName": {"value": "'${INGESTION_EH_NS}'"}, \ - "eventHubName": {"value": "'${INGESTION_EH_NAME}'"}, \ - "consumerGroupName": {"value": "'${INGESTION_EH_CONSUMERGROUP_NAME}'"} \ -}' -``` -Note: you could also create this from [the Azure Portal](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create) Build the Ingestion service From 7e8934d153472b34662d6351166128f37bea627f Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 13:55:06 -0300 Subject: [PATCH 015/246] add ARM template for Scheduler Storage Account --- azuredeploy.json | 29 ++++++++++++++++++++++++++++- deploymentF5.md | 9 +-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index d7eb4295..5f8ad605 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -162,6 +162,18 @@ "metadata": { "description": "A value that indicates whether diagnostics should be saved to the specified storage account." } + }, + "schedulerStorageType": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Standard_LRS", + "Standard_ZRS", + "Standard_GRS" + ], + "metadata": { + "description": "Type of the storage account that will store Scheduler checkpoints." + } } }, "variables": { @@ -189,7 +201,8 @@ "packageMongoDbName": "[concat(parameters('dnsNamePrefix'),'-package-service-cosmosdb')]", "ingestionEHNamespace": "[concat(parameters('dnsNamePrefix'),'ingehns')]", "ingestionEHName": "[concat(parameters('dnsNamePrefix'),'ingeh')]", - "ingestionEHConsumerGroupName": "[concat(parameters('dnsNamePrefix'),'ingehcg')]" + "ingestionEHConsumerGroupName": "[concat(parameters('dnsNamePrefix'),'ingehcg')]", + "schedulerStorageName": "[concat('sch', uniqueString(resourceGroup().id))]" }, "resources": [ { @@ -385,6 +398,20 @@ ] } ] + }, + { + "name": "[variables('schedulerStorageName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "comments": "This storage account is used by the Scheduler checkpoints.", + "dependsOn": [], + "tags": { + "displayName": "Scheduler Checkpoints storage" + }, + "properties": { + "accountType": "[parameters('schedulerStorageType')]" + } } ], "outputs": { diff --git a/deploymentF5.md b/deploymentF5.md index 452946f3..f7cc4299 100644 --- a/deploymentF5.md +++ b/deploymentF5.md @@ -32,7 +32,7 @@ export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ export MONGODB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" && \ export INGESTION_EH_NS="${UNIQUE_APP_NAME_PREFIX}ingehns" && \ export INGESTION_EH_NAME="${UNIQUE_APP_NAME_PREFIX}ingeh" && \ -export INGESTION_EH_CONSUMERGROUP_NAME="${UNIQUE_APP_NAME_PREFIX}ingehcg" +export INGESTION_EH_CONSUMERGROUP_NAME="${UNIQUE_APP_NAME_PREFIX}ingehcg" ``` Infrastructure Prerequisites @@ -247,13 +247,6 @@ kubectl --namespace bc-shipping apply -f ./microservices-reference-implementatio ## Deploy the Scheduler service -Provision Azure resources -```bash -export SCHEDULER_STORAGE_ACCOUNT_NAME=[SCHEDULER_STORAGE_ACCOUNT_NAME_HERE] - -az storage account create --resource-group $RESOURCE_GROUP --name $SCHEDULER_STORAGE_ACCOUNT_NAME --sku Standard_LRS -``` - Build the Scheduler service ```bash From 15d4b3b329368546ddef81d055821fcea3c25933 Mon Sep 17 00:00:00 2001 From: cfarre Date: Fri, 19 Jan 2018 17:07:32 -0800 Subject: [PATCH 016/246] controller tests --- src/shipping/ingestion/pom.xml | 15 ++- .../controller/IngestionController.java | 2 +- .../ingestion/IngestionControllerTest.java | 127 ++++++++++++++---- 3 files changed, 111 insertions(+), 33 deletions(-) diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 7c33cbf8..9143a500 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -57,12 +57,21 @@ types 1.0.5 - + + - org.mockito - mockito-core + org.mockito + mockito-core + 2.13.0 + test + + + + + + junit junit diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java index a06b3e46..5fe5b38a 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java @@ -103,7 +103,7 @@ public CompletableFuture> scheduleDeliveryAsync externalDelivery.getDeadline()); externalDelivery.setDeliveryId(delivery.getDeliveryId()); - response.setHeader("Content-Type", "application/json; charset=utf-8 "); + response.setHeader("Content-Type", "application/json;charset=utf-8"); response.setHeader("Location", "http://deliveries/api/deliveries/" + delivery.getDeliveryId()); // Extract the headers as a map and pass on to eventhub message diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index d0b71186..318f429d 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -1,9 +1,16 @@ package com.fabrikam.dronedelivery.ingestion; +import static org.hamcrest.CoreMatchers.any; +import static org.hamcrest.CoreMatchers.containsString; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.refEq; +import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import java.util.Date; +import java.util.Map; import java.util.UUID; import org.junit.After; @@ -11,110 +18,171 @@ import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; //import org.mockito.runners.MockitoJUnitRunner; - +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; //import org.springframework.test.web.servlet.setup.MockMvcBuilders; //import org.springframework.web.context.WebApplicationContext; //import org.springframework.web.context.WebApplicationContext; +import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; +import com.fabrikam.dronedelivery.ingestion.controller.IngestionController; import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; +import com.fabrikam.dronedelivery.ingestion.models.Delivery; +import com.fabrikam.dronedelivery.ingestion.models.DeliveryBase; import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; import com.fabrikam.dronedelivery.ingestion.models.ExternalRescheduledDelivery; import com.fabrikam.dronedelivery.ingestion.models.PackageInfo; +import com.fabrikam.dronedelivery.ingestion.service.Ingestion; +import com.fabrikam.dronedelivery.ingestion.service.IngestionImpl; import com.fasterxml.jackson.databind.ObjectMapper; import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.springframework.test.context.junit4.SpringRunner; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; - +import java.util.*; @RunWith(SpringRunner.class) @SpringBootTest public class IngestionControllerTest { - @Autowired - private WebApplicationContext wac; private MockMvc mockMvc; private PackageInfo packageInfo; - private ExternalDelivery delivery; + private ExternalDelivery externalDelivery; ExternalRescheduledDelivery externalRDelivery; + + @Mock + private IngestionImpl ingestionimplMock; + + @Mock + private ApplicationProperties appPropsMock; + + @InjectMocks + private IngestionController ingestionController; @Before public void setUp() throws Exception { - mockMvc = webAppContextSetup(this.wac).build(); + + + + + MockitoAnnotations.initMocks(this); + mockMvc = MockMvcBuilders + .standaloneSetup(ingestionController) + .build(); + + packageInfo = new PackageInfo(); packageInfo.setSize(ContainerSize.Large); packageInfo.setPackageId(UUID.randomUUID().toString()); - delivery = new ExternalDelivery(); - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(new Date()); - delivery.setDropOffLocation("Austin"); - delivery.setPickupLocation("Texas"); - delivery.setExpedited(false); - delivery.setConfirmationRequired(ConfirmationRequired.FingerPrint); - delivery.setDeadline("LineOfDeadlyZombiatedPeople"); - delivery.setPackageInfo(packageInfo); + externalDelivery = new ExternalDelivery(); + externalDelivery.setOwnerId(UUID.randomUUID().toString()); + externalDelivery.setPickupTime(new Date()); + externalDelivery.setDropOffLocation("Austin"); + externalDelivery.setPickupLocation("Texas"); + externalDelivery.setExpedited(false); + externalDelivery.setConfirmationRequired(ConfirmationRequired.FingerPrint); + externalDelivery.setDeadline("LineOfDeadlyZombiatedPeople"); + externalDelivery.setPackageInfo(packageInfo); externalRDelivery = new ExternalRescheduledDelivery(); externalRDelivery.setDeadline("deadline"); externalRDelivery.setDeliveryId(UUID.randomUUID().toString()); externalRDelivery.setDropOffLocation("location"); externalRDelivery.setPickupLocation("location"); + + } @After public void tearDown() throws Exception { } - - @Ignore + @Test - public void ScheduleDeliveryIsOk() throws Exception { + public void ScheduleDeliveryIsAccepted() throws Exception { + + Mockito.doNothing().when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + MvcResult resultActions = mockMvc.perform( - post("/api/deliveryrequests").content(asJsonString(delivery)).contentType(MediaType.APPLICATION_JSON)) - .andExpect(request().asyncStarted()).andReturn(); - - mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)) + .andExpect(header().string("Content-Type", containsString("application/json"))) + .andExpect(status().isAccepted()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } - @Ignore @Test public void RescheduleDeliveryIsOk() throws Exception { + + Mockito.doNothing().when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); MvcResult resultActions = mockMvc.perform(patch("/api/deliveryrequests/" + deliveryId) .content(asJsonString(externalRDelivery)).contentType(MediaType.APPLICATION_JSON)) .andExpect(request().asyncStarted()).andReturn(); - mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + mockMvc.perform(asyncDispatch(resultActions)) + .andExpect(header().string("Content-Type", containsString("application/json"))) + .andExpect(status().is2xxSuccessful()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } - @Ignore @Test public void CancelDeliveryIsOk() throws Exception { String deliveryId = externalRDelivery.getDeliveryId().toString(); + Mockito.doNothing().when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); MvcResult resultActions = mockMvc .perform(delete("/api/deliveryrequests/" + deliveryId).contentType(MediaType.APPLICATION_JSON)) .andExpect(request().asyncStarted()).andReturn(); - - mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + + mockMvc.perform(asyncDispatch(resultActions)) + .andExpect(status().is2xxSuccessful()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } - @Ignore + @Test public void ProbeDeliveryIsOk() throws Exception { MvcResult resultActions = mockMvc.perform(get("/api/probe/").contentType(MediaType.APPLICATION_JSON)) @@ -122,7 +190,8 @@ public void ProbeDeliveryIsOk() throws Exception { mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); } - + + private static String asJsonString(final Object obj) { try { final ObjectMapper mapper = new ObjectMapper(); From 898d7b87d025639d08e334fced7ae36fa4a62270 Mon Sep 17 00:00:00 2001 From: cfarre Date: Sun, 21 Jan 2018 15:24:43 -0800 Subject: [PATCH 017/246] exception tests --- src/shipping/ingestion/pom.xml | 16 +- .../ingestion/util/ClientPoolImpl.java | 3 +- .../IngestionControllerLoggerTest.java | 166 ++++++++++++++++++ .../ingestion/IngestionControllerTest.java | 136 ++++++++++++-- 4 files changed, 297 insertions(+), 24 deletions(-) create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 9143a500..e91b6893 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -66,11 +66,19 @@ test + + org.powermock + powermock-module-junit4-legacy + 1.7.1 + test + + + org.powermock + powermock-api-mockito2 + 1.7.1 + test + - - - - junit diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java index f8f219eb..7a1cbf12 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java @@ -51,8 +51,7 @@ public EventHubClient getConnection() eventHubClients[poolId] = EventHubClient.createFromConnectionString(connectionString.toString()).get(); } - EventHubClient vclient = eventHubClients[poolId]; - + return eventHubClients[poolId]; } diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java new file mode 100644 index 00000000..28dd2870 --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java @@ -0,0 +1,166 @@ +package com.fabrikam.dronedelivery.ingestion; + +import static org.hamcrest.CoreMatchers.containsString; +import org.apache.logging.log4j.LogManager; +import static org.mockito.Mockito.times; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import com.microsoft.azure.servicebus.ServiceBusException; + + +import java.util.Date; +import java.util.UUID; + +import org.apache.logging.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +//import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.legacy.PowerMockRunner; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +//import org.springframework.test.web.servlet.setup.MockMvcBuilders; +//import org.springframework.web.context.WebApplicationContext; +//import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; + +import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; +import com.fabrikam.dronedelivery.ingestion.controller.IngestionController; +import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; +import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; +import com.fabrikam.dronedelivery.ingestion.models.DeliveryBase; +import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; +import com.fabrikam.dronedelivery.ingestion.models.ExternalRescheduledDelivery; +import com.fabrikam.dronedelivery.ingestion.models.PackageInfo; +import com.fabrikam.dronedelivery.ingestion.service.IngestionImpl; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + +import java.io.IOException; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({LogManager.class}) +public class IngestionControllerLoggerTest { + + private MockMvc mockMvc; + private PackageInfo packageInfo; + private ExternalDelivery externalDelivery; + ExternalRescheduledDelivery externalRDelivery; + + @Mock + private IngestionImpl ingestionimplMock; + + @Mock + private ApplicationProperties appPropsMock; + + @Mock + private Logger loggerMock; + + @InjectMocks + private IngestionController ingestionController; + + @Before + public void setUp() throws Exception { + + + + + MockitoAnnotations.initMocks(this); + mockMvc = MockMvcBuilders + .standaloneSetup(ingestionController) + .build(); + + + packageInfo = new PackageInfo(); + packageInfo.setSize(ContainerSize.Large); + packageInfo.setPackageId(UUID.randomUUID().toString()); + + externalDelivery = new ExternalDelivery(); + externalDelivery.setOwnerId(UUID.randomUUID().toString()); + externalDelivery.setPickupTime(new Date()); + externalDelivery.setDropOffLocation("Austin"); + externalDelivery.setPickupLocation("Texas"); + externalDelivery.setExpedited(false); + externalDelivery.setConfirmationRequired(ConfirmationRequired.FingerPrint); + externalDelivery.setDeadline("LineOfDeadlyZombiatedPeople"); + externalDelivery.setPackageInfo(packageInfo); + + externalRDelivery = new ExternalRescheduledDelivery(); + externalRDelivery.setDeadline("deadline"); + externalRDelivery.setDeliveryId(UUID.randomUUID().toString()); + externalRDelivery.setDropOffLocation("location"); + externalRDelivery.setPickupLocation("location"); + + + PowerMockito.mockStatic(LogManager.class); + Mockito.when(LogManager.getLogger(IngestionController.class)).thenReturn(loggerMock); + + + + + + + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void ScheduleDeliveryAsyncLogs() throws Exception { + + Mockito.doNothing().when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + MvcResult resultActions = mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)) + .andExpect(header().string("Content-Type", containsString("application/json"))) + .andExpect(status().isAccepted()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + + } + + + + + + + + + private static String asJsonString(final Object obj) { + try { + final ObjectMapper mapper = new ObjectMapper(); + final String jsonContent = mapper.writeValueAsString(obj); + return jsonContent; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index 318f429d..4d2c8c8f 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -1,65 +1,55 @@ package com.fabrikam.dronedelivery.ingestion; -import static org.hamcrest.CoreMatchers.any; import static org.hamcrest.CoreMatchers.containsString; -import static org.mockito.Matchers.anyMap; -import static org.mockito.Matchers.eq; -import static org.mockito.Matchers.refEq; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import com.microsoft.azure.servicebus.ServiceBusException; + import java.util.Date; -import java.util.Map; import java.util.UUID; +import org.apache.logging.log4j.Logger; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.InjectMocks; //import org.mockito.runners.MockitoJUnitRunner; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; - -import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.web.context.WebApplicationContext; //import org.springframework.test.web.servlet.setup.MockMvcBuilders; //import org.springframework.web.context.WebApplicationContext; //import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; import com.fabrikam.dronedelivery.ingestion.controller.IngestionController; import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; -import com.fabrikam.dronedelivery.ingestion.models.Delivery; import com.fabrikam.dronedelivery.ingestion.models.DeliveryBase; import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; import com.fabrikam.dronedelivery.ingestion.models.ExternalRescheduledDelivery; import com.fabrikam.dronedelivery.ingestion.models.PackageInfo; -import com.fabrikam.dronedelivery.ingestion.service.Ingestion; import com.fabrikam.dronedelivery.ingestion.service.IngestionImpl; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.ObjectMapper; -import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.springframework.test.context.junit4.SpringRunner; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import java.util.*; -@RunWith(SpringRunner.class) -@SpringBootTest +import java.io.IOException; + + public class IngestionControllerTest { private MockMvc mockMvc; @@ -72,6 +62,9 @@ public class IngestionControllerTest { @Mock private ApplicationProperties appPropsMock; + + @Mock + private Logger loggerMock; @InjectMocks private IngestionController ingestionController; @@ -192,6 +185,113 @@ public void ProbeDeliveryIsOk() throws Exception { } + @Test + public void IngestionControllerHandlesServiceBusException() throws Exception { + + Mockito.doThrow(new RuntimeException(new ServiceBusException(true,"message"))) + .when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void IngestionControllerHandlesIllegalArgumentException() throws Exception { + + Mockito.doThrow(new RuntimeException(new IllegalArgumentException())) + .when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void IngestionControllerHandlesIOException() throws Exception { + + Mockito.doThrow(new RuntimeException(new IOException())) + .when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void IngestionControllerHandlesJsonProcessingException() throws Exception { + + Mockito.doThrow(new RuntimeException(new JsonParseException(null, "error"))) + .when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + @Test + public void IngestionControllerHandlesMethodArgumentTypeMismatchException() throws Exception { + + Mockito.doThrow(new RuntimeException(new MethodArgumentTypeMismatchException(appPropsMock, null, null, null, null))) + .when(ingestionimplMock) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + mockMvc.perform( + post("/api/deliveryrequests") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + + + + + private static String asJsonString(final Object obj) { try { final ObjectMapper mapper = new ObjectMapper(); From 4c17b22a8685bdae82bf6b131fed0ebb9a5c0904 Mon Sep 17 00:00:00 2001 From: cfarre Date: Tue, 23 Jan 2018 18:50:37 -0800 Subject: [PATCH 018/246] ingestion controller tests --- src/shipping/ingestion/pom.xml | 22 +- .../ingestion/ApplicationPropertiesTest.java | 48 ++++ .../IngestionControllerLoggerTest.java | 166 ------------ .../ingestion/IngestionControllerTest.java | 253 +++++++++++++++++- 4 files changed, 307 insertions(+), 182 deletions(-) create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java delete mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index e91b6893..28ddf400 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -66,18 +66,19 @@ test - + org.powermock - powermock-module-junit4-legacy + powermock-module-junit4 1.7.1 test - + - org.powermock + org.powermock powermock-api-mockito2 1.7.1 - test - + test + + @@ -111,6 +112,15 @@ springfox-swagger2 2.7.0 + + + + org.springframework.boot + spring-boot-starter-test + 1.5.9.RELEASE + test + + io.springfox diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java new file mode 100644 index 00000000..f762cfdf --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java @@ -0,0 +1,48 @@ +package com.fabrikam.dronedelivery.ingestion; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.times; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; + +public class ApplicationPropertiesTest { + + public class Environment { + public String getenv(String envName) { + return System.getenv(envName); + } + } + + + @Configuration + public static class Config { + @Bean + public static PropertySourcesPlaceholderConfigurer propertyConfigurer() { + return new PropertySourcesPlaceholderConfigurer(); + } + } + + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + } + + @Test + public void canGetPropertiesfromEnvVariables() { + Environment systemMock = Mockito.mock(Environment.class); + Mockito.when(systemMock.getenv("ENV_HUB_NAME")).thenReturn("variableMock"); + ApplicationProperties appProps = new ApplicationProperties(); + assertEquals(systemMock.getenv(appProps.getEnvHubName()),"variableMock"); + Mockito.verify(systemMock, times(1)) + .getenv(appProps.getEnvHubName()); + } + +} diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java deleted file mode 100644 index 28dd2870..00000000 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerLoggerTest.java +++ /dev/null @@ -1,166 +0,0 @@ -package com.fabrikam.dronedelivery.ingestion; - -import static org.hamcrest.CoreMatchers.containsString; -import org.apache.logging.log4j.LogManager; -import static org.mockito.Mockito.times; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import com.microsoft.azure.servicebus.ServiceBusException; - - -import java.util.Date; -import java.util.UUID; - -import org.apache.logging.log4j.Logger; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -//import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.legacy.PowerMockRunner; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -//import org.springframework.test.web.servlet.setup.MockMvcBuilders; -//import org.springframework.web.context.WebApplicationContext; -//import org.springframework.web.context.WebApplicationContext; -import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; - -import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; -import com.fabrikam.dronedelivery.ingestion.controller.IngestionController; -import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; -import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; -import com.fabrikam.dronedelivery.ingestion.models.DeliveryBase; -import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; -import com.fabrikam.dronedelivery.ingestion.models.ExternalRescheduledDelivery; -import com.fabrikam.dronedelivery.ingestion.models.PackageInfo; -import com.fabrikam.dronedelivery.ingestion.service.IngestionImpl; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; - -import java.io.IOException; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({LogManager.class}) -public class IngestionControllerLoggerTest { - - private MockMvc mockMvc; - private PackageInfo packageInfo; - private ExternalDelivery externalDelivery; - ExternalRescheduledDelivery externalRDelivery; - - @Mock - private IngestionImpl ingestionimplMock; - - @Mock - private ApplicationProperties appPropsMock; - - @Mock - private Logger loggerMock; - - @InjectMocks - private IngestionController ingestionController; - - @Before - public void setUp() throws Exception { - - - - - MockitoAnnotations.initMocks(this); - mockMvc = MockMvcBuilders - .standaloneSetup(ingestionController) - .build(); - - - packageInfo = new PackageInfo(); - packageInfo.setSize(ContainerSize.Large); - packageInfo.setPackageId(UUID.randomUUID().toString()); - - externalDelivery = new ExternalDelivery(); - externalDelivery.setOwnerId(UUID.randomUUID().toString()); - externalDelivery.setPickupTime(new Date()); - externalDelivery.setDropOffLocation("Austin"); - externalDelivery.setPickupLocation("Texas"); - externalDelivery.setExpedited(false); - externalDelivery.setConfirmationRequired(ConfirmationRequired.FingerPrint); - externalDelivery.setDeadline("LineOfDeadlyZombiatedPeople"); - externalDelivery.setPackageInfo(packageInfo); - - externalRDelivery = new ExternalRescheduledDelivery(); - externalRDelivery.setDeadline("deadline"); - externalRDelivery.setDeliveryId(UUID.randomUUID().toString()); - externalRDelivery.setDropOffLocation("location"); - externalRDelivery.setPickupLocation("location"); - - - PowerMockito.mockStatic(LogManager.class); - Mockito.when(LogManager.getLogger(IngestionController.class)).thenReturn(loggerMock); - - - - - - - } - - @After - public void tearDown() throws Exception { - } - - @Test - public void ScheduleDeliveryAsyncLogs() throws Exception { - - Mockito.doNothing().when(ingestionimplMock) - .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); - - MvcResult resultActions = mockMvc.perform( - post("/api/deliveryrequests") - .contentType(MediaType.APPLICATION_JSON) - .content(asJsonString(externalDelivery))) - .andExpect(request().asyncStarted()).andReturn(); - - mockMvc.perform(asyncDispatch(resultActions)) - .andExpect(header().string("Content-Type", containsString("application/json"))) - .andExpect(status().isAccepted()); - - Mockito.verify(ingestionimplMock, times(1)) - .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); - - } - - - - - - - - - private static String asJsonString(final Object obj) { - try { - final ObjectMapper mapper = new ObjectMapper(); - final String jsonContent = mapper.writeValueAsString(obj); - return jsonContent; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - -} diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index 4d2c8c8f..598a4bc4 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -10,7 +10,6 @@ import java.util.Date; import java.util.UUID; -import org.apache.logging.log4j.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -49,6 +48,8 @@ import java.io.IOException; +//@RunWith(SpringJUnit4ClassRunner.class) +//@SpringApplicationConfiguration(classes = ApplicationProperties.class) public class IngestionControllerTest { @@ -63,9 +64,6 @@ public class IngestionControllerTest { @Mock private ApplicationProperties appPropsMock; - @Mock - private Logger loggerMock; - @InjectMocks private IngestionController ingestionController; @@ -73,7 +71,6 @@ public class IngestionControllerTest { public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); mockMvc = MockMvcBuilders @@ -186,7 +183,7 @@ public void ProbeDeliveryIsOk() throws Exception { @Test - public void IngestionControllerHandlesServiceBusException() throws Exception { + public void scheduleDeliveryHandlesServiceBusException() throws Exception { Mockito.doThrow(new RuntimeException(new ServiceBusException(true,"message"))) .when(ingestionimplMock) @@ -206,7 +203,7 @@ public void IngestionControllerHandlesServiceBusException() throws Exception { } @Test - public void IngestionControllerHandlesIllegalArgumentException() throws Exception { + public void schedulerDeliveryHandlesIllegalArgumentException() throws Exception { Mockito.doThrow(new RuntimeException(new IllegalArgumentException())) .when(ingestionimplMock) @@ -226,7 +223,7 @@ public void IngestionControllerHandlesIllegalArgumentException() throws Exceptio } @Test - public void IngestionControllerHandlesIOException() throws Exception { + public void scheduleDeliveryHandlesIOException() throws Exception { Mockito.doThrow(new RuntimeException(new IOException())) .when(ingestionimplMock) @@ -246,7 +243,7 @@ public void IngestionControllerHandlesIOException() throws Exception { } @Test - public void IngestionControllerHandlesJsonProcessingException() throws Exception { + public void scheduleDeliveryHandlesJsonProcessingException() throws Exception { Mockito.doThrow(new RuntimeException(new JsonParseException(null, "error"))) .when(ingestionimplMock) @@ -267,7 +264,7 @@ public void IngestionControllerHandlesJsonProcessingException() throws Exception @Test - public void IngestionControllerHandlesMethodArgumentTypeMismatchException() throws Exception { + public void scheduleDeliveryHandlesMethodArgumentTypeMismatchException() throws Exception { Mockito.doThrow(new RuntimeException(new MethodArgumentTypeMismatchException(appPropsMock, null, null, null, null))) .when(ingestionimplMock) @@ -287,6 +284,242 @@ public void IngestionControllerHandlesMethodArgumentTypeMismatchException() thro } + //------------------ + + @Test + public void rescheduleDeliveryHandlesServiceBusException() throws Exception { + + Mockito.doThrow(new RuntimeException(new ServiceBusException(true,"message"))) + .when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + patch("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void reschedulerDeliveryHandlesIllegalArgumentException() throws Exception { + + Mockito.doThrow(new RuntimeException(new IllegalArgumentException())) + .when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + + mockMvc.perform( + patch("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void rescheduleDeliveryHandlesIOException() throws Exception { + + Mockito.doThrow(new RuntimeException(new IOException())) + .when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + patch("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void rescheduleDeliveryHandlesJsonProcessingException() throws Exception { + + Mockito.doThrow(new RuntimeException(new JsonParseException(null, "error"))) + .when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + mockMvc.perform( + patch("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + @Test + public void rescheduleDeliveryHandlesMethodArgumentTypeMismatchException() throws Exception { + + Mockito.doThrow(new RuntimeException(new MethodArgumentTypeMismatchException(appPropsMock, null, null, null, null))) + .when(ingestionimplMock) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + patch("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + //--------------------- + + @Test + public void cancelscheduleDeliveryHandlesServiceBusException() throws Exception { + + Mockito.doThrow(new RuntimeException(new ServiceBusException(true,"message"))) + .when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + delete("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void cancelschedulerDeliveryHandlesIllegalArgumentException() throws Exception { + + Mockito.doThrow(new RuntimeException(new IllegalArgumentException())) + .when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + + mockMvc.perform( + delete("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void cancelscheduleDeliveryHandlesIOException() throws Exception { + + + Mockito.doThrow(new RuntimeException(new IOException())) + .when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + delete("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + @Test + public void cancelscheduleDeliveryHandlesJsonProcessingException() throws Exception { + + Mockito.doThrow(new RuntimeException(new JsonParseException(null, "error"))) + .when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + mockMvc.perform( + delete("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().is5xxServerError()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + @Test + public void cancelscheduleDeliveryHandlesMethodArgumentTypeMismatchException() throws Exception { + + Mockito.doThrow(new RuntimeException(new MethodArgumentTypeMismatchException(appPropsMock, null, null, null, null))) + .when(ingestionimplMock) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + mockMvc.perform( + delete("/api/deliveryrequests/" + deliveryId) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(externalDelivery))) + .andExpect(status().isBadRequest()); + + Mockito.verify(ingestionimplMock, times(1)) + .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); + + Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); + } + + + + + + + From 0581b354eae5dfc5d9e098fc4ba4d360a0b2660e Mon Sep 17 00:00:00 2001 From: cfarre Date: Wed, 24 Jan 2018 14:07:49 -0800 Subject: [PATCH 019/246] delete performancetests --- .../ingestion/IngestionPerformanceTest.java | 221 ------------------ 1 file changed, 221 deletions(-) delete mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java deleted file mode 100644 index 63b8fac0..00000000 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionPerformanceTest.java +++ /dev/null @@ -1,221 +0,0 @@ -package com.fabrikam.dronedelivery.ingestion; - -import static org.junit.Assert.*; -import com.google.gson.*; - -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -//import org.junit.runner.RunWith; -import org.springframework.web.client.*; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; - -import org.springframework.util.concurrent.ListenableFuture; -//import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.HttpEntity; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.converter.StringHttpMessageConverter; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; - -import com.fabrikam.dronedelivery.ingestion.models.*; -import com.fasterxml.jackson.databind.ObjectMapper; - -public class IngestionPerformanceTest { - - private AsyncRestTemplate asyncRest; - private HttpHeaders requestHeaders; - private String uri; - private String IpAddress; - private String Port; - - @Before - public void setUp() throws Exception { - - asyncRest = new AsyncRestTemplate(); - asyncRest.getMessageConverters().add(new MappingJackson2HttpMessageConverter()); - asyncRest.getMessageConverters().add(new StringHttpMessageConverter()); - requestHeaders = new HttpHeaders(); - requestHeaders.setAccept(Arrays.asList(MediaType.APPLICATION_JSON)); - // 157.55.175.96 - // 23.96.201.211 - // 52.179.159.74 - // 52.179.158.181 - IpAddress = "localhost"; - Port = "8080"; - - } - - @Test - @Ignore - public void PostDeliveryAsync() throws InterruptedException, ExecutionException { - - uri = new StringBuffer().append("https://").append(IpAddress).append(":").append(Port) - .append("/api/deliveryrequests").toString(); - PackageInfo pack = new PackageInfo(); - - pack.setPackageId(UUID.randomUUID().toString()); - pack.setSize(ContainerSize.Medium); - - List Packages = new ArrayList(); - Packages.add(pack); - - ExternalDelivery delivery = new ExternalDelivery(); - Date date = new Date(); - - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(date); - delivery.setDropOffLocation("Austin"); - delivery.setPickupLocation("Austin"); - delivery.setDeadline("deadline"); - delivery.setConfirmationRequired(ConfirmationRequired.Picture); - - delivery.setPackageInfo(pack); - - HttpEntity entity = new HttpEntity(delivery, requestHeaders); - - ListenableFuture> response = asyncRest.postForEntity(uri, entity, - ExternalDelivery.class, delivery); - - System.out.println(response.get().getBody().getPickupTime()); - - assertEquals(HttpStatus.ACCEPTED, response.get().getStatusCode()); - - } - - @Test - @Ignore - public void DeleteCancelDeliveryAsync() throws InterruptedException, ExecutionException { - - UUID deliveryId = UUID.randomUUID(); - uri = new StringBuffer().append("https://").append(IpAddress).append(":").append(Port) - .append("/api/canceldelivery/").append(deliveryId).toString(); - - @SuppressWarnings("unchecked") - ListenableFuture> response = (ListenableFuture>) asyncRest.delete(uri, - deliveryId); - - response.get(); - // response.get().getBody(); - - // assertEquals(HttpStatus.OK,response.get().getStatusCode()); - - } - - @Test - @Ignore - public void GetProbeAsync() throws InterruptedException, ExecutionException { - - uri = new StringBuffer().append("https://").append(IpAddress).append(":").append(Port).append("/api/probe") - .toString(); - - ListenableFuture> response = asyncRest.getForEntity(uri, String.class); - assertEquals(HttpStatus.OK, response.get().getStatusCode()); - - } - - @Test - @Ignore - public void CanConvertToJson() throws InterruptedException { - - PackageInfo pack = new PackageInfo(); - - pack.setPackageId(UUID.randomUUID().toString()); - pack.setSize(ContainerSize.Medium); - - List Packages = new ArrayList(); - Packages.add(pack); - - ExternalDelivery delivery = new ExternalDelivery(); - Date date = new Date(); - - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(date); - delivery.setDropOffLocation("Austin"); - delivery.setPickupLocation("Austin"); - delivery.setDeadline("deadline"); - delivery.setConfirmationRequired(ConfirmationRequired.Picture); - - delivery.setPackageInfo(pack); - - Gson gson = new GsonBuilder().create(); - System.out.println(gson.toJson(delivery)); - System.out.println(asJsonString(delivery)); - - } - - @Test - @Ignore - public void CanDeserializeDelivery() throws InterruptedException { - - PackageInfo pacKage = new PackageInfo(); - pacKage.setSize(ContainerSize.Small); - pacKage.setPackageId(UUID.randomUUID().toString()); - - List Packages = new ArrayList(); - Packages.add(pacKage); - - ExternalDelivery delivery = new ExternalDelivery(); - Date date = new Date(); - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(date); - delivery.setDropOffLocation("Austin"); - delivery.setPickupLocation("Austin"); - delivery.setConfirmationRequired(ConfirmationRequired.Picture); - delivery.setDeadline("dealine"); - delivery.setExpedited(true); - delivery.setPackageInfo(pacKage); - - Gson gson = new GsonBuilder().create(); - - DeliveryBase intdelivery = new Delivery(UUID.randomUUID().toString(), delivery.getOwnerId().toString(), delivery.getPickupLocation(), - delivery.getDropOffLocation(), delivery.getPickupTime(), delivery.isExpedited(), - delivery.getConfirmationRequired(), delivery.getPackageInfo(), delivery.getDeadline()); - - String jsondelivery = gson.toJson(intdelivery); - System.out.println(gson.toJson(jsondelivery)); - - } - - @Test - @Ignore - public void CanConvertToJson2() throws InterruptedException { - - // ExternalRescheduledDelivery delivery = - // new ExternalRescheduledDelivery(); - // delivery.setDeadline("deadline"); - // delivery.setDelivery(UUID.randomUUID()); - // delivery.setDropOffLocation("location"); - // delivery.setPickupLocation("pickuplocation"); - // - // Gson gson = new GsonBuilder().create(); - // System.out.println(gson.toJson(delivery)); - // System.out.println(asJsonString(delivery)); - // - - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - } - - private static String asJsonString(ExternalDelivery obj) { - try { - ObjectMapper mapper = new ObjectMapper(); - String jsonContent = mapper.writeValueAsString(obj); - return jsonContent; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - -} From 79f134f319d5c3b642cfee4fca4f89a449564bd4 Mon Sep 17 00:00:00 2001 From: cfarre Date: Wed, 24 Jan 2018 14:33:26 -0800 Subject: [PATCH 020/246] removed code comment in test --- .../dronedelivery/ingestion/IngestionControllerTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index 598a4bc4..50e755f9 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -48,8 +48,7 @@ import java.io.IOException; -//@RunWith(SpringJUnit4ClassRunner.class) -//@SpringApplicationConfiguration(classes = ApplicationProperties.class) + public class IngestionControllerTest { From cb73e31b98d247dd1f48309bb38f6d588712d560 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 14:21:18 -0300 Subject: [PATCH 021/246] reduce ARM params, flesh output out and use them from deployment steps: 1. expose ARM template outputs for auto-generated resource names during creation time 2. az CLI to query secrets --- README.md | 2 +- azuredeploy.json | 168 +++++++--------- deployment.md | 2 +- deploymentARM.md | 391 ++++++++++++++++++++++++++++++++++++ deploymentF5.md | 349 -------------------------------- service-principal-creds.png | Bin 14259 -> 0 bytes 6 files changed, 471 insertions(+), 441 deletions(-) create mode 100644 deploymentARM.md delete mode 100644 deploymentF5.md delete mode 100644 service-principal-creds.png diff --git a/README.md b/README.md index a6f86af7..c7a71354 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The Drone Delivery application is a sample application that consists of several ## Deployment -To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure you are going to create or just go for a [quick start using the ARM templates](./deploymentF5.md). +To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure or [just use ARM templates](./deploymentARM.md). diff --git a/azuredeploy.json b/azuredeploy.json index 5f8ad605..d0fcafde 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -2,12 +2,24 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { - "dnsNamePrefix": { + "sshRSAPublicKey": { "type": "string", "metadata": { - "description": "Sets the Domain name prefix for the cluster. The concatenation of the domain name and the regionalized DNS zone make up the fully qualified domain name associated with the public IP address." + "description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example 'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm'" } }, + "servicePrincipalClientId": { + "metadata": { + "description": "Client ID (used by cloudprovider)" + }, + "type": "string" + }, + "servicePrincipalClientSecret": { + "metadata": { + "description": "The Service Principal Client Secret." + }, + "type": "securestring" + }, "agentCount": { "type": "int", "defaultValue": 3, @@ -19,7 +31,11 @@ }, "agentVMSize": { "type": "string", - "defaultValue": "Standard_D2_v2", + "defaultValue": "Standard_D2_v2" , + "allowedValues": [ + "Standard_D2_v2", + "Standard_F8s_v2" + ], "metadata": { "description": "The size of the Virtual Machine." } @@ -31,18 +47,6 @@ }, "defaultValue" : "azureuser" }, - "orchestratorType": { - "type": "string", - "defaultValue": "Kubernetes", - "allowedValues": [ - "Kubernetes", - "DCOS", - "Swarm" - ], - "metadata": { - "description": "The type of orchestrator used to manage the applications on the cluster." - } - }, "masterCount": { "type": "int", "defaultValue": 1, @@ -53,39 +57,6 @@ "description": "The number of Kubernetes masters for the cluster." } }, - "sshRSAPublicKey": { - "type": "string", - "metadata": { - "description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example 'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm'" - } - }, - "servicePrincipalClientId": { - "metadata": { - "description": "Client ID (used by cloudprovider)" - }, - "type": "securestring" - }, - "servicePrincipalClientSecret": { - "metadata": { - "description": "The Service Principal Client Secret." - }, - "type": "securestring" - }, - "acrName": { - "type": "string", - "minLength": 5, - "maxLength": 50, - "metadata": { - "description": "Name of your Azure Container Registry" - } - }, - "acrAdminUserEnabled": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Enable admin user that have push / pull permission to the registry." - } - }, "acrStorageType": { "type": "string", "defaultValue": "Standard_LRS", @@ -136,28 +107,22 @@ "delivery_redisCacheCapacity": { "type": "int", "allowedValues": [ - 0, 1, 2, 3, - 4, - 5, - 6 + 4 ], - "defaultValue": 1, + "defaultValue": 4, "metadata": { "description": "The size of the new Azure Redis Cache instance. " } }, - "delivery_redisEnableNonSslPort": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "A boolean value that indicates whether to allow access via non-SSL ports." - } - }, "delivery_redisDiagnosticsEnabled": { "type": "bool", + "allowedValues": [ + false, + true + ], "defaultValue": false, "metadata": { "description": "A value that indicates whether diagnostics should be saved to the specified storage account." @@ -177,14 +142,14 @@ } }, "variables": { - "agentsEndpointDNSNamePrefix":"[concat(parameters('dnsNamePrefix'),'agents')]", - "mastersEndpointDNSNamePrefix":"[concat(parameters('dnsNamePrefix'),'mgmt')]", + "agentsEndpointDNSNamePrefix":"[concat('agents',uniqueString(resourceGroup().id))]", + "mastersEndpointDNSNamePrefix":"[concat('mgmt',uniqueString(resourceGroup().id))]", "useServicePrincipalDictionary": { "DCOS": 0, "Swarm": 0, "Kubernetes": 1 }, - "useServicePrincipal": "[variables('useServicePrincipalDictionary')[parameters('orchestratorType')]]", + "useServicePrincipal": "[variables('useServicePrincipalDictionary')['Kubernetes']]", "servicePrincipalFields": [ null, { @@ -192,27 +157,30 @@ "Secret": "[parameters('servicePrincipalClientSecret')]" } ], - "acrStorageName": "[concat('acr', uniqueString(resourceGroup().id))]", + "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", + "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", - "deliveryRedisStorageName": "[concat('red', uniqueString(resourceGroup().id))]", + "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", + "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", - "deliveryCosmosDbName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-cosmosdb')]", - "deliveryRedisName": "[concat(parameters('dnsNamePrefix'),'-delivery-service-redis')]", - "packageMongoDbName": "[concat(parameters('dnsNamePrefix'),'-package-service-cosmosdb')]", - "ingestionEHNamespace": "[concat(parameters('dnsNamePrefix'),'ingehns')]", - "ingestionEHName": "[concat(parameters('dnsNamePrefix'),'ingeh')]", - "ingestionEHConsumerGroupName": "[concat(parameters('dnsNamePrefix'),'ingehcg')]", - "schedulerStorageName": "[concat('sch', uniqueString(resourceGroup().id))]" + "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-cosmosdb')]", + "deliveryRedisName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-redis')]", + "packageMongoDbName": "[concat(uniqueString(resourceGroup().id),'-package-service-cosmosdb')]", + "ingestionEHNamespace": "[concat(uniqueString(resourceGroup().id),'ingehns')]", + "ingestionEHName": "[concat(uniqueString(resourceGroup().id),'ingeh')]", + "ingestionEHConsumerGroupName": "[concat(uniqueString(resourceGroup().id),'ingehcg')]", + "schedulerStorageName": "[concat('schsto', uniqueString(resourceGroup().id))]", + "schedulerStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('schedulerStorageName'))]" }, "resources": [ { "apiVersion": "2016-09-30", "type": "Microsoft.ContainerService/containerServices", "location": "[resourceGroup().location]", - "name":"[concat('containerservice-',resourceGroup().name)]", + "name":"[variables('acsK8sClusterName')]", "properties": { "orchestratorProfile": { - "orchestratorType": "[parameters('orchestratorType')]" + "orchestratorType": "Kubernetes" }, "masterProfile": { "count": "[parameters('masterCount')]", @@ -248,14 +216,14 @@ "dependsOn": [], "tags": { "displayName": "ACR Image's storage", - "container.registry": "[parameters('acrName')]" + "container.registry": "[variables('acrName')]" }, "properties": { "accountType": "[parameters('acrStorageType')]" } }, { - "name": "[parameters('acrName')]", + "name": "[variables('acrName')]", "type": "Microsoft.ContainerRegistry/registries", "apiVersion": "2016-06-27-preview", "location": "[resourceGroup().location]", @@ -265,10 +233,10 @@ ], "tags": { "displayName": "Container Registry", - "container.registry": "[parameters('acrName')]" + "container.registry": "[variables('acrName')]" }, "properties": { - "adminUserEnabled": "[parameters('acrAdminUserEnabled')]", + "adminUserEnabled": "true", "storageAccount": { "accessKey": "[listKeys(variables('acrStorageId'),'2015-06-15').key1]", "name": "[variables('acrStorageName')]" @@ -298,7 +266,7 @@ "[variables('deliveryRedisStorageId')]" ], "properties": { - "redisEnableNonSslPort": "[parameters('delivery_redisEnableNonSslPort')]", + "redisEnableNonSslPort": "false", "sku": { "capacity": "[parameters('delivery_redisCacheCapacity')]", "family": "[parameters('delivery_redisCacheFamily')]", @@ -415,21 +383,41 @@ } ], "outputs": { - "masterFQDN": { - "type": "string", - "value": "[reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).masterProfile.fqdn]" + "acrName": { + "value": "[variables('acrName')]", + "type": "string" }, - "sshMaster0": { - "type": "string", - "value": "[concat('ssh ', parameters('adminUsername'), '@', reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).masterProfile.fqdn, ' -A -p 22')]" + "acrLoginServer": { + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('acrName')),'2016-06-27-preview').loginServer]", + "type": "string" }, - "agentFQDN": { - "type": "string", - "value": "[reference(concat('Microsoft.ContainerService/containerServices/', 'containerservice-', resourceGroup().name)).agentPoolProfiles[0].fqdn]" + "acsK8sClusterName": { + "value": "[variables('acsK8sClusterName')]", + "type": "string" }, - "acrLoginServer": { - "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',parameters('acrName')),'2016-06-27-preview').loginServer]", + "deliveryRedisName": { + "value": "[variables('deliveryRedisName')]", "type": "string" + }, + "deliveryCosmosDbName": { + "value": "[variables('deliveryCosmosDbName')]", + "type": "string" + }, + "packageMongoDbName": { + "value": "[variables('packageMongoDbName')]", + "type": "string" + }, + "ingestionEHNamespace": { + "value": "[variables('ingestionEHNamespace')]", + "type": "string" + }, + "ingestionEHName": { + "value": "[variables('ingestionEHName')]", + "type": "string" + }, + "schedulerStorageAccountName": { + "value": "[listKeys(variables('schedulerStorageName'),'2016-12-01')]", + "type": "object" } } } diff --git a/deployment.md b/deployment.md index 13328bb6..e073317e 100644 --- a/deployment.md +++ b/deployment.md @@ -48,7 +48,7 @@ sudo az acs kubernetes install-cli # Get the Kubernetes cluster credentials az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -# Create the Shipping BC namespace +# Create the BC namespaces kubectl create namespace shipping && \ kubectl create namespace accounts && \ kubectl create namespace dronemgmt && \ diff --git a/deploymentARM.md b/deploymentARM.md new file mode 100644 index 00000000..5a5e9080 --- /dev/null +++ b/deploymentARM.md @@ -0,0 +1,391 @@ +# Deploying the Reference Implementation + + + +## Prerequisites + +- Azure suscription +- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Docker](https://docs.docker.com/) +- [Docker Compose](https://docs.docker.com/compose/install/) + +Clone or download this repo locally. + +```bash +git clone https://github.com/mspnp/microservices-reference-implementation.git +``` + +> The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. + +## Generate a SSH rsa public/private key pair + +the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). +> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. + +## Azure Resources Provisioning + +Set environment variables. + +```bash +export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] +export SSH_PRIVATE_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PRIVAYE_KEY_FILE_HERE] + +export LOCATION=[YOUR_LOCATION_HERE] + +export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] + +export SP_CLIENT_SECRET=$(uuidgen) +``` + +Infrastructure Prerequisites + +```bash +# Log in to Azure +az login + +# Create a resource group and service principal for ACS +az group create --name $RESOURCE_GROUP --location $LOCATION && \ +export SP_APP_ID=$(az ad sp create-for-rbac --role="Contributor" -p $SP_CLIENT_SECRET | grep -oP '(?<="appId": ")[^"]*') && \ +export BEARER_TOKEN=$(az account get-access-token --query accessToken | sed -e 's/^"//' -e 's/"$//') && \ +export SUBS_ID=$(az account show --query id | sed -e 's/^"//' -e 's/"$//') +``` + +Deployment + +> Note: this deployment might take up to 20 minutes + +* using Azure CLI 2.0 + + ```bash + az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json --parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" + ``` + +* from Azure Portal + + [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Ffeature%2F48_AddARMTemplates%2Fazuredeploy.json) + > Note: + > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group + > 2. paste the content of your ssh-rsa public key file in the Ssh RSA Plublic Key field. + > 3. paste the $SP_APP_ID and $SP_CLIENT_SECRET in the Client Id and Secret fields. + +Get outputs from Azure Deploy +```bash +# Shared +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value | sed -e 's/^"//' -e 's/"$//') && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value | sed -e 's/^"//' -e 's/"$//') && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acsK8sClusterName.value | sed -e 's/^"//' -e 's/"$//') +``` + +Download kubectl and create a k8s namespace +```bash +# Install kubectl +sudo az acs kubernetes install-cli + +# Get the Kubernetes cluster credentials +az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME \ + --ssh-key-file $SSH_PRIVATE_KEY_FILE + +# Create the BC namespaces +kubectl create namespace shipping && \ +kubectl create namespace accounts && \ +kubectl create namespace dronemgmt && \ +kubectl create namespace 3rdparty +``` + +## Deploy the Delivery service + +Provision Azure resources + +```bash +export REDIS_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryRedisName.value | sed -e 's/^"//' -e 's/"$//') && \ +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value | sed -e 's/^"//' -e 's/"$//') && \ +export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ +export COLLECTION_NAME="${DATABASE_NAME}-col" + +# Create a Cosmos DB database +az cosmosdb database create \ + --name $COSMOSDB_NAME \ + --db-name=$DATABASE_NAME \ + --resource-group $RESOURCE_GROUP + +# Create a Cosmos DB collection +az cosmosdb collection create \ + --collection-name $COLLECTION_NAME \ + --name $COSMOSDB_NAME \ + --db-name $DATABASE_NAME \ + --resource-group $RESOURCE_GROUP +``` + +Build the Delivery service + +```bash +export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery +docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/Fabrikam.DroneDelivery.DeliveryService/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 + +``` + +Create Kubernetes secrets + +```bash +export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] + +export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ +export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) + +kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ + --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ + --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ + --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ + --from-literal=EH_ConnectionString= +``` + +Deploy the Delivery service: + +```bash +# Update the image tag in the deployment YAML +sed -i "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" ./microservices-reference-implementation/k8s/delivery.yaml + +## Update config values in the deployment YAML +sed -i "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ +sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ +sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" + +# Deploy the service +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml +``` + +## Deploy the Package service + +Build the Package service + +```bash +export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package + +# Build the app +docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up + +# Build the docker image +sudo docker build -f $PACKAGE_PATH/build/prod.dockerfile -t $ACR_SERVER/package-service:0.1.0 $PACKAGE_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/package-service:0.1.0 +``` + +Deploy the Package service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml + +# Create secret +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value | sed -e 's/^"//' -e 's/"$//') && \ +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") +kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} + +# Deploy service +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/package.yml +``` + +## Deploy the Ingestion service + +Build the Ingestion service + +```bash +export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion + +# Build the app +docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ +docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk + +# Build the docker image +docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/ingestion:0.1.0 +``` + +Deploy the Ingestion service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" ./microservices-reference-implementation/k8s/ingestion.yaml + +# Get the EventHub shared access policy name and key from the Azure Portal +export INGESTION_EH_NS=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionEHNamespace.value | sed -e 's/^"//' -e 's/"$//') && \ +export INGESTION_EH_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionEHName.value | sed -e 's/^"//' -e 's/"$//') && \ +export EH_KEYS=$(curl -X POST "https://management.azure.com/subscriptions/${SUBS_ID}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.EventHub/namespaces/${INGESTION_EH_NS}/AuthorizationRules/RootManageSharedAccessKey/ListKeys?api-version=2017-04-01" -H "Content-Type: application/json" -H "Authorization: Bearer ${BEARER_TOKEN}" -H "Content-Length: 0" --stderr - --silent) && \ +export EH_ACCESS_KEY_NAME=$(echo $EH_KEYS | grep -oP '(?<="keyName":")[^"]*') && \ +export EH_ACCESS_KEY_VALUE=$(echo $EH_KEYS | grep -oP '(?<="primaryKey":")[^"]*') && \ +export EH_CONNECTION_STRING=$(echo $EH_KEYS | grep -oP '(?<="primaryConnectionString":")[^"]*') + +# Create secret +kubectl -n shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ +--from-literal=eventhub_name=${INGESTION_EH_NAME} \ +--from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ +--from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} + +# Deploy service +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml +``` + +## Deploy the Scheduler service + +Build the Scheduler service + +```bash +export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler + +# Build the app +docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ +docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk + +# Build the docker image +docker build -f $SCHEDULER_PATH/Dockerfile -t $ACR_SERVER/scheduler:0.1.0 $SCHEDULER_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/scheduler:0.1.0 +``` + +Deploy the Scheduler service + +```bash +# Update deployment YAML with image tage +sed -i "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" ./microservices-reference-implementation/k8s/scheduler.yaml + +# Get the following values from the Azure Portal +export SCHEDULER_STORAGE_ACCOUNT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.schedulerStorageAccountName.value | sed -e 's/^"//' -e 's/"$//') && \ +export EH_KEYS=$(curl -X POST "https://management.azure.com/subscriptions/${SUBS_ID}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.EventHub/namespaces/${INGESTION_EH_NS}/AuthorizationRules/RootManageSharedAccessKey/ListKeys?api-version=2017-04-01" -H "Content-Type: application/json" -H "Authorization: Bearer ${BEARER_TOKEN}" -H "Content-Length: 0" --stderr - --silent) && \ +export EH_CONNECTION_STRING=$(echo $EH_KEYS | grep -oP '(?<="primaryConnectionString":")[^"]*') + +export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] +export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" + +# Create secrets +kubectl -n shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ +--from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ +--from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ +--from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ +--from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} + +# Deploy service +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml +``` + +## Deploy mock services + +Build the mock services + +```bash +export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery +docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ +docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ +docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/account:0.1.0 && \ +docker push $ACR_SERVER/dronescheduler:0.1.0 && \ +docker push $ACR_SERVER/thirdparty:0.1.0 +``` + +Deploy the mock services: + +```bash +# Update the image tag in the deployment YAML +sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ +sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ +sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml + +# Deploy the service +kubectl --namespace accounts apply -f ./microservices-reference-implementation/k8s/account.yaml && \ +kubectl --namespace dronemgmt apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ +kubectl --namespace 3rdparty apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml +``` + +## Verify all services are running: + +```bash +kubectl get all --all-namespaces -l co=fabrikam +``` + +Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch + +```bash +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml +``` + +Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd + +```bash +# The example elasticsearch yaml files deploy a service named "elasticsearch" +wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml + +# Commenting out X-Pack credentials for demo purposes. +# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads +sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ +kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml +``` + +#### Deploy linkerd + +For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) + +> Note: +> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. +> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified. This will consist of a small change in the dtab rules. + +Deploy linkerd defaulting the namespace to shipping instead: + +```bash +wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ +sed -i "s#/default#/shipping#g" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ +kubectl apply -f servicemesh.yml +``` + +Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy + +It is recommended to put an API Gateway in front of all APIs you want exposed to the public, +however for convenience, we exposed the Ingestion service with a public IP address. + +You can send delivery requests to the ingestion service using the swagger ui. + +```bash +export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST +``` + diff --git a/deploymentF5.md b/deploymentF5.md deleted file mode 100644 index f7cc4299..00000000 --- a/deploymentF5.md +++ /dev/null @@ -1,349 +0,0 @@ -# Deploying the Reference Implementation - - - -## Prerequisites - -- Azure suscription -- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) -- [Docker](https://docs.docker.com/) -- [Docker Compose](https://docs.docker.com/compose/install/) - -Clone or download this repo locally. - -```bash -git clone https://github.com/mspnp/microservices-reference-implementation.git -``` - -## Create the Kubernetes cluster - -Set environment variables. - -```bash -export LOCATION=[YOUR_LOCATION_HERE] - -export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] -export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] - -export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ -export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ -export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ -export MONGODB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" && \ -export INGESTION_EH_NS="${UNIQUE_APP_NAME_PREFIX}ingehns" && \ -export INGESTION_EH_NAME="${UNIQUE_APP_NAME_PREFIX}ingeh" && \ -export INGESTION_EH_CONSUMERGROUP_NAME="${UNIQUE_APP_NAME_PREFIX}ingehcg" -``` - -Infrastructure Prerequisites - -```bash -# Log in to Azure -az login - -# Create a resource group for ACS -az group create --name $RESOURCE_GROUP --location $LOCATION - -# Create an Azure Service Principal for RBAC -az ad sp create-for-rbac --role="Contributor" -``` - -Obtain a SSH rsa public key - -the SSH rsa public key is typically generated by ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could providate the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). - -Create the ACS cluster -* using Azure CLI 2.0 - - ```bash - az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ - '{ \ - "acrName": {"value": "'${ACR_NAME}'"}, \ - "dnsNamePrefix": {"value": "'${UNIQUE_APP_NAME_PREFIX}'"} - }' - - ``` - > Note: - > 1. when sshRSAPublicKey parameter prompts you for the string, do not enclose within quotes, or they will be treated as part of the public key. - > 2. Highlighted are the servicePrincipalClientId (appId) and the servicePrincipalClientSecret (password) that you use as service principal parameters for cluster deployment. Please note that these params are secure strings, so when you enter or paste the value it is not going to be displayed in your screen. - - ![](./service-principal-creds.png) - -* from Azure Portal - - [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Ffeature%2F48_AddARMTemplates%2Fazuredeploy.json) - > Note: - > 1. when deployed using the Preview Portal, you should paste in the contents of your ssh-rsa public key file as a string. - > 2. paste the $RESOURCE_GROUP value and choose use existing resource group - > 3. paste the $ACR_NAME value in the acraname parameter - -Download kubectl and create a k8s namespace -```bash -# Install kubectl -sudo az acs kubernetes install-cli - -# Get the Kubernetes cluster credentials -az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME - -# Create the Shipping BC namespace -kubectl create namespace bc-shipping -``` - -Obtain your Azure Container Registry instance details. - -```bash - -# Log in to ACR -az acr login --name $ACR_NAME - -# Get the ACR login server name -export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer") - -# Strip quotes -export ACR_SERVER=("${ACR_SERVER[@]//\"/}") -``` - -## Deploy the Delivery service - -Provision Azure resources - -```bash -export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ -export COLLECTION_NAME="${DATABASE_NAME}-col" - -# Create a Cosmos DB database -az cosmosdb database create \ - --name $COSMOSDB_NAME \ - --db-name=$DATABASE_NAME \ - --resource-group $RESOURCE_GROUP - -# Create a Cosmos DB collection -az cosmosdb collection create \ - --collection-name $COLLECTION_NAME \ - --name $COSMOSDB_NAME \ - --db-name $DATABASE_NAME \ - --resource-group $RESOURCE_GROUP -``` - -Build the Delivery service - -```bash -export DELIVERY_PATH=./microservices-reference-implementation/src/bc-shipping/delivery -docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up -``` - -Build and publish the container image - -```bash -# Build the Docker image -docker build -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/Fabrikam.DroneDelivery.DeliveryService/. - -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 - -``` - -Create Kubernetes secrets - -```bash -export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] - -export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ -export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) - -kubectl --namespace bc-shipping create --save-config=true secret generic delivery-storageconf \ - --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ - --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ - --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ - --from-literal=EH_ConnectionString= -``` - -Deploy the Delivery service: - -```bash -# Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" ./microservices-reference-implementation/k8s/delivery.yaml - -## Update config values in the deployment YAML -sed -i "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" - -# Deploy the service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml -``` - -## Deploy the Package service - -Build the Package service - -```bash -export PACKAGE_PATH=microservices-reference-implementation/src/bc-shipping/package - -# Build the app -docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up - -# Build the docker image -sudo docker build -f $PACKAGE_PATH/build/prod.dockerfile -t $ACR_SERVER/package-service:0.1.0 $PACKAGE_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/package-service:0.1.0 -``` - -Deploy the Package service - -```bash -# Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml - -# Create secret -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $MONGODB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") -kubectl -n bc-shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} - -# Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/package.yml -``` - -## Deploy the Ingestion service - -Build the Ingestion service - -```bash -export INGESTION_PATH=./microservices-reference-implementation/src/bc-shipping/ingestion - -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ -docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk - -# Build the docker image -docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/ingestion:0.1.0 -``` - -Deploy the Ingestion service - -```bash -# Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" ./microservices-reference-implementation/k8s/ingestion.yaml - -# Get the EventHub shared access policy name and key from the Azure Portal -export EH_ACCESS_KEY_NAME=[YOUR_SHARED_ACCESS_POLICY_NAME_HERE] -export EH_ACCESS_KEY_VALUE=[YOUR_SHARED_ACCESS_POLICY_VALUE_HERE] - -# Create secret -kubectl -n bc-shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ ---from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ ---from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} - -# Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml -``` - -## Deploy the Scheduler service - -Build the Scheduler service - -```bash -export SCHEDULER_PATH=./microservices-reference-implementation/src/bc-shipping/scheduler - -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ -docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk - -# Build the docker image -docker build -f $SCHEDULER_PATH/Dockerfile -t $ACR_SERVER/scheduler:0.1.0 $SCHEDULER_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/scheduler:0.1.0 -``` - -Deploy the Scheduler service - -```bash -# Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" ./microservices-reference-implementation/k8s/scheduler.yaml - -# Get the following values from the Azure Portal -export EH_CONNECTION_STRING="[YOUR_EVENT_HUB_CONNECTION_STRING_HERE]" -export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] -export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" - -# Create secrets -kubectl -n bc-shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ ---from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ ---from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ ---from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} - -# Deploy service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml -``` - -## Deploy mock services - -Build the mock services - -```bash -export MOCKS_PATH=microservices-reference-implementation/src/bc-shipping/delivery -docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up -``` - -Build and publish the container image - -```bash -# Build the Docker image -docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ -docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ -docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. - -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/account:0.1.0 && \ -docker push $ACR_SERVER/dronescheduler:0.1.0 && \ -docker push $ACR_SERVER/thirdparty:0.1.0 -``` - -Deploy the mock services: - -```bash -# Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml - -# Deploy the service -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/account.yaml && \ -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -kubectl --namespace bc-shipping apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml -``` - -## Verify all services are running: - -```bash -kubectl get all -n bc-shipping -``` - -Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch - -Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd - -Deploy linkerd. For more information, see https://linkerd.io/getting-started/k8s/ - -Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy - -It is recommended to put an API Gateway in front of all APIs you want exposed to the public, -however for convenience, we exposed the Ingestion service with a public IP address. - -You can send delivery requests to the ingestion service using the swagger ui. - -```bash -export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace bc-shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") -curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` diff --git a/service-principal-creds.png b/service-principal-creds.png deleted file mode 100644 index e3161993c2257694c9c2be9d44b69621d6742cdc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14259 zcmaibbyOV9w=F?J1`qCT!QCaeyE}usy9N(VaCdii3GPmCcX!tZ`M%%1>)pH7dw+CK z&zVy))m_!K&)z3kR$2rW3KI$h1O!%0R8SrSx@N?`H;}u-*Fyz(!Qf9s~rg z=i~SkM+t}ZZiI9YlM;g50HB}}QJr<_|6>wTb>MS!FtN4+DJmZkefNTKcsB_e+Uwhy z+Ble6TY(?}2#DV8P#<;yYa3TPQ)3eckW%;&j(02EU#p!V@WU0~!PL?ag!v90@cs+n z;}->MBL`=oogs*p&D)1N^xw)&?e#5y@2++r4Zmw6-_2h>%ql=TD^n|Dkl$FlLGL!m zf66=B=^KLFy!jpd$4$W6($dh%0pxZhzT@5Y`NK`o)WO0Kr0{f#3;4wcd3DJf9iyL$s3DN1o=E*R>Ab=Y18$~zml{70kPhB5O7d$=ldbWE% z9aVI^Li&8iNBE-r`|WY%E5a9W(9R6MLPc;;pYN3nf_a4R7T-7m3X~70QW&|ScLyI` z!=K_p?`6O#1r#aX9r0B$#!D25c_*hCVY=6;PTlj!0q<_>ZI>MxSNb|kBlU`8NL;%b zdv)2wANAc0)woeeFNinDp`wI*G;Tz=`)`ZsT1D61*E&SFAJyK2sW!|1I;Mcr$mlql ztH8kf=tQf2d~zYYF%0OD=c%avOcc5$SJBF_2&w82FTxo1`9qc<)i z9WOU?I&W7kL0IFKZnPf4l2K@U~ac;pL3nBiQzLNQ3?}j`556JJb$E zAFZD!Hn1oWxcxh963P@n27!P|+OZ^H6rjVZ#|sYcqUNG8>5YW}NO}rXQVGofqGNO= zrdVMb+xGCA`>bPxeH((+PgL>#iuIJ+5idOF^^_sjYUNr_k)DV2igUkd+{029<9)mu zwal4+>7HGSeF-Nrh@UaX18U=K92IWZbjnTkCGFY=O{2!&F43J0ziEDsh};TGIdiEs zD~f0L`DtI$>6)0}3{&~X6NmA)G}^k=&B|XkCuBcAr+U7i z*u^Yw-AnPpbea=9FY-1N`2Qf6V#250mp(v+_gM2=95lTJ%^VF=ib7(1EKTXZ)51Bc^9Q5}^ZQA6J)55OY=KqsZSdV$bGM^vm4r(#n zu-(Nz#Eo|`LXhR>IrNM68pNG#%%?y`4L;hk@n=5K@qrwsd%q%Tz)5KKnP+x7z+U#3 zoI~0KU#iVXl&1Ny@h?vncgscP12P4f^VMI^whUj%aEOAG$)^*QzRh==cqK`nNuhn2%7lvI6ig~a;}*OE zXnL#oQ1r);5fzz2@tK4fKAlQMw~?6zhucK^>+RG>5d$@+lTG~FKc0qo?AKzO)&c|$ z(@*ShaUfM!Roz%E`vp39&A$rdo^z^6CHn?!YO(gNN4T_yB5~l2=@>+g`()S|VPJ9u zc@B*ivc>IAVn*Ow#v?TVO#%Vmk39}?`>K$}V(G+9XZGSFn*?INhuMz|$pF&mVA|q1 ziIFMgPrg^g=gs;@W^lppjY%-NVr6KOl;xR>B3ddV`ol?+H@Hna2scgk>N)EMo)YeB z6==Nh*Dv}=2PnSAm;VW`Zeg=vUhk8qjd`5So-2`)jBTnZ-BLP%=e-mJxO&?pwPm+B z=!xBL>5xq?Ady(d4*0YFfW2bh&Vf-7lb?WZ`0_Z{?zQaO^YnaCS=NL@7`9q{7Rm-Tv-1latsfjNZGjS8u>FkK1dj(o3`^ zH#lxaWrd)edTlM?p=Rc<7ZkN7%w1utnOZI@iJD;Ckr5|3^8~si`iItE34%E_i@+Kw zA|mpp?26qWCml8$oN*@4@|H*+;`q^+zn)_ulb=pOWO3E_`8-CnW#0`%gGfeIN!Ay` zDuH&#i@=GP^4gngyS2%5cF4}fruteo$)f6eMG!_aT1%P4){w`bVTLfrabE~TN_RWH z1s+XHfK!ere3T}I+u}F}{!$=(kAUJ*;4oDY35OQ^H96j1;~@utR56#2ZOSrT^_25G zO><~ndQ~{o?Yq1fc$xzZB6foNIk(7AqrmDc9X#0=4o*$th;Mh9Dm*u5i@iggSU0H zS$X?A2Dity+2@r(h_O%n<;7~&Tb=S^Wz0Vr*uCMf699FdGy=Qe-T-zUs0gu zcIZGWSg;d)Wh=%yEr*)vd$qFl7u%IUCe*M=+sOt*AFDFR;)x_NhI~VDFA;sM4;5m7 zq-BXbw+){JnAfF!b*G>(TXMCuFXoGNCj!&C-k1B+NOh|xYHtwmvpR$9hQ zaneIpr!7I>l+ti#va@_ICauNcjPu_nUfUNN7dbNc-oK~Ls)sh_m59!ICc(^^B;*kF z8)_7jslVHq64yyJiXo+#P^j5LJ}+aabv-bYt)f1v>{GM9HrIRp*vbEa%M^6hm4xu3 zjm5%5xP;dzPL}S)5J}2$MjmcFe@3=l!PV2sayrJ8k9E{`g>2~U^W!SL10sb07;!V^Ocy{oVHb?QoaZq-R?{Q;o-O{*)5McsuA! z@!Mxqot_9ZfCkgg8q3U4vo2bnr)iP>+N1@CDlt{6BW(1~kXzn(g6ht_WNjPu?Lirw1cqaZ(@oEelEMjw|KB zBOOzD2bqB8>;eXu*))K+CBhb|TCua;TEF5$9(Ob=)m^~HR;gUkw%NKL;hX8cafuYW zM1psJuyyTBWBtxgjferwR6<$t$jJ7i8E?Lv*i<1H=JnAy&oA>4ihwC9tiE92M z^(#8(w;#a!1octHGY6nl7Z2+5&oTJN7mZ85hmj@+%v7@Sb1bjp2sIyWA1$KwdRgA# zvGRE*CJ*yF#hcBrrg&h0z3}ZIoM+^$@43}v%bpBnjj;k=0YutFJbrMp;}s)e2uO9+}i^v>Y|>yot!sl z@aM%ey6Pucm;o#d!;R@ipKoI9B?P^6J-KC=8p*bR73eU&h*w9N{04+mhIJBO-nT@z z9wZN2#qGrRD-Ss~*IQl_@!lTzGnAXG6K9@ns?H$9a2JZbFX7BX<0gk^COosJCCEl& zmGG3|3ox#qx(Oo>_@m}ITC_4SmgD`s za;v}T+OyyCWmMlEWn{Jv3qs2Ra!iP8tt2CRxBBIfr^+8djC2D+94S+P_JsCCse()XDZUHU&tB0x)in`_lc?neZff(i zEfsgkMiYbyqxR+(zsS+i1HP17C2fzyKgr0l3-h)?So>?*n%$^W{7k2sUitEin#=1g z=;~V_P#4+3Y3D4JbQfkpS|2`#t&FMSs>c0WJ6(Gn%}uCUTwq$|X3t{l?uBnQLO+pptFHC2K*mVGw3K@`~c_fwOX_@&xa0<766&G|2xacf|OKba@R*3)3C z#aRU%Mz_pf5sfvQ-^N+%OC$!wJLZFBP*Z->;Sv$M9!D*|o?uilDGy|Q6Z#8?;)(Q; zRa>m(xuss?p&qXU1^O3~H3aYf z5A?Y;80_|e{l26Rm+bp@Ct)N@%zbt3&WzPOiAdQP=|-!wmMeRxY0WRb&nV*anitd6 z3%vm-Mw5UMc^jP8QX)L-z52I4oOUQ+Sh=8$q2H@uoa^qXO1y-v|1KnNfAt>UW>d6- zaduJtMmfPLQSOQOh$~I1v@qQQ4xf~Fk`~Vp&^&(+eRHnS0MikvSp4fNnei}wqb$rv)Bx7EJ*8gOR)8KEeLzdlGl>``*A{f1R zsOIp27}w`ZrtZaFxm6f{lmQN=hZG^5!a8&oN3QTz zNay_tY%iiHPYE6`?#ZgaxP8hOrOM3omrk$p!P9p$cErzGB6g_cU7WEkbS~fLi`9)D z#)G@P-p_=f*y7|Msc1Q1!o}LP3yQ&FqnO>--2{`M7e(7f4UCB_v1dw zem0&H#dX_P$u0~LXPaK1)&{r4;KOxN?3LQnBr7Zw42Lrhxy7H0(JUOSNIktVef4+q z$lZdYD3tRXOZG)|Md*d7H?*<`Dw2ri(^O$_=z0&r0{3_$Ax!IuiB@iVboO}7?-?(5 zqq*aT65U#S{0$fegJMv5Ii^-K0Ooo00h`g4i1&_Rrf0YEUkt(l+7~>X{HB zc~Xtmm**si33cwJ6T>#L>`AH)UPPZR!vf9W&-&fVqu&_XD55h=I7V(OnVBXL^``>3 z?dT*or#ld@Ue_<1+@D8a-KCT1=d-11>n0767qi*8}vj{t>L6Bk{_Yr}!&IwyZ8){r5-8tTk zC_9qynZ*8FBWLBe;J^XXnUFmzO>r}($i4L9qE|T6WMAt#JC>c$cknfK+0z!)Q@O!k zi!-)RW&=|}ICD!VPVa4X2x(mt7%8ii2oRp=R(}dHnLSRrZqI9kLtc=t&va|UKSyoq zp4it(vk1>eT#tb)WXLb$=}cQTvJc*u_D|z zt2cRNG9ptGpmEaeXQk26m5zRfLVHn~FmPv`hEn>vxK8f*v%z@wI0SQ7&#Yh_XkNl3 z!EXqkb+TZZ_F1H?s;wxEtSrUd;$|$k*eG=1t7^RxqTqswyf}nhEdHY*e1E#IDySPo z@eR*W`Ff$yw-a|;**&w7=IJq=Y5St%5h$&N_bo4MY<|Kuw!6s&(8RovR|~1bM1N_u zTRU8jiFfnz350VaH?Tp6fMW_kDVV;1>4vkLi;-_&GGoR=VLN9jKeih`tN0u9bsja% z1A)Xvc&UEWY64~%S32AENimxu1e0_+N{(=nq(7GA`2rkr4A<~wLrcBh3Vl`Oa(Cw) zA0MPW^j+|UeaFTWRcPR^qW#=H7M=W&tMXI7{+ank=~hn|LGzz9IfSNkFORtPB4}xt}1e1Ap_Or4khQWqBQ81sykom9A&dB#7 z29^7u0=>M=HYb{Az=Rk1CCu*UczNat4j3?#o{VJSdu<>{$O{{BEGZTpB2F%LvOCrA z{XI9if6Y%to4npkR^o)gKiwQSgQQ@*ZM=c5NBye2U^hN@$n08)_nt#kto>wJ#9=`<6Y3K8d_G=xb8J%1SlCbit&m*z`U9225YB%H1UKk@-QI z4Vn7ZV!#OtNQg zmCL9K9oWEek0YqLl=!4En?yX+`&)z+BpTXsJw>da)ZO~3Tl<`AH!lg}{@59av2x4% z1EuV>TKCs)st4An(sS%L!@VT{W`CZC79dNLjA~@hm)R?7b&z%aF4sC~&Tg*JM0SONj}ma?&`m(&1LVIjt$m zd0b9QSI8rXuP%WCcknbd^yy3OiFmhMI1LzcHVi&`<~I3w@Vndb0Qb(TeJXLrbT71e zR@9^Vw@A(`w&tIKpZ*ZNXyN@0q3MKg5SITZI)pl!s|PLE7T%+)IM@$aA0T>LD61 z`&(;6PIU^js2N;M0Z)N=)G@j(<~GeU2=V?pyuD0D=DGP2H)sqx{cU_~F%0e^ZvJ_* z$6>1HT=CL17_Ev8ZNz5P{=oOhnRj>f42deAOx}R$U}$?nc44A<^Hl>>=i~()1`(Yy zUGF<}AmHrKOzFC|y~#wsAffgoS#A;$?2y8$V76S)U5*R`8o|vE;fJ7=%nG|G8J<#i zsrl#SnEeoWkLtu^r{aF8qrV>*uJTP69Q!x`3Y~yaJDg=Yi@iNwhc)*s1?(wMf z^sT~3F1aZibBP5S+zvgb5I!i&%m7*?RTz#O9nv91CDz(7mc;z08O5L}7ymh1KlwgJ zw`7g!uTYXIVrJjqE|Z-J6LOPB?obT$sts1~X#vqsFhrHt6qQ-4NjD&5@WBktPRw!J ziXjMGRJz#G9o^G&EO84}RZq78o~4dQ21buld%$*MdExW4B8Ow$0ZVegD`oqY(ku zCdKeLOn3YQabbgAy;fh)ZfFJioO8DtR)EBU0q-2lDuZ7-oaLdgUElNK(t3tQ5loOk zvy^@G#mNofdmXxA?4b;+2-aTre~FDV$PI6 zl{zrS!e(l=<}k2{uX5vWErD~=^zDZutD}ZN+iJnr)qy$Ap%Vb;RUUpas>E@xJ^!UbLXk&qsiNnk{$t*;XW-AdiynU(>F$GD=}Vo=%~BE2|!8$DhDtN$xF*C zP5M60Md`(;O~AQw=jPrqzVGr==uUTPH|j37HY_IvjRcPV?p&G^zc$&4_-Ukjl{Z`# z(m#dYx8Q)MpBbjQngo;?Y7L+(9LD-BK|EzE%;R-4OKhxjS=wFqnFy*uEzy0z@?QSl zV{S>Abb>Wl_EUe_r<*$q{D)mUj6h5RP#}D#Obs43Bb7UzCUCGwL=#_8CS-f z&dUwr$tgp~g7=r|UeJ@-DcTb5?cRm5tZ|D!rXfyNmB6*}!)=DjX{GMM)+tlzj(&`zDUzu%vfB4l!MdVVm=gB}}L*>A|Q zdRad8u|-NE%FEqXk9W&)83a2Ea<*zm=rv>tUtH51%pXT22_3rL>l%JP#zpr1Av$2~ z2Z|VPCTHwKj|-s5SFSDA+C=tBUV}rN$a@SxJ6pC+j{Jr4E|H#}I^HJ#`nuH#L8!}&jFeV_bV5!KSk-=!dgLu;=_b-k-FK-$V3U~ zH)Vm>d4}1_xYq-KcfbI=uVZU|sk;}7-hJg|{QO{D+WH3Ym%is_sM9zj@+{ldp#v>Cjg4_B%|@SL%bl4BB2K`wF|x#0WB4!d2D7I&I3>7B>+! z=}G&`n;7j0kj70G$sW8D&Z$X&O|FiCe8 zn#Fnu^tmJKg`b9!-LFV~IaO}?w#9{uJVr6e51nH7V;2**%%+u<3zt}}ZXFJXZp8EI zsvoYoy}zLE;Jd17)*1VEmAs9B0+!sFCX4Qu37j_kw}Eyd=_`&?X}E!zx~0tL zLjEFMRCY_!p{ESpLGfNmQG1C?>>%+U^8+OZL|#t9XfwvsARwrAA6ftp9il78i98O8 znhts=A`Q%g=Y`A`rA>3UjG%7B@)sr@iSE;RhXx5F5xlc<*r3&s)5Wbx#5ji{K|5!g z)5&i-L3`;tVi6)#shw=0H7!_Vyp6Ad%iA~+(j5`yPlaE9aS8%@*o<}IVA3Ql&y2`Y zzX4;X3ddKRoPqOT9RKiLft**x9!~0o8F(pWInr^*57eFok@)MFY+1cqV8 zd5Ms4-L>tbo^9JPNv3lAZxKeZVi|xeQRn>~(maqdv8{3p-D_WBzF};OEOAQ|9w7G3 zv3Z%w&4>ZseCKwM}e-?m8A^p4Of{0P_DEHKWwI|nBAhaL4G zOw7S7&X!&u%2BeI5D|OX)UgTglb z%jRJuYM||8Pp5V9!`+IM z%{6fV-M?6lOCMeHed4V8x+xtS1*t&Kne~&|TzP?|3?lYH_h7>G2RHVfE>`iN4`#zu zZxdR|GR5j9DROktA%)p9R4P{;-B4f)IISb^JJlzm&tkUtVzYYeP+7asMauH4Y~e&F+06m|5$B# z>{IHxaF~Xqq?9e3QuiI)GB{nb!b9@3IFvE*d)#lNKlAZcgZ*|M1tJe@{YrCUbXkP3 zu@zI6tJe-54m3c-a2v#& zHo}<@uw#iyV`P~@=n_c=d;K_W^v*_d2X=f?dH6lFUAi(?G~yuID2oCa--4Oow>CbC zcL_;#oy4{LDaEs|j`d5*>3(NlMd;Od!xosHOk+qlMh$4VrFy5?X~c8mrw&ZzVx$NZ z^J_Qe>OZ8YEnsKs^Mq-1s+F!UTt%Ig0~QwURWi(^hiNqN@XERg21SDEy-XmtoJ?jk zzN?PSe^QIK^8}bvHmkX3DIhBynnwEuX5FsKX-yr<2Bd%gi@N1bK-y5-j8kwlw&B3Y zeZEkxNiwB&(!qojAy^$MmIszW`JG8YOLqi{^f_b`dzQjR(8v#P7*Y_{FX=fr*0V}NTsEAR@1KA?+W;RnDS?cc7514o#yT-e zIuvXj4YtThB-{>k+cBdHsl@k2Uo%xty=R7JKy>O5o@(fmbmDihl%kkE-4}Y;0|W}V z4W^v-RBn!>jpz)#hg?5Z4uL`UgQnR~N>E+-7|@AkBV>Dfi?BOO)-$+ZwUda>I}nQd zry^#(D@zik$7vjRbPt?N2)Cj$r*WMal3I;3qe2zdJ3P#G)Tcy;3MXHDjpAVP#ZX{th)2-=KWM@!GI^WEuwA=1pB5c5Vh zM7>79dpPj0ZjzmLYVD@_D5}!QOmiC80%=|iT?kvg&0o)7QFmMn35}}Y|HT4d&8MMW zwrF~t%Qj%`oYC&q(;fGxATUmWm&k>@vzyPXJggQj z9HSJqi}Wn+3H$xMeDgfVEGFaF8X&}E)s7Z&16*lziQTwBT3#C3#;ebwg}s2acM1gQ z_C0{Q31^M(qmF&XXN3?!Id4VVXzQ_yf4JeMIZ}&%EM&GEZiC(HG~65RHx4PsHCI&S zYw%jP*YD0Zf5tW%c&$E%T!9I`63y;hl_}X|<@{S8c>BBG9(jjbeU4q+k+(`l+9h}L zaDq+7W{340o^j3h`Qk;w|NXVuG6;Iox3j&x6A@^Al3uKh{q!WWow3$46xcx(UGyW4 z8%U^)b&}l4%q2m#Z0=bwd3rHJu27&=u!_OCxIa8Yo4m`vJ|mKGJP`RIx!tM z$;E)bHQD82X=x=LAVxEjJ68X^d{KcWIYS$I%qR7z+EF2w?zLwK zsbQ9CF)(IHFubibt~{Wv!_pVnPpnxSI5(TGT}}<>&XDTUd`fA7MBt$Emrwm9@ZHiR^$8#GDRxkUQE*uV!m)7C5@*;!0v zB(hwncNu|*e#{DVmv&E}u%KFwoTN=Og$<>pJ@Q@TF7}?bJKsMmhy~0-4kA2T6;nrw zJzD4xa!wzP)l4pjzUT}4q8Xs&?Pumvm)2+^3L}>p;@Ts3-N6Ve>zU8-Jb$<`V8QyE zL4PEi`!~ZW1;<|bhdm`*+gMh*u!?>Ad_Cn^1+}NtoO_JKS!K8X@XcV zk&~8o_Q?kKYPWNk}Nv06ZToGNl(pju{MYo zNtbhLj3{4-hx0gn5GNbC8g)5~ss>>XN?nP_79^Gd0Y9C@a&ZM!ENa6GF%qu8drv^K zY>?mn(2uUSR@PjRrIYa5YIV-C{7m+VA-7Wa5)k}YA^J7V{sc6F{eowpVieYW(%>aU ziZTDvvqPdqj1?+(E60J!?ceYF%P7bN2Xh;r3S& zPQxdeN5#H7kED*fgd|xz*G7uJ(F1SLn48~kz4oz!Zx$Qk@W0&jW~4miXg?&60R~~9 z=66!X`b20o|ArP&?Fz{yUaTPEhUOVGKhVf?j#{z&S2N(pHI?7_|DxRrAJCCK}iDX1RuWMfUD`L54+Rx2C zqn)a1hu;ynMjFL209=_v4>WSAf21a0X>`l1wD*YY7{}jRI<2_)ZR{-(I^2`dbbhAB z;#p!{5Icnrr29pD-NDz;c;)KE>#;jKx5$reBT}y0`8g7iPP~oNx^?F`CeNW`rjKRX zd@Hu^&W5328W~iX6T@?^XWkyM=_0^Ml2;1EP#~8%JLK9fzpi&;R7;YczwGqg={_X7 z*P}v=-YN2{C=yFYx^0AW$)p~gd%D&L?>w!>1zlojXre1dBvwe6hry}jt$Ir9X|~8) zBImvArkw(az@=V?kr!uY1~S%ej~|OjCi!cH+N>V~1(lG6NK#D+yjRH>PWV%Tu8+m5Vh zeK6lDw)v&Jb{r~XuIijp8-gaE8PaaIZ}6D<{obx67MHK?$fZ^TZ(n*jfI80>x#4wI zeMa!T7RNEK>TYy^K4&~9HQ2xELeh?inaqd%dzuog7=edR%Y_!rzG6w6GUr7Ed1qCJ*_0VrM4-E`5{wdHLmy1 z%a$|EtG3YZk~e_$7!E#DGc-h#IchA>&)OyYC0FZ-Rl|ta$+70#nIa`q*?s}G5TpD% z6#`z%&>900wJo%YTQuv2z>eegRr6$OuOJf5C$URoKM(ejJsz)&keFA?U|Rj9)3ZKf z(>}BS?$|p|0*#aJ1^g7=5_5;b3lQ^vIXI$`+RLG(`)vwSY9aI8{2c|TD-12@P)p6W z^rXcmwSRSmyU9i*PD=2XS^>{~d%6z{dnm3Kf}T4r@IrFrD@oC5a?~7$kq;9$+&s+E zkcZx}}I6)vKwRri4uQlI1UfN=0!ird{o zc0Njd=$_2ojqNg*$Z5mwX^>WBG{x<;KUvQU(9FHWAKa>mI)^CdZ+SDhQHHllQ`cC1MKO&E1GrR1Pq16IJ2bKXY`# z4Ei-3Xi8oL@QwHk(RQB;+S4kFKFyN4Rptt=<9$}~SE%-U!8-j%an#Q1h-gz@mu+~m zjlh|0evC&ar|iDcovuN-Jojf9R4wjW7)Ms(NcY&VmX%#lqPQ!cITOF>Gh2*%$?JS--PU5U-BFg6KS&2Y`@NUeN+l zd2~o_l>wnYqFSZ|Hy(!NiWIXDu_XZ#0Ek;26DeuQ%PaeO3sQ}O`HBxmYY=%ivKGLI z{wW{EPhw)|>oDg4o=L7HVuVrfey!8ouf6XQ`Z3r{5|bj2>N2n3%)j#Yo_|ZZsfQmC zc)vr1I~%V?0`Br1t>6CyBPI2a&O}q!G^9xVqpBTAom$i-C~;M zl^;{W6EVsF)W8NtyFxT9=V_1GZG^1H1;w{y^a}m9%<;^N!Ey=HV8q_ecUuW_uAlmi z$Mok&$P`V>ae|2{gxkpF=v~dnBzR+O?{%Hx@J$(})J@_FHdt>aumYZaVVUjD=@&n&@q zPfNH!qijltc0p)B3}}YN-bOmSaY|_40b@XXH9;_5!jIF4nm)n&)DYmt{=7`PloaIT=(o>#*xXdFQ8Y5A#fHY(=`br!iF6>#&Ujc}D1r1#unF zE>b6_p)-vY#Vei*o^9`t`yP&0az?B=p#R7;5VeP~dJAK_Sqc(9zNz=O0&wlWg?#5= z{|Nb16a+PJJr*sfY@#HjG0Zzeu;6(LP(qo%k9`SyA$-@{g zjOe^4?r}U%TAymb<@C@RM5|!S6F>8*hc^0c=;={g{=&2Kees{Z!|NI%zhh2990wja zy?rxUNeu*8Gv{^Xn8HZUPUx=ozQVeyMx{;8%XOUFn%5bV&YD75^P8$B{zhUzLbBjm zJl<%yDsfQKohC)Ib=p&L=R+8Z^-iB*}bndXt+Nstt zUN3eJzUL#<%2FgY3@mL`hDl+h-2kSqz2eNC_nGn3q-<^by6`0A=Cx&30~#r?y$JwNa%bH<89{Vu(>#bm|I0GQV=OaGZ`pZBj+d(I-nJ=b}(R z4bYMzVQ)Lw;G%+A-QXw~1fGpOf1jj-Q5ZE}AvQwS$D@No8-i)aP=l9TR=KDZ6p$B1 z3C(q)j=gHoj}s44KEhqvD^0qSr?A_K?O|*`x%x}UX>ytUY{&UG=|&Yp?~L|izxe;i zJ<68<|Bzi(nt2S%Y*t-{AK6Txoy5aR6z_=_vG{|@hC~0_z{b&E4`m0}rs|VvkUrY` zlJB~jDqeEeI*R>Huzf#|X*{+s*6RU>aYb=3Y_>ThloZ?ds6#o7R-Be8eLCY591 zN72_wE(NkuL;#x!N*6Y=Myr-o*oWB*a{Rbxz@xI(^e5Ec^eILJq??b;@IUEO(D5oE zKR^6ddB+PC4Y+`4^7yAu4nJ#pH3l)`l6@rxoIF`Lehfe{JOsX2gDVFZ2$lO From 7eb685334f1d9322f859634666984029e3c70283 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 30 Jan 2018 12:36:49 -0300 Subject: [PATCH 022/246] improve param names and command spaces identation from the deployment steps --- azuredeploy.json | 20 ++++++++++---------- deploymentARM.md | 7 ++++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index d0fcafde..34c7169b 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -81,7 +81,7 @@ "description": "Type of the storage account that will store Redis Cache." } }, - "delivery_redisCacheSKU": { + "deliveryRedisCacheSKU": { "type": "string", "allowedValues": [ "Basic", @@ -93,7 +93,7 @@ "description": "The pricing tier of the new Azure Redis Cache." } }, - "delivery_redisCacheFamily": { + "deliveryRedisCacheFamily": { "type": "string", "defaultValue": "P", "metadata": { @@ -104,7 +104,7 @@ "P" ] }, - "delivery_redisCacheCapacity": { + "deliveryRedisCacheCapacity": { "type": "int", "allowedValues": [ 1, @@ -117,7 +117,7 @@ "description": "The size of the new Azure Redis Cache instance. " } }, - "delivery_redisDiagnosticsEnabled": { + "deliveryRedisDiagnosticsEnabled": { "type": "bool", "allowedValues": [ false, @@ -157,10 +157,10 @@ "Secret": "[parameters('servicePrincipalClientSecret')]" } ], + "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", - "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-cosmosdb')]", @@ -268,9 +268,9 @@ "properties": { "redisEnableNonSslPort": "false", "sku": { - "capacity": "[parameters('delivery_redisCacheCapacity')]", - "family": "[parameters('delivery_redisCacheFamily')]", - "name": "[parameters('delivery_redisCacheSKU')]" + "capacity": "[parameters('deliveryRedisCacheCapacity')]", + "family": "[parameters('deliveryRedisCacheFamily')]", + "name": "[parameters('deliveryRedisCacheSKU')]" }, "vm-size": "P4" }, @@ -289,10 +289,10 @@ "metrics": [ { "timeGrain": "AllMetrics", - "enabled": "[parameters('delivery_redisDiagnosticsEnabled')]", + "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]", "retentionPolicy": { "days": 90, - "enabled": "[parameters('delivery_redisDiagnosticsEnabled')]" + "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]" } } ] diff --git a/deploymentARM.md b/deploymentARM.md index 5a5e9080..b8af1e23 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -57,9 +57,10 @@ Deployment * using Azure CLI 2.0 ```bash - az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json --parameters servicePrincipalClientId=${SP_APP_ID} \ - servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ - sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" + az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ + --parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" ``` * from Azure Portal From f355d3038143a27edb9fe0560e742affffe796ff Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Jan 2018 14:21:18 -0300 Subject: [PATCH 023/246] reduce ARM params, flesh output out and use them from deployment steps: 1. expose ARM template outputs for auto-generated resource names during creation time 2. az CLI to query secrets --- azuredeploy.json | 11 +++++++++++ deploymentARM.md | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/azuredeploy.json b/azuredeploy.json index 34c7169b..e38c76f8 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -117,7 +117,11 @@ "description": "The size of the new Azure Redis Cache instance. " } }, +<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c "deliveryRedisDiagnosticsEnabled": { +======= + "delivery_redisDiagnosticsEnabled": { +>>>>>>> reduce ARM params, flesh output out and use them from deployment steps: "type": "bool", "allowedValues": [ false, @@ -157,10 +161,17 @@ "Secret": "[parameters('servicePrincipalClientSecret')]" } ], +<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", +======= + "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", + "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", + "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", + "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", +>>>>>>> reduce ARM params, flesh output out and use them from deployment steps: "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-cosmosdb')]", diff --git a/deploymentARM.md b/deploymentARM.md index b8af1e23..188bdadb 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -57,10 +57,16 @@ Deployment * using Azure CLI 2.0 ```bash +<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" +======= + az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json --parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" +>>>>>>> reduce ARM params, flesh output out and use them from deployment steps: ``` * from Azure Portal @@ -389,4 +395,7 @@ You can send delivery requests to the ingestion service using the swagger ui. export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST ``` +<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c +======= +>>>>>>> reduce ARM params, flesh output out and use them from deployment steps: From 8e70ae33f1c7aaf87f1a3c5410b0034e8352e047 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 22 Jan 2018 15:32:49 -0300 Subject: [PATCH 024/246] align delivery and package services on how to create cosmosdb database and collections --- azuredeploy.json | 11 ------ deployment.md | 13 ------- deploymentARM.md | 35 ++----------------- k8s/delivery.yaml | 1 - .../Services/DocumentDBRepository.cs | 12 +++++++ 5 files changed, 14 insertions(+), 58 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index e38c76f8..34c7169b 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -117,11 +117,7 @@ "description": "The size of the new Azure Redis Cache instance. " } }, -<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c "deliveryRedisDiagnosticsEnabled": { -======= - "delivery_redisDiagnosticsEnabled": { ->>>>>>> reduce ARM params, flesh output out and use them from deployment steps: "type": "bool", "allowedValues": [ false, @@ -161,17 +157,10 @@ "Secret": "[parameters('servicePrincipalClientSecret')]" } ], -<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", -======= - "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", - "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", - "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", - "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", ->>>>>>> reduce ARM params, flesh output out and use them from deployment steps: "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-cosmosdb')]", diff --git a/deployment.md b/deployment.md index e073317e..27bac8c3 100644 --- a/deployment.md +++ b/deployment.md @@ -99,19 +99,6 @@ az cosmosdb create \ --resource-group $RESOURCE_GROUP \ --max-interval 10 \ --max-staleness-prefix 200 - -# Create a Cosmos DB database -az cosmosdb database create \ - --name $COSMOSDB_NAME \ - --db-name=$DATABASE_NAME \ - --resource-group $RESOURCE_GROUP - -# Create a Cosmos DB collection -az cosmosdb collection create \ - --collection-name $COLLECTION_NAME \ - --name $COSMOSDB_NAME \ - --db-name $DATABASE_NAME \ - --resource-group $RESOURCE_GROUP ``` Build the Delivery service diff --git a/deploymentARM.md b/deploymentARM.md index 188bdadb..97cb086e 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -57,16 +57,10 @@ Deployment * using Azure CLI 2.0 ```bash -<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" -======= - az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json --parameters servicePrincipalClientId=${SP_APP_ID} \ - servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ - sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" ->>>>>>> reduce ARM params, flesh output out and use them from deployment steps: ``` * from Azure Portal @@ -103,28 +97,6 @@ kubectl create namespace 3rdparty ## Deploy the Delivery service -Provision Azure resources - -```bash -export REDIS_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryRedisName.value | sed -e 's/^"//' -e 's/"$//') && \ -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value | sed -e 's/^"//' -e 's/"$//') && \ -export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ -export COLLECTION_NAME="${DATABASE_NAME}-col" - -# Create a Cosmos DB database -az cosmosdb database create \ - --name $COSMOSDB_NAME \ - --db-name=$DATABASE_NAME \ - --resource-group $RESOURCE_GROUP - -# Create a Cosmos DB collection -az cosmosdb collection create \ - --collection-name $COLLECTION_NAME \ - --name $COSMOSDB_NAME \ - --db-name $DATABASE_NAME \ - --resource-group $RESOURCE_GROUP -``` - Build the Delivery service ```bash @@ -149,6 +121,7 @@ Create Kubernetes secrets ```bash export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value | sed -e 's/^"//' -e 's/"$//') && \ export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) @@ -394,8 +367,4 @@ You can send delivery requests to the ingestion service using the swagger ui. ```bash export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` -<<<<<<< 98fac7fe73a75557c3daf6d232fd8dc8de228f6c - -======= ->>>>>>> reduce ARM params, flesh output out and use them from deployment steps: +``` \ No newline at end of file diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index fc6f4c7a..d35da49f 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -65,7 +65,6 @@ spec: - name: DOCDB_DATABASEID value: "CosmosDB_DatabaseId" - name: DOCDB_COLLECTIONID - valueFrom: value: "CosmosDB_CollectionId" - name: REDIS_CONNSTR valueFrom: diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs index 825d0adb..0e671d08 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DocumentDBRepository.cs @@ -35,6 +35,18 @@ public static void Configure(string endpoint, string key, string databaseId, str client = new DocumentClient(new Uri(Endpoint), Key); logger = loggerFactory.CreateLogger(nameof(DocumentDBRepository)); + logger.LogInformation($"Creating CosmosDb Database {DatabaseId} if not exists..."); + var taskCreateDb = client.CreateDatabaseIfNotExistsAsync(new Database { Id = DatabaseId }); + taskCreateDb.GetAwaiter().GetResult(); + logger.LogInformation($"CosmosDb Database {DatabaseId} creation if not exists: OK!"); + var dataBaseUri = UriFactory.CreateDatabaseUri(DatabaseId); + logger.LogInformation($"Creating CosmosDb Collection {CollectionId} for {dataBaseUri.ToString()} if not exists..."); + var taskCreateDocCollection = client.CreateDocumentCollectionIfNotExistsAsync( + UriFactory.CreateDatabaseUri(DatabaseId), + new DocumentCollection { Id = CollectionId }, + new RequestOptions { OfferThroughput = 1000 }); + taskCreateDocCollection.GetAwaiter().GetResult(); + logger.LogInformation($"CosmosDb Collection {CollectionId} creation if not exists: OK!"); } public static async Task GetItemAsync(string id, string partitionKey) From 686c74570f4f4457e32c13f383ed14ab9227e377 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 2 Feb 2018 13:50:58 -0300 Subject: [PATCH 025/246] revert remove dup config for RedisCache --- .../Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs | 1 + .../delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs index 95a834c9..ca944bb6 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/Constants.cs @@ -9,5 +9,6 @@ public class Constants { public const int PartitionKeyLength = 4; public const int RedisCacheDBId_Delivery = 0; + public const int RedisCacheDBId_DeliveryStatus = 1; } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 8a6b40fc..592e7597 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -88,6 +88,7 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF //TODO look into creating a factory of DocDBRepos/RedisCache/EventHubMessenger DocumentDBRepository.Configure(Configuration["DOCDB_ENDPOINT"], Configuration["DOCDB_KEY"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["REDIS_CONNSTR"], loggerFactory); + RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["REDIS_CONNSTR"], loggerFactory); EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); } } From 8f5a8e2dafe6670940e1051050344be32c880780 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 5 Feb 2018 09:28:59 -0300 Subject: [PATCH 026/246] fix uri for ARM template button --- deploymentARM.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploymentARM.md b/deploymentARM.md index 97cb086e..73621f9f 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -65,7 +65,7 @@ Deployment * from Azure Portal - [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Ffeature%2F48_AddARMTemplates%2Fazuredeploy.json) + [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Fmaster%2Fazuredeploy.json) > Note: > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group > 2. paste the content of your ssh-rsa public key file in the Ssh RSA Plublic Key field. @@ -367,4 +367,4 @@ You can send delivery requests to the ingestion service using the swagger ui. ```bash export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` \ No newline at end of file +``` From 838d63e4f423bbd7b5b0a4e08838c0397dab0af1 Mon Sep 17 00:00:00 2001 From: cfarre Date: Thu, 8 Feb 2018 12:19:24 -0800 Subject: [PATCH 027/246] added integration tests+profile --- src/shipping/ingestion/pom.xml | 29 ++-- .../ingestion/IngestionControllerTest.java | 3 - .../dronedelivery/ingestion/IngestionIT.java | 125 ++++++++++++++++++ 3 files changed, 146 insertions(+), 11 deletions(-) create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 28ddf400..ab779f5d 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -45,13 +45,6 @@ test - - - - - - - com.google.common.html.types types @@ -170,7 +163,27 @@ org.springframework.boot spring-boot-maven-plugin - + + + org.apache.maven.plugins + maven-failsafe-plugin + + + integration-test + + integration-test + + + + verify + + verify + + + + + + org.apache.maven.plugins maven-dependency-plugin diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index 50e755f9..dc51a1cd 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -22,9 +22,6 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -//import org.springframework.test.web.servlet.setup.MockMvcBuilders; -//import org.springframework.web.context.WebApplicationContext; -//import org.springframework.web.context.WebApplicationContext; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java new file mode 100644 index 00000000..9376a9be --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java @@ -0,0 +1,125 @@ +package com.fabrikam.dronedelivery.ingestion; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; + +import java.util.Date; +import java.util.UUID; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; + +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.web.context.WebApplicationContext; + + +import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; +import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; +import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; +import com.fabrikam.dronedelivery.ingestion.models.ExternalRescheduledDelivery; +import com.fabrikam.dronedelivery.ingestion.models.PackageInfo; +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.springframework.test.context.junit4.SpringRunner; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class IngestionIT { + + @Autowired + private WebApplicationContext wac; + private MockMvc mockMvc; + private PackageInfo packageInfo; + private ExternalDelivery externalDelivery; + ExternalRescheduledDelivery externalRDelivery; + + + @Before + public void setUp() throws Exception { + mockMvc = webAppContextSetup(this.wac).build(); + packageInfo = new PackageInfo(); + packageInfo.setSize(ContainerSize.Large); + packageInfo.setPackageId(UUID.randomUUID().toString()); + + externalDelivery = new ExternalDelivery(); + externalDelivery.setOwnerId(UUID.randomUUID().toString()); + externalDelivery.setPickupTime(new Date()); + externalDelivery.setDropOffLocation("Austin"); + externalDelivery.setPickupLocation("Texas"); + externalDelivery.setExpedited(false); + externalDelivery.setConfirmationRequired(ConfirmationRequired.FingerPrint); + externalDelivery.setDeadline("LineOfDeadlyZombiatedPeople"); + externalDelivery.setPackageInfo(packageInfo); + + externalRDelivery = new ExternalRescheduledDelivery(); + externalRDelivery.setDeadline("deadline"); + externalRDelivery.setDeliveryId(UUID.randomUUID().toString()); + externalRDelivery.setDropOffLocation("location"); + externalRDelivery.setPickupLocation("location"); + + } + + @Test + public void ScheduleDeliveryITIsAccepted() throws Exception { + MvcResult resultActions = mockMvc.perform( + post("/api/deliveryrequests").content(asJsonString(externalDelivery)).contentType(MediaType.APPLICATION_JSON)) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().isAccepted()); + } + + @Test + public void RescheduleDeliveryITIsOk() throws Exception { + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + MvcResult resultActions = mockMvc.perform(patch("/api/deliveryrequests/" + deliveryId) + .content(asJsonString(externalRDelivery)).contentType(MediaType.APPLICATION_JSON)) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + } + + + @Test + public void CancelDeliveryITIsOk() throws Exception { + String deliveryId = externalRDelivery.getDeliveryId().toString(); + + MvcResult resultActions = mockMvc + .perform(delete("/api/deliveryrequests/" + deliveryId).contentType(MediaType.APPLICATION_JSON)) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + } + + + @Test + public void ProbeDeliveryITIsOk() throws Exception { + MvcResult resultActions = mockMvc.perform(get("/api/probe/").contentType(MediaType.APPLICATION_JSON)) + .andExpect(request().asyncStarted()).andReturn(); + + mockMvc.perform(asyncDispatch(resultActions)).andExpect(status().is2xxSuccessful()); + } + + private static String asJsonString(final Object obj) { + try { + final ObjectMapper mapper = new ObjectMapper(); + final String jsonContent = mapper.writeValueAsString(obj); + return jsonContent; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} From 7bce06c8ee17f45874b5825aadb6a62e832d6645 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Fri, 20 Jul 2018 10:36:04 -0700 Subject: [PATCH 028/246] Update Maven version to 3.5.4 (#102) looks good --- src/shipping/ingestion/Dockerfilemaven | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shipping/ingestion/Dockerfilemaven b/src/shipping/ingestion/Dockerfilemaven index 2188d84a..6fca0b81 100644 --- a/src/shipping/ingestion/Dockerfilemaven +++ b/src/shipping/ingestion/Dockerfilemaven @@ -1,8 +1,8 @@ FROM openjdk:8-jdk -ARG MAVEN_VERSION=3.5.2 +ARG MAVEN_VERSION=3.5.4 ARG USER_HOME_DIR="/root" -ARG SHA=707b1f6e390a65bde4af4cdaf2a24d45fc19a6ded00fff02e91626e3e42ceaff +ARG SHA=ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ From cf4576d815c20e610e9f98ef8b3332233d77414f Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Sat, 21 Jul 2018 01:45:37 -0700 Subject: [PATCH 029/246] Update maven version --- src/shipping/scheduler/Dockerfilemaven | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shipping/scheduler/Dockerfilemaven b/src/shipping/scheduler/Dockerfilemaven index 2188d84a..6fca0b81 100644 --- a/src/shipping/scheduler/Dockerfilemaven +++ b/src/shipping/scheduler/Dockerfilemaven @@ -1,8 +1,8 @@ FROM openjdk:8-jdk -ARG MAVEN_VERSION=3.5.2 +ARG MAVEN_VERSION=3.5.4 ARG USER_HOME_DIR="/root" -ARG SHA=707b1f6e390a65bde4af4cdaf2a24d45fc19a6ded00fff02e91626e3e42ceaff +ARG SHA=ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ From abebf02443c2716140dbc9bae9fb02018dc0c31b Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Sat, 21 Jul 2018 02:24:41 -0700 Subject: [PATCH 030/246] Fix build error --- src/shipping/scheduler/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/shipping/scheduler/pom.xml b/src/shipping/scheduler/pom.xml index a3ae66ef..7bb4e584 100644 --- a/src/shipping/scheduler/pom.xml +++ b/src/shipping/scheduler/pom.xml @@ -9,6 +9,11 @@ UTF-8 + + spring-milestone + Spring Milestone Repository + https://repo.spring.io/milestone + alfresco-public https://artifacts.alfresco.com/nexus/content/groups/public From 0ce03996a1b6073a899faa08dc55902c8d8c539c Mon Sep 17 00:00:00 2001 From: Kirpa Singh Date: Wed, 15 Aug 2018 21:50:36 -0700 Subject: [PATCH 031/246] Feature/scheduler leak (#99) * Minimal changes to fix bugs and add unit tests * Fixing the yaml attributes to match with code * Moved junit to universal scope as mvn fails to distinguish the folder structure between src and test. * Moved the delivery request processing final stages out of Main to handle exceptions in a refined way --- k8s/scheduler.yaml | 10 +- src/shipping/scheduler/.gitignore | 24 +++ src/shipping/scheduler/.project | 4 +- src/shipping/scheduler/Dockerfile | 4 +- src/shipping/scheduler/conf/Config.properties | 2 +- src/shipping/scheduler/conf/application.conf | 4 +- src/shipping/scheduler/pom.xml | 15 +- .../deliveryscheduler/akkareader/Main.java | 57 ++---- .../DeliveryRequestEventProcessor.java | 97 +++++----- .../scheduler/SchedulerSettings.java | 5 +- .../StorageQueueClientFactory.java | 31 --- .../StorageQueue/StorageQueueTest.java | 59 ------ .../compensation/RetryableDelivery.java | 45 +++++ .../StorageQueueClientFactory.java | 27 +++ .../compensation/tests/StorageQueueTest.java | 55 ++++++ .../models/invoker/DeliverySchedule.java | 9 - .../scheduler/models/invoker/Location.java | 14 ++ .../services/AccountServiceCallerImpl.java | 11 +- .../services/DeliveryServiceCallerImpl.java | 12 +- .../DroneSchedulerServiceCallerImpl.java | 10 +- .../services/PackageServiceCallerImpl.java | 10 +- .../scheduler/services/ServiceCallerImpl.java | 84 +++++--- .../services/ThirdPartyServiceCallerImpl.java | 9 +- .../tests/InvokeMockBackendServicesTest.java | 157 +++++++++++++++ ...onOptions.java => ModelsAndUtilsTest.java} | 52 ++--- .../services/tests/PostDeliveryRequests.java | 55 +----- .../services/tests/TestBackendServices.java | 180 ------------------ .../services/tests/helpers/ModelsUtils.java | 140 ++++++++++++++ .../utils/IdleConnectionMonitorThread.java | 1 + 29 files changed, 669 insertions(+), 514 deletions(-) create mode 100644 src/shipping/scheduler/.gitignore delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java create mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java create mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java create mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java create mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java rename src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/{ValidateSerializationOptions.java => ModelsAndUtilsTest.java} (60%) delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java create mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java diff --git a/k8s/scheduler.yaml b/k8s/scheduler.yaml index 906595ac..246b88c5 100644 --- a/k8s/scheduler.yaml +++ b/k8s/scheduler.yaml @@ -47,10 +47,10 @@ spec: value: http://package/api/packages - name: SERVICE_URI_THIRDPARTY value: http://thirdparty/api/ThirdPartyDeliveries - # set this env to read from period of time - # otherwise is from last known checkpoint - - name: CHECKP_TIME_MINUTES - value: "0" + ## set this env to read from period of time + ## otherwise is from last known checkpoint + - name: CHECKPOINT_TIME_MINUTES + value: "0" - name: IOTHUB_EVENTHUB_NAME valueFrom: secretKeyRef: @@ -77,7 +77,7 @@ spec: name: scheduler-secrets key: storageaccount_key - name: STORAGE_QUEUE_NAME - value: "CompensatingTransactionQueue" + value: "compensationqueue" - name: STORAGE_QUEUE_CONNECTION_STRING valueFrom: secretKeyRef: diff --git a/src/shipping/scheduler/.gitignore b/src/shipping/scheduler/.gitignore new file mode 100644 index 00000000..2af7cefb --- /dev/null +++ b/src/shipping/scheduler/.gitignore @@ -0,0 +1,24 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ \ No newline at end of file diff --git a/src/shipping/scheduler/.project b/src/shipping/scheduler/.project index faa44025..fdb9e0c4 100644 --- a/src/shipping/scheduler/.project +++ b/src/shipping/scheduler/.project @@ -11,12 +11,12 @@ - org.eclipse.m2e.core.maven2Builder + org.springframework.ide.eclipse.core.springbuilder - org.springframework.ide.eclipse.core.springbuilder + org.eclipse.m2e.core.maven2Builder diff --git a/src/shipping/scheduler/Dockerfile b/src/shipping/scheduler/Dockerfile index 736b823a..858c8aa7 100644 --- a/src/shipping/scheduler/Dockerfile +++ b/src/shipping/scheduler/Dockerfile @@ -1,12 +1,12 @@ FROM openjdk:8-jdk-alpine MAINTAINER Kirpa Singh -ENTRYPOINT ["/usr/bin/java", "-jar", "/usr/share/dronedelivery/deliveryscheduler.delivery-dispatcher-service-0.0.1-SNAPSHOT.jar"] +ENTRYPOINT ["/usr/bin/java", "-jar", "/usr/share/dronedelivery/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar"] # Add Maven dependencies (not shaded into the artifact; Docker-cached) ADD target/lib /usr/share/dronedelivery/lib # Add the service itself -ADD target/deliveryscheduler.delivery-dispatcher-service-0.0.1-SNAPSHOT.jar /usr/share/dronedelivery/deliveryscheduler.delivery-dispatcher-service-0.0.1-SNAPSHOT.jar +ADD target/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar /usr/share/dronedelivery/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar diff --git a/src/shipping/scheduler/conf/Config.properties b/src/shipping/scheduler/conf/Config.properties index 5909b105..f9d5e053 100644 --- a/src/shipping/scheduler/conf/Config.properties +++ b/src/shipping/scheduler/conf/Config.properties @@ -8,7 +8,7 @@ env.hostname.key = HOST_POD_NAME env.proxyname.key = http_proxy env.service.mesh.headers.key = SERVICEMESH_HEADERS env.service.mesh.correlation.header.key = SERVICEMESH_CORRELATION_HEADER -env.checkpoint.time = CHECKP_TIME_MINUTES +env.checkpoint.time = CHECKPOINT_TIME_MINUTES env.storage.queue.connection.string = STORAGE_QUEUE_CONNECTION_STRING env.storage.queue.name = STORAGE_QUEUE_NAME \ No newline at end of file diff --git a/src/shipping/scheduler/conf/application.conf b/src/shipping/scheduler/conf/application.conf index d4b3f006..7786e55a 100644 --- a/src/shipping/scheduler/conf/application.conf +++ b/src/shipping/scheduler/conf/application.conf @@ -36,7 +36,7 @@ iothub-react { receiverTimeout = 3s // How many messages to retrieve on each pull, max is 900 - receiverBatchSize = 999 + receiverBatchSize = 800 // Whether to retrieve information about the partitions while streming events from IoT Hub retrieveRuntimeInfo = true @@ -52,7 +52,7 @@ iothub-react { // Since the stream position is saved in the Source, before the rest of the // Graph (Flows/Sinks), this provides a mechanism to replay buffered messages. // The value should be greater than receiverBatchSize - countThreshold = 2700 + countThreshold = 1000 // Store a position if its value is older than this amount of time, ignoring the threshold. // For instance when the telemetry stops, this will force to write the last offset after some time. diff --git a/src/shipping/scheduler/pom.xml b/src/shipping/scheduler/pom.xml index 7bb4e584..5c940746 100644 --- a/src/shipping/scheduler/pom.xml +++ b/src/shipping/scheduler/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.fabrikam.dronedelivery - deliveryscheduler.delivery-dispatcher-service + dronedelivery.delivery-scheduler-service 0.0.1-SNAPSHOT Processes the messages dropped to the event hub @@ -124,9 +124,10 @@ - junit - junit - 4.9 + junit + junit + 4.12 + @@ -191,6 +192,12 @@ azure-storage 6.1.0 + + + com.opentable + wiremock-body-transformer + 1.1.6 + \ No newline at end of file diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java index 5441b752..315c6829 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java @@ -10,6 +10,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,32 +47,27 @@ public static void main(String args[]) { SchedulerSettings.HostNameValue = System.getenv(configSet.get("env.hostname.key")); SchedulerSettings.HttpProxyValue = System.getenv(configSet.get("env.proxyname.key")); - SchedulerSettings.storageQueueConnectionString = System.getenv(configSet.get("env.storage.queue.connection.string")); - SchedulerSettings.storageQueueName = System.getenv(configSet.get("env.storage.queue.name")); + SchedulerSettings.StorageQueueConnectionString = System + .getenv(configSet.get("env.storage.queue.connection.string")); + SchedulerSettings.StorageQueueName = System.getenv(configSet.get("env.storage.queue.name")); + SchedulerSettings.CheckpointTimeInMinutes = Integer + .parseInt(System.getenv(configSet.get("env.checkpoint.time"))); List partitionsList = getPartitionsList(); String partitionNumber = partitionsList.stream().map(Object::toString).collect(Collectors.joining(",")); Log.info("Reading from partitions: {}", partitionNumber); - - int checkpointMin = Integer.parseInt(System.getenv(configSet.get("env.checkpoint.time"))); - - + SourceOptions options; - // either we read from time - // or from last known checkpoint - if (checkpointMin > 0){ - options = new SourceOptions().partitions(partitionsList) - .fromTime(java.time.Instant.now().minus(checkpointMin,ChronoUnit.MINUTES)); - } - else{ - options = new SourceOptions().partitions(partitionsList) - .fromCheckpoint(null); + + // Either we read from time or from last known checkpoint + if (SchedulerSettings.CheckpointTimeInMinutes > 0) { + options = new SourceOptions().partitions(partitionsList).fromTime( + java.time.Instant.now().minus(SchedulerSettings.CheckpointTimeInMinutes, ChronoUnit.MINUTES)); + } else { + options = new SourceOptions().partitions(partitionsList).fromCheckpoint(null); } - // .fromCheckpoint(java.time.Instant.now().minus(checkpointMin , ChronoUnit.MINUTES)); - //java.time.Instant.now().minus(4, ChronoUnit.HOURS) - IoTHub iotHub = new IoTHub(); Source messages = iotHub.source(options); @@ -109,29 +105,10 @@ private static List getPartitionsList() { private static Flow deliveryProcessor() { return Flow.of(AkkaDelivery.class).map(delivery -> { - CompletableFuture completableSchedule = DeliveryRequestEventProcessor - .processDeliveryRequestAsync(delivery.getDelivery(), - delivery.getMessageFromDevice().properties()); - - completableSchedule.whenComplete((deliverySchedule,error) -> { - if (error!=null){ - Log.info("failed delivery" + error.getStackTrace()); - } - else{ - Log.info("Completed Delivery",deliverySchedule.toString()); - } - - }); - - completableSchedule = null; + DeliveryRequestEventProcessor.processDeliveryRequestAsync(delivery.getDelivery(), + delivery.getMessageFromDevice().properties()); + return delivery.getMessageFromDevice(); }); } - - - - /* - * Implementation of workflow using fluent api - */ - } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java index 4472e375..9c53af6d 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java @@ -1,7 +1,9 @@ package com.fabrikam.dronedelivery.deliveryscheduler.scheduler; import com.fabrikam.dronedelivery.deliveryscheduler.akkareader.AkkaDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.StorageQueue.StorageQueueClientFactory; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.RetryableDelivery; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.StorageQueueClientFactory; import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; @@ -33,9 +35,7 @@ public class DeliveryRequestEventProcessor { private static final String CorrelationHeaderTag = "CorrelationId"; - private static Gson deserializer = new GsonBuilder().setPrettyPrinting().create(); - - + private static final Gson gsonSerializer = new GsonBuilder().setPrettyPrinting().create(); private static final EnumMap backendServicesMap = new EnumMap( ServiceName.class); @@ -56,7 +56,7 @@ public static AkkaDelivery parseDeliveryRequest(MessageFromDevice message) { try { String dataReceived = message.contentAsString(); - Delivery delivery = deserializer.fromJson(dataReceived, Delivery.class); + Delivery delivery = gsonSerializer.fromJson(dataReceived, Delivery.class); deliveryRequest = new AkkaDelivery(); deliveryRequest.setDelivery(delivery); deliveryRequest.setMessageFromDevice(message); @@ -67,7 +67,7 @@ public static AkkaDelivery parseDeliveryRequest(MessageFromDevice message) { return deliveryRequest; } - public static CompletableFuture processDeliveryRequestAsync(Delivery deliveryRequest, + public static void processDeliveryRequestAsync(Delivery deliveryRequest, Map properties) { DeliverySchedule deliverySchedule = null; // Extract the correlation id and log it @@ -99,12 +99,22 @@ public static CompletableFuture processDeliveryRequestAsync(De } } - return CompletableFuture.completedFuture(deliverySchedule); + CompletableFuture.completedFuture(deliverySchedule).whenCompleteAsync((ds, error) -> { + if (ds == null) { + Log.error("Failed Delivery"); + superviseFailureAsync(deliveryRequest, ServiceName.DeliveryService, + error == null ? "Unknown error" : ExceptionUtils.getStackTrace(error).toString()) + .thenAcceptAsync(result -> Log.debug(result)); + } else { + Log.info("Completed Delivery", ds.toString()); + } + + }); } public static Boolean invokeAccountServiceAsync(Delivery deliveryRequest, Map properties) { - Boolean accountResult = new Boolean(false); + Boolean accountResult = null; try { AccountServiceCallerImpl backendService = (AccountServiceCallerImpl) backendServicesMap.get(ServiceName.AccountService); appendServiceMeshHeaders(backendService, properties); @@ -112,19 +122,18 @@ public static Boolean invokeAccountServiceAsync(Delivery deliveryRequest, Map { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - }); - - throw e; + superviseFailureAsync(deliveryRequest, ServiceName.AccountService, ExceptionUtils.getStackTrace(e)) + .thenAcceptAsync(result -> { + Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); + Log.debug(result); + }); } return accountResult; } private static Boolean invokeThirdPartyServiceAsync(Delivery deliveryRequest, Map properties) { - Boolean thirdPartyResult = new Boolean(false); + Boolean thirdPartyResult = null; try { ThirdPartyServiceCallerImpl backendService = (ThirdPartyServiceCallerImpl) backendServicesMap .get(ServiceName.ThirdPartyService); @@ -134,12 +143,11 @@ private static Boolean invokeThirdPartyServiceAsync(Delivery deliveryRequest, Ma } catch (Exception e) { // Assume failure of service here - a crude supervisor // implementation - superviseFailureAsync(deliveryRequest, ServiceName.ThirdPartyService, ExceptionUtils.getMessage(e)) - .thenRunAsync(() -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - }); - - throw e; + superviseFailureAsync(deliveryRequest, ServiceName.ThirdPartyService, ExceptionUtils.getStackTrace(e)) + .thenAcceptAsync(result -> { + Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); + Log.debug(result); + }); } return thirdPartyResult; @@ -156,19 +164,18 @@ private static String invokeDroneSchedulerServiceAsync(Delivery deliveryRequest, } catch (Exception e) { // Assume failure of service here - a crude supervisor // implementation - superviseFailureAsync(deliveryRequest, ServiceName.DroneSchedulerService, ExceptionUtils.getMessage(e)) - .thenRunAsync(() -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - }); - - throw e; + superviseFailureAsync(deliveryRequest, ServiceName.DroneSchedulerService, ExceptionUtils.getStackTrace(e)) + .thenAcceptAsync(result -> { + Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); + Log.debug(result); + }); } return droneScheduleResult; } public static PackageGen invokePackageServiceAsync(Delivery deliveryRequest, Map properties) { - PackageGen packageResult = new PackageGen(); + PackageGen packageResult = null; try { PackageInfo packageInfo = deliveryRequest.getPackageInfo(); PackageServiceCallerImpl backendService = (PackageServiceCallerImpl) backendServicesMap.get(ServiceName.PackageService); @@ -177,11 +184,11 @@ public static PackageGen invokePackageServiceAsync(Delivery deliveryRequest, Map } catch (Exception e) { // Assume failure of service here - a crude supervisor // implementation - superviseFailureAsync(deliveryRequest, ServiceName.PackageService, ExceptionUtils.getMessage(e)) - .thenRunAsync(() -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - }); - throw e; + superviseFailureAsync(deliveryRequest, ServiceName.PackageService, ExceptionUtils.getStackTrace(e)) + .thenAcceptAsync(result -> { + Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); + Log.debug(result); + }); } return packageResult; @@ -197,12 +204,11 @@ private static DeliverySchedule invokeDeliverySchedulerServiceAsync(Delivery del } catch (Exception e) { // Assume failure of service here - a crude supervisor // implementation - superviseFailureAsync(deliveryRequest, ServiceName.DeliveryService, ExceptionUtils.getMessage(e)) - .thenRunAsync(() -> { + superviseFailureAsync(deliveryRequest, ServiceName.DeliveryService, ExceptionUtils.getStackTrace(e)) + .thenAcceptAsync(result -> { Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); + Log.debug(result); }); - - throw e; } return deliveryResult; @@ -225,23 +231,24 @@ private static void appendServiceMeshHeaders(ServiceCallerImpl service, Map superviseFailureAsync(Delivery deliveryRequest, ServiceName serviceName, String errorMessage) { - CloudQueueClient queueClient = StorageQueueClientFactory.get(); + private static CompletableFuture superviseFailureAsync(Delivery deliveryRequest, ServiceName serviceName, String errorMessage) { + CloudQueueClient queueClient = StorageQueueClientFactory.INSTANCE.get(SchedulerSettings.StorageQueueConnectionString); + String endResult = "Failed delivery request added to compensation queue!"; try { - CloudQueue queueReference = queueClient.getQueueReference(SchedulerSettings.storageQueueName); + CloudQueue queueReference = queueClient.getQueueReference(SchedulerSettings.StorageQueueName); queueReference.createIfNotExists(); - - String requestInJson = deserializer.toJson(deliveryRequest, Delivery.class); + + // Store the metadata so that delivery could be retried from failure state if needed + String requestInJson = gsonSerializer.toJson(new RetryableDelivery(deliveryRequest, serviceName, errorMessage)); byte[] requestInJsonBytes = requestInJson.getBytes(StandardCharsets.UTF_8); CloudQueueMessage message = new CloudQueueMessage(requestInJsonBytes); queueReference.addMessage(message); } catch (URISyntaxException | StorageException e) { - e.printStackTrace(); - } finally { - return null; + endResult = ExceptionUtils.getStackTrace(e); } + return CompletableFuture.completedFuture(endResult); } } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java index c6371aa8..12bf603c 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java @@ -17,6 +17,7 @@ public class SchedulerSettings { public static String HostNameValue; //StorageQueue - public static String storageQueueConnectionString; - public static String storageQueueName; + public static String StorageQueueConnectionString; + public static String StorageQueueName; + public static int CheckpointTimeInMinutes; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java deleted file mode 100644 index 83b76729..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueClientFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.StorageQueue; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.queue.CloudQueueClient; - -import java.net.URISyntaxException; -import java.security.InvalidKeyException; - -public class StorageQueueClientFactory { - - private static CloudQueueClient queueClient; - - static { - - try { - CloudStorageAccount cloudStorageAccount = CloudStorageAccount.parse(SchedulerSettings.storageQueueConnectionString); - queueClient = cloudStorageAccount.createCloudQueueClient(); - } catch (URISyntaxException | InvalidKeyException e) { - e.printStackTrace(); - } - } - - public static CloudQueueClient get() { - - return queueClient; - - } - - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java deleted file mode 100644 index 369eb0a9..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/StorageQueue/StorageQueueTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.StorageQueue; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.microsoft.azure.storage.queue.CloudQueue; -import com.microsoft.azure.storage.queue.CloudQueueClient; -import com.microsoft.azure.storage.queue.CloudQueueMessage; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - - -public class StorageQueueTest { - - private CloudQueueClient storageClient ; - - private String someStringContent; - -// This test will work only if environment variables are set -// STORAGE_QUEUE_CONNECTION_STRING and STORAGE_QUEUE_NAME - - @Before - public void setup() { - - SchedulerSettings.storageQueueConnectionString = System.getenv("STORAGE_QUEUE_CONNECTION_STRING"); - SchedulerSettings.storageQueueName = System.getenv("STORAGE_QUEUE_NAME"); - storageClient = StorageQueueClientFactory.get(); - someStringContent = "This is test message to queue"; - } - - @Test - public void it_should_add_message_to_queue() { - - //Arrange - CloudQueueMessage queueMessage = new CloudQueueMessage(someStringContent); - - CloudQueue queueReference = null; - try { - - queueReference = storageClient.getQueueReference(SchedulerSettings.storageQueueName); - queueReference.createIfNotExists(); - - - //Act - queueReference.addMessage(queueMessage); - - CloudQueueMessage peekMessage = queueReference.peekMessage(); - - - //Assert - Assert.assertTrue(peekMessage.getMessageContentAsString().equalsIgnoreCase(someStringContent)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java new file mode 100644 index 00000000..5d23a476 --- /dev/null +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java @@ -0,0 +1,45 @@ +package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation; + +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.ServiceName; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; + +public class RetryableDelivery { + private Delivery deliveryRequest; + private ServiceName serviceName; + private String errorMessage; + + /** + * @param deliveryRequest + * @param serviceName + * @param errorMessage + */ + public RetryableDelivery(Delivery deliveryRequest, ServiceName serviceName, String errorMessage) { + this.deliveryRequest = deliveryRequest; + this.serviceName = serviceName; + this.errorMessage = errorMessage; + } + + public Delivery getDeliveryRequest() { + return deliveryRequest; + } + + public void setDeliveryRequest(Delivery deliveryRequest) { + this.deliveryRequest = deliveryRequest; + } + + public ServiceName getServiceName() { + return serviceName; + } + + public void setServiceName(ServiceName serviceName) { + this.serviceName = serviceName; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java new file mode 100644 index 00000000..d75a8e8e --- /dev/null +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java @@ -0,0 +1,27 @@ +package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation; + +import java.net.URISyntaxException; +import java.security.InvalidKeyException; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.queue.CloudQueueClient; + +public enum StorageQueueClientFactory { + INSTANCE; + private static final Logger Log = LogManager.getLogger(StorageQueueClientFactory.class); + + public CloudQueueClient get(String connectionString) { + CloudQueueClient queueClient = null; + try { + CloudStorageAccount cloudStorageAccount = CloudStorageAccount.parse(connectionString); + queueClient = cloudStorageAccount.createCloudQueueClient(); + } catch (URISyntaxException | InvalidKeyException e) { + Log.error("throwable: {}", ExceptionUtils.getStackTrace(e).toString()); + } + return queueClient; + } +} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java new file mode 100644 index 00000000..fe40e680 --- /dev/null +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java @@ -0,0 +1,55 @@ +package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.tests; + +import java.net.URISyntaxException; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; + +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.StorageQueueClientFactory; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.queue.CloudQueue; +import com.microsoft.azure.storage.queue.CloudQueueClient; +import com.microsoft.azure.storage.queue.CloudQueueMessage; + +public class StorageQueueTest { + @Rule + public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); + + private static final String envStorageConnectionString = "STORAGE_QUEUE_CONNECTION_STRING"; + private static final String envStorageQueueName = "STORAGE_QUEUE_NAME"; + + @Before + public void setup() { + environmentVariables.set(envStorageConnectionString, ""); + environmentVariables.set(envStorageQueueName, ""); + } + + /* + * Ignoring for Maven build since storage connection and queue name needs to be set to run this test + */ + @Ignore + @Test + public void can_add_message_to_queue() throws URISyntaxException, StorageException { + // Arrange + String connStr = System.getenv(envStorageConnectionString); + CloudQueueClient storageClient = StorageQueueClientFactory.INSTANCE.get(connStr); + String someStringContent = "This is test message to queue"; + CloudQueueMessage queueMessage = new CloudQueueMessage(someStringContent); + + CloudQueue queueReference = storageClient.getQueueReference(System.getenv(envStorageQueueName)); + queueReference.createIfNotExists(); + + // Act + queueReference.addMessage(queueMessage); + + CloudQueueMessage peekMessage = queueReference.peekMessage(); + + // Assert + Assert.assertTrue(peekMessage.getMessageContentAsString().equalsIgnoreCase(someStringContent)); + } + +} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java index 83eb903c..d91cc7a7 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java @@ -9,7 +9,6 @@ public class DeliverySchedule { private Boolean expedited; private ConfirmationType confirmationRequired; private String droneId; -// private String packageId; public Location getPickup() { return pickup; @@ -43,14 +42,6 @@ public void setDroneId(String droneId) { this.droneId = droneId; } -// public String getPackageId() { -// return packageId; -// } -// -// public void setPackageId(String packageId) { -// this.packageId = packageId; -// } - public String getDeadline() { return this.deadline; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java index 51197916..208bf11a 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java @@ -5,6 +5,20 @@ public class Location { private double latitude; private double longitude; + public Location(){ + super(); + this.altitude = 0.0; + this.latitude = 0.0; + this.longitude = 0.0; + } + + public Location(double altitude, double latitude, double longitude) { + super(); + this.altitude = altitude; + this.latitude = latitude; + this.longitude = longitude; + } + public double getAltitude() { return altitude; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java index a6568b3d..4cfec84c 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java @@ -15,6 +15,7 @@ public final class AccountServiceCallerImpl extends ServiceCallerImpl { private Boolean isAccountActive = false; + private String exceptionMsg = null; // Calls the super constructor and sets the HTTP context public AccountServiceCallerImpl() { @@ -49,12 +50,14 @@ public Boolean isAccountActiveAsync(String accountId, String uri) { throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); } }).exceptionally(e -> { - throw new BackendServiceCallFailedException(ExceptionUtils.getStackTrace(e)); + exceptionMsg = ExceptionUtils.getStackTrace(e); + return null; }); - future = null; - cfuture = null; - + if(isAccountActive==null){ + throw new BackendServiceCallFailedException(exceptionMsg); + } + return isAccountActive; } } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java index c63820c4..288121de 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java @@ -24,6 +24,7 @@ public class DeliveryServiceCallerImpl extends ServiceCallerImpl { private DeliverySchedule deliverySchedule = null; + private String exceptionMsg = null; // Calls the super constructor and sets the HTTP context public DeliveryServiceCallerImpl() { @@ -76,13 +77,13 @@ public DeliverySchedule scheduleDeliveryAsync(Delivery deliveryRequest, String d throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); } }).exceptionally(e -> { - throw new BackendServiceCallFailedException(ExceptionUtils.getStackTrace(e)); + exceptionMsg = ExceptionUtils.getStackTrace(e); + return null; }); - future = null; - cfuture = null; - deliveryRequest = null; - schedule = null; + if(deliverySchedule==null){ + throw new BackendServiceCallFailedException(exceptionMsg); + } return deliverySchedule; } @@ -95,7 +96,6 @@ private DeliverySchedule createDeliverySchedule(Delivery deliveryRequest, String scheduleDelivery.setOwner(account); scheduleDelivery.setPickup(LocationRandomizer.getRandomLocation()); scheduleDelivery.setDropoff(LocationRandomizer.getRandomLocation()); - //scheduleDelivery.setPackageId(deliveryRequest.getPackageInfo().getPackageId()); scheduleDelivery.setDeadline(deliveryRequest.getDeadline()); scheduleDelivery.setExpedited(deliveryRequest.isExpedited()); scheduleDelivery diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java index 27a3a17a..96a0dccd 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java @@ -21,6 +21,8 @@ public class DroneSchedulerServiceCallerImpl extends ServiceCallerImpl { private String droneId = null; + private String exceptionMsg = null; + // Calls the super constructor and sets the HTTP context public DroneSchedulerServiceCallerImpl() { super(); @@ -84,11 +86,13 @@ public String getDroneIdAsync(Delivery deliveryRequest, String uri) { throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); } }).exceptionally(e -> { - throw new BackendServiceCallFailedException(ExceptionUtils.getStackTrace(e)); + exceptionMsg = ExceptionUtils.getStackTrace(e); + return null; }); - future = null; - cfuture = null; + if(droneId==null){ + throw new BackendServiceCallFailedException(exceptionMsg); + } return droneId; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java index 3be7cfed..7b7900a3 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java @@ -26,6 +26,7 @@ public class PackageServiceCallerImpl extends ServiceCallerImpl { private final static JsonParser jsonParser = new JsonParser(); private PackageGen packageGen = null; + private String exceptionMsg = null; // Calls the super constructor and sets the HTTP context public PackageServiceCallerImpl() { @@ -85,12 +86,13 @@ public PackageGen createPackageAsync(PackageInfo packageInfo, String uri) { throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); } }).exceptionally(e -> { - throw new BackendServiceCallFailedException(ExceptionUtils.getStackTrace(e)); + exceptionMsg = ExceptionUtils.getStackTrace(e); + return null; }); - future = null; - cfuture = null; - packageInfo = null; + if(packageGen==null){ + throw new BackendServiceCallFailedException(exceptionMsg); + } return packageGen; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java index e2a667cc..265e38e8 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java @@ -27,18 +27,17 @@ import org.apache.http.HttpResponse; public abstract class ServiceCallerImpl implements ServiceCaller { - private AsyncRestTemplate asyncRestTemplate; private HttpHeaders requestHeaders; - private static final Logger Log = LogManager.getLogger(ServiceCallerImpl.class); + private final static Logger Log = LogManager.getLogger(ServiceCallerImpl.class); + private final static int Second = 1000; + private HttpComponentsAsyncClientHttpRequestFactory clientHttpRequestFactory; public AsyncRestTemplate getAsyncRestTemplate() { + AsyncRestTemplate asyncRestTemplate = new AsyncRestTemplate(this.clientHttpRequestFactory); + asyncRestTemplate.setErrorHandler(new ServiceCallerResponseErrorHandler()); return asyncRestTemplate; } - public void setAsyncRestTemplate(AsyncRestTemplate asyncRest) { - this.asyncRestTemplate = asyncRest; - } - public HttpHeaders getRequestHeaders() { return requestHeaders; } @@ -53,23 +52,69 @@ public ListenableFuture getData(String url, T data) { } public ServiceCallerImpl() { - ConnectingIOReactor ioReactor = null; + // Initialize http headers + initHttpHeaders(); + + // Initialize factory that is shared by all new instances of AsyncRestTemplate + this.clientHttpRequestFactory = getCustomClientHttpRequestFactory(); + } + + private void initHttpHeaders() { + this.requestHeaders = new HttpHeaders(); + this.requestHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); + this.requestHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); + } + + private HttpComponentsAsyncClientHttpRequestFactory getCustomClientHttpRequestFactory() { + PoolingNHttpClientConnectionManager poolingConnManager = + new PoolingNHttpClientConnectionManager(getIOReactor()); + + HttpHost httpProxy = getHttpProxy(); + + CloseableHttpAsyncClient client = getCustomHttpClient(poolingConnManager, httpProxy); + + // Start idle connection monitor + IdleConnectionMonitorThread staleMonitor = new IdleConnectionMonitorThread(poolingConnManager); + staleMonitor.start(); + + return getCustomClientHttpRequestFactory(client); + } + + private ConnectingIOReactor getIOReactor() { + ConnectingIOReactor ioReactor = null; try { ioReactor = new DefaultConnectingIOReactor(); } catch (IOReactorException e) { Log.error(ExceptionUtils.getStackTrace(e)); } - - PoolingNHttpClientConnectionManager poolingConnManager = - new PoolingNHttpClientConnectionManager(ioReactor); - + return ioReactor; + } + + private HttpHost getHttpProxy() { HttpHost httpProxy = null; if (StringUtils.isNotEmpty(SchedulerSettings.HttpProxyValue)) { String[] address = SchedulerSettings.HttpProxyValue.split("\\s*:\\s*"); httpProxy = new HttpHost(address[0], Integer.parseInt(address[1])); } + return httpProxy; + } + + private HttpComponentsAsyncClientHttpRequestFactory getCustomClientHttpRequestFactory( + CloseableHttpAsyncClient client) { + HttpComponentsAsyncClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsAsyncClientHttpRequestFactory(); + clientHttpRequestFactory.setConnectionRequestTimeout(30*Second); + clientHttpRequestFactory.setConnectTimeout(30*Second); + clientHttpRequestFactory.setBufferRequestBody(false); + clientHttpRequestFactory.setReadTimeout(0); + + clientHttpRequestFactory.setHttpAsyncClient(client); + return clientHttpRequestFactory; + } + + private CloseableHttpAsyncClient getCustomHttpClient(PoolingNHttpClientConnectionManager poolingConnManager, + HttpHost httpProxy) { HttpAsyncClientBuilder builder = HttpAsyncClients.custom().setConnectionManager(poolingConnManager) .setConnectionReuseStrategy(new ConnectionReuseStrategy() { @@ -86,22 +131,7 @@ public boolean keepAlive(HttpResponse response, HttpContext context) { } CloseableHttpAsyncClient client = builder.build(); - - IdleConnectionMonitorThread worker = new IdleConnectionMonitorThread(poolingConnManager); - worker.start(); - - HttpComponentsAsyncClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsAsyncClientHttpRequestFactory(); - clientHttpRequestFactory.setConnectionRequestTimeout(0); - clientHttpRequestFactory.setConnectTimeout(0); - clientHttpRequestFactory.setBufferRequestBody(false); - clientHttpRequestFactory.setReadTimeout(0); - - clientHttpRequestFactory.setHttpAsyncClient(client); - asyncRestTemplate = new AsyncRestTemplate(clientHttpRequestFactory); - this.requestHeaders = new HttpHeaders(); - this.requestHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); - this.requestHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); - asyncRestTemplate.setErrorHandler(new ServiceCallerResponseErrorHandler()); + return client; } @Override diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java index cf785513..48cd7486 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java @@ -20,6 +20,7 @@ public class ThirdPartyServiceCallerImpl extends ServiceCallerImpl { private Boolean isThirdPartyRequired = true; + private String exceptionMsg = null; public ThirdPartyServiceCallerImpl() { super(); @@ -70,11 +71,13 @@ public Boolean isThirdPartyServiceRequiredAsync(String dropOffLocation, String u throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); } }).exceptionally(e -> { - throw new BackendServiceCallFailedException(ExceptionUtils.getStackTrace(e)); + exceptionMsg = ExceptionUtils.getStackTrace(e); + return null; }); - future = null; - cfuture = null; + if(isThirdPartyRequired==null){ + throw new BackendServiceCallFailedException(exceptionMsg); + } return isThirdPartyRequired; } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java new file mode 100644 index 00000000..ef4aed2a --- /dev/null +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java @@ -0,0 +1,157 @@ +package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.put; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.AsyncRestTemplate; + +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DroneDelivery; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; +import com.github.tomakehurst.wiremock.junit.WireMockClassRule; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.opentable.extension.BodyTransformer; + +import wiremock.org.eclipse.jetty.http.HttpStatus; + +public class InvokeMockBackendServicesTest { + @ClassRule + public static WireMockClassRule wiremock = new WireMockClassRule(wireMockConfig().dynamicPort().dynamicHttpsPort().extensions(new BodyTransformer())); + + public final String baseUri = "http://localhost:" + wiremock.port(); + + private final AsyncRestTemplate restTemplate = new AsyncRestTemplate(); + + private final Gson deserializer = new GsonBuilder().create(); + + private static final String DeliveryId = "some-random-delivery-id"; + + private static final String Tag = "some-random-tag"; + + private static final String DroneId = UUID.randomUUID().toString(); + + private final PackageInfo packInfo = ModelsUtils.getPackageInfo(Tag); + + private final PackageGen packGen = ModelsUtils.createPackageGen(Tag); + + private final Delivery delivery = ModelsUtils.createDeliveryRequest(); + + private final DroneDelivery droneDelivery = ModelsUtils.createDroneDelivery(delivery); + + private final DeliverySchedule deliverySchedule = ModelsUtils.createDeliverySchedule(delivery, DroneId); + + @Before + public void setUp() throws Exception { + // Account service stub + wiremock.stubFor(get(urlPathMatching("/api/Account/.*")).willReturn( + aResponse().withStatus(HttpStatus.OK_200).withHeader("Content-Type", "text/plain").withBody("true"))); + + // Third party deliveries service stub + wiremock.stubFor(put(urlPathEqualTo("/api/ThirdPartyDeliveries/" + DeliveryId)) + .withRequestBody(equalToJson(deserializer.toJson(ModelsUtils.getRandomLocation(0.0)))) + .willReturn( + aResponse().withStatus(HttpStatus.CREATED_201) + .withBody("false"))); + + // Package service stub + String json = deserializer.toJson(packInfo); + String jsonReturned = deserializer.toJson(packGen); + wiremock.stubFor(put(urlPathEqualTo("/api/packages/" + Tag)) + .withRequestBody(equalToJson(json)) + .willReturn( + aResponse().withStatus(HttpStatus.CREATED_201) + .withHeader("Content-Type", "application/json") + .withBody(jsonReturned))); + + // Drone service stub + wiremock.stubFor(put(urlPathEqualTo("/api/DroneDeliveries/" + droneDelivery.getDeliveryId())) + .withRequestBody(equalToJson(deserializer.toJson(droneDelivery))) + .willReturn(aResponse().withStatus(HttpStatus.CREATED_201).withHeader("Content-Type", "text/plain") + .withBody("AssignedDroneId$(uuid)").withTransformers("body-transformer"))); + + // Delivery service stub + String json0 = deserializer.toJson(deliverySchedule); + wiremock.stubFor(put(urlPathEqualTo("/api/Deliveries/" + delivery.getDeliveryId())) + .withRequestBody(equalToJson(json0)) + .willReturn(aResponse().withStatus(HttpStatus.CREATED_201).withHeader("Content-Type", "application/json") + .withBody(json0))); + + } + + @Test + public void can_retrieve_account_status() throws IOException, InterruptedException, ExecutionException { + String uri = this.baseUri + "/api/Account/some-random-account-id"; + + CompletableFuture> cfuture = toCompletableFuture( + this.restTemplate.getForEntity(uri, String.class)); + String result = ((HttpEntity)cfuture.get()).getBody(); + assertEquals(result, "true"); + } + + @Test + public void can_retrieve_thirdparty_status() throws IOException, InterruptedException, ExecutionException { + String uri = this.baseUri + "/api/ThirdPartyDeliveries/" + DeliveryId; + String json = deserializer.toJson(ModelsUtils.getRandomLocation(0.0)); + HttpEntity entity = new HttpEntity(json); + CompletableFuture> cfuture = toCompletableFuture( + this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); + String result = ((HttpEntity)cfuture.get()).getBody(); + assertEquals(result, "false"); + } + + @Test + public void can_retrieve_package() throws IOException, InterruptedException, ExecutionException { + String uri = this.baseUri + "/api/packages/" + Tag; + HttpEntity entity = new HttpEntity(deserializer.toJson(packInfo)); + CompletableFuture> cfuture = toCompletableFuture( + this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); + HttpEntity resultEntity = cfuture.get(); + PackageGen result = (PackageGen)ModelsUtils.getPackageGen(resultEntity.getBody()); + assertEquals(result.getTag(), Tag); + } + + @Test + public void can_assign_drone() throws InterruptedException, ExecutionException{ + String uri = this.baseUri + "api/DroneDeliveries/" + droneDelivery.getDeliveryId(); + HttpEntity entity = new HttpEntity(deserializer.toJson(droneDelivery)); + CompletableFuture> cfuture = toCompletableFuture( + this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); + String result = ((HttpEntity)cfuture.get()).getBody(); + assertTrue(result.startsWith("AssignedDroneId")); + } + + @Test + public void can_schedule_delivery() throws IOException, InterruptedException, ExecutionException { + String uri = this.baseUri + "/api/Deliveries/" + deliverySchedule.getId(); + HttpEntity entity = new HttpEntity(deserializer.toJson(deliverySchedule)); + CompletableFuture> cfuture = toCompletableFuture( + this.restTemplate.exchange(uri, HttpMethod.PUT, entity, DeliverySchedule.class)); + HttpEntity resultEntity = cfuture.get(); + DeliverySchedule result = (DeliverySchedule)resultEntity.getBody(); + assertEquals(result.getDroneId(), DroneId); + } +} + diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java similarity index 60% rename from src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java rename to src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java index 5816b7da..9c2df27d 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ValidateSerializationOptions.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java @@ -1,34 +1,24 @@ package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.util.Date; -import java.util.Random; -import java.util.UUID; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.springframework.util.Assert; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import org.junit.contrib.java.lang.system.EnvironmentVariables; -public class ValidateSerializationOptions { +public class ModelsAndUtilsTest { private Gson serializer; private ObjectMapper objMapper; - private final String Locations[] = { "Austin", "Seattle", "Berkley", "Oregon", "Florida", "Blaine", "Renton" }; - private final static Random random = new Random(); - - private final ContainerSize[] containers = ContainerSize.values(); - private final ConfirmationRequired[] confirmations = ConfirmationRequired.values(); private static final String envHostNameString = "HOST_POD_NAME"; private static final String envHttpProxyString = "http_proxy"; @@ -43,36 +33,36 @@ public void setUp() throws Exception { } @Test - public void ValidateSerializationWithGson() { - Delivery delivery = createDelivery(); + public void can_serialize_delivery_with_gson() { + Delivery delivery = ModelsUtils.createDeliveryRequest(); String jsonDelivery = serializer.toJson(delivery); Delivery rehydratedDelivery = serializer.fromJson(jsonDelivery, Delivery.class); Assert.isInstanceOf(Delivery.class, rehydratedDelivery, "Delivery serialized correctly"); } @Test - public void ValidateSerializationWithJackson() throws IOException { - Delivery delivery = createDelivery(); + public void can_serialize_delivery_with_jackson() throws IOException { + Delivery delivery = ModelsUtils.createDeliveryRequest(); String jsonDelivery = objMapper.writeValueAsString(delivery); Delivery rehydratedDelivery = objMapper.readValue(jsonDelivery, Delivery.class); Assert.isInstanceOf(Delivery.class, rehydratedDelivery, "Delivery serialized correctly"); } @Test - public void ValidateParseDeliveryWithSpecificPayload() { + public void can_serialize_specific_workload_correctly() { String jsonPayload = "{\"OwnerId\":\"f0a8680e-574a-4d30-89a6-9c8d8ca2302d\",\"DeliveryId\":\"37bbe418-2254-4ebf-a848-bd9929f10b75\",\"Packages\":[{\"PackageId\":\"5114e11d-0e23-47d4-a36c-2a297c83037d\",\"Size\":1}],\"deadline\":\"LineOfDeadlyZombiatedPeople\",\"confirmationRequired\":3,\"expedited\":false,\"DropoffLocation\":\"Seattle\",\"PickupTime\":\"Jul 28, 2017 04:06:24 PM\",\"PickupLocation\":\"Florida\"}"; Delivery delivery = serializer.fromJson(jsonPayload, Delivery.class); Assert.isInstanceOf(Delivery.class, delivery); } @Test - public void validateSetEnvironmentVariable() { + public void can_set_env_vars_correctly() { environmentVariables.set(envHostNameString, "mypod-1"); assertEquals("mypod-1", System.getenv(envHostNameString)); } @Test - public void validateSetEnvironmentVariableHttpProxy(){ + public void can_parse_http_proxy_correctly(){ environmentVariables.set(envHttpProxyString, "k8s-agent-f1cfb2cf-0:4140"); String[] address = System.getenv(envHttpProxyString).split("\\s*:\\s*"); assertEquals(address[0], "k8s-agent-f1cfb2cf-0"); @@ -80,7 +70,7 @@ public void validateSetEnvironmentVariableHttpProxy(){ } @Test - public void validateParseEnvironmentVariable() { + public void can_parse_stateful_set_config_correctly() { environmentVariables.set(envHostNameString, "mypod-1"); String hostName = System.getenv(envHostNameString); int partitionId = -1; @@ -93,22 +83,4 @@ public void validateParseEnvironmentVariable() { assertEquals(1, partitionId); } - - private Delivery createDelivery() { - PackageInfo pack = new PackageInfo(); - pack.setSize(containers[random.nextInt(containers.length)]); - pack.setPackageId(UUID.randomUUID().toString()); - - Delivery delivery = new Delivery(); - - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(new Date()); - delivery.setDropOffLocation(Locations[random.nextInt(Locations.length)]); - delivery.setPickupLocation(Locations[random.nextInt(Locations.length)]); - delivery.setConfirmationRequired(confirmations[random.nextInt(confirmations.length)]); - - delivery.setPackageInfo(pack); - - return delivery; - } } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java index b487a41a..331e8a57 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java @@ -3,55 +3,45 @@ import static org.junit.Assert.assertEquals; import java.util.Collections; -import java.util.Date; -import java.util.Random; -import java.util.UUID; import java.util.concurrent.ExecutionException; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; - import org.springframework.util.concurrent.ListenableFuture; import org.springframework.web.client.AsyncRestTemplate; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; public class PostDeliveryRequests { private AsyncRestTemplate asyncRest; private HttpHeaders requestHeaders; - private static String postUri = "http://23.101.149.1:8080/api/deliveryrequests"; - private final String Locations[] = {"Austin", "Seattle", "Berkley", "Oregon", "Florida", "Blaine", "Renton"}; - private static Random random = new Random(); - - private final ContainerSize[] containers = ContainerSize.values(); - private final ConfirmationRequired[] confirmations = ConfirmationRequired.values(); + // Set below to match with your deployment + private static String postUri = "http:ingestion/api/deliveryrequests"; - private final double leftLimit = 10D; - private final double rightLimit = 100D; - @Before public void setUp() throws Exception { asyncRest = new AsyncRestTemplate(); requestHeaders = new HttpHeaders(); requestHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); requestHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); - //addServiceMeshHeaders(); } + /* + * Ignoring for Maven build since Uri needs to be set to run this test + */ + @Ignore @Test - public void PostDeliveryRequestsToWeb() throws InterruptedException, ExecutionException { - for(int i=0;i<5000;i++){ - Delivery delivery = createDeliveryRequest(); + public void can_post_delivery_request() throws InterruptedException, ExecutionException { + Delivery delivery = ModelsUtils.createDeliveryRequest(); System.out.println(delivery.toString()); HttpEntity entity = new HttpEntity(delivery, requestHeaders); @@ -60,31 +50,6 @@ public void PostDeliveryRequestsToWeb() throws InterruptedException, ExecutionEx Delivery.class, delivery); assertEquals(HttpStatus.ACCEPTED, response.get().getStatusCode()); - } - } - - private Delivery createDeliveryRequest() { - PackageInfo pack = new PackageInfo(); - - pack.setPackageId(UUID.randomUUID().toString()); - pack.setSize(containers[random.nextInt(containers.length)]); - pack.setTag(UUID.randomUUID().toString()); - pack.setWeight(leftLimit + new Random().nextDouble() * (rightLimit - leftLimit)); - - Delivery delivery = new Delivery(); - delivery.setDeliveryId(UUID.randomUUID().toString()); - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(new Date()); - delivery.setDropOffLocation(Locations[random.nextInt(Locations.length)]); - delivery.setPickupLocation(Locations[random.nextInt(Locations.length)]); - delivery.setConfirmationRequired(confirmations[random.nextInt(confirmations.length)]); - delivery.setDeadline("LineOfDeadPeople"); - delivery.setExpedited(true); - - delivery.setPackageInfo(pack); - - return delivery; } - } diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java deleted file mode 100644 index 53da2fde..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/TestBackendServices.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.DeliveryRequestEventProcessor; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.Location; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.LocationRandomizer; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.AsyncRestTemplate; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; -import static org.junit.Assert.assertEquals; - -public class TestBackendServices { - @ClassRule - public static WireMockClassRule wiremock = new WireMockClassRule(wireMockConfig().dynamicPort().dynamicHttpsPort()); - - public final String baseUri = "http://localhost:" + wiremock.port(); - - private Random random = new Random(); - - private final String Locations[] = { "Austin", "Seattle", "Berkley", "Oregon", "Florida", "Blaine", "Renton" }; - - private final ContainerSize[] containers = ContainerSize.values(); - private final ConfirmationRequired[] confirmations = ConfirmationRequired.values(); - - private AsyncRestTemplate restTemplate = new AsyncRestTemplate(); - - @Before - public void setUp() throws Exception { - - } - - - private void setupStubAccountService() { - // Account service stub - wiremock.stubFor(get(urlPathMatching("/api/Account/.*")).willReturn( - aResponse().withStatus(200).withHeader("Content-Type", "text/plain").withBody("true"))); - } - - private void setupStubThirdPartyService(){ - wiremock.stubFor(put(urlPathMatching("/api/ThirdPartyDeliveries/.*")) - .withHeader("Content-Type", equalTo("application/json")) - .withRequestBody(matchingJsonPath("{ \"altitude\": 0, \"latitude\": 0, \"longitude\": 0 }")) - .willReturn( - aResponse().withStatus(200) - .withHeader("Content-Type", "application/json; charset=utf-8") - .withBody("false"))); - - } - - @Test - public void CanRetrieveAccountStatusFromMockServiceAsync() throws IOException, InterruptedException, ExecutionException { - setupStubAccountService(); - String uri = this.baseUri + "/api/Account/some-random-account-id"; - - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.getForEntity(uri, String.class)); - String result = ((HttpEntity)cfuture.get()).getBody(); - assertEquals(result, "true"); - } - - @Test - public void CanRetrieveThirdPartyStatusFromMockServiceAsync() throws IOException, InterruptedException, ExecutionException { - setupStubThirdPartyService(); - String uri = this.baseUri + "/api/ThirdPartyDeliveries/some-random-delivery-id"; - - HttpEntity entity = new HttpEntity(LocationRandomizer.getRandomLocation()); - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); - String result = ((HttpEntity)cfuture.get()).getBody(); - assertEquals(result, "false"); - } - - @Test - public void CanRetrieveAccountStatusFromAccountServiceAsync() throws InterruptedException, ExecutionException { - - Map properties = new HashMap(); - SchedulerSettings.AccountServiceUri = "http://51.179.155.83/api/Account"; - DeliveryRequestEventProcessor.invokeAccountServiceAsync(this.createDeliveryRequest(), properties); - -// CompletableFuture cfuture = toCompletableFuture( -// accountService.getData("http://51.179.155.83/api/Account/", accountId)); -// String result = ((HttpEntity)cfuture.get()).getBody(); -// assertEquals(result, "true"); - } - - -// @Test -// public void CanRetrieveThirdPartyConsentFromServiceAsync() throws InterruptedException, ExecutionException { -// // TODO: Revisit this implementation since response body parameters are -// // split though passing one works -// ThirdPartyServiceCallerImpl thirdpartySvc = new ThirdPartyServiceCallerImpl(); -// DeferredResult result = thirdpartySvc.isThirdPartyServiceRequiredAsync("Idaho", SchedulerSettings.ThirdPartyServiceUri); -// result.onCompletion(() -> { -// assertEquals(false, result.getResult()); -// }); -// } - -// @Test -// public void CanRetrieveDroneIdFromDroneDeliveryServiceAsync() throws InterruptedException, ExecutionException { -// DroneSchedulerServiceCallerImpl droneSvc = new DroneSchedulerServiceCallerImpl(); -// Delivery deliveryRequest = this.createDeliveryRequest(); -// DeferredResult droneId = droneSvc.getDroneIdAsync(deliveryRequest, SchedulerSettings.DroneSchedulerServiceUri); -// -// droneId.onCompletion(() -> { -// assertTrue(DroneServiceReturnValuePrefix, -// ((String) droneId.getResult()).startsWith(DroneServiceReturnValuePrefix)); -// }); -// } - -// @Test -// public void CanScheduleDeliveryWithDeliveryServiceAsync() throws InterruptedException, ExecutionException { -// String droneId = UUID.randomUUID().toString(); -// DeliveryServiceCallerImpl deliverySvc = new DeliveryServiceCallerImpl(); -// DeferredResult deliveryScheduled = deliverySvc -// .scheduleDeliveryAsync(this.createDeliveryRequest(), droneId, SchedulerSettings.DeliveryServiceUri); -// -// deliveryScheduled.onCompletion(() -> { -// DeliverySchedule delivery = (DeliverySchedule) deliveryScheduled.getResult(); -// assertEquals(droneId, delivery.getDroneId()); -// }); -// } - -// @Test -// public void CanRetrievePackagesInfoFromPackageServiceAsync() throws InterruptedException, ExecutionException { -// PackageInfo pack = new PackageInfo(); -// -// pack.setPackageId(UUID.randomUUID().toString()); -// pack.setSize(containers[random.nextInt(containers.length)]); -// -// PackageServiceCallerImpl packageSvc = new PackageServiceCallerImpl(); -// DeferredResult defResult = packageSvc.createPackageAsync(pack, -// SchedulerSettings.PackageServiceUri); -// -// defResult.onCompletion(() -> { -// PackageGen packGen = (PackageGen) defResult.getResult(); -// assertEquals(pack.getPackageId(), packGen.getTag()); -// }); -// } - - private Delivery createDeliveryRequest() { - PackageInfo pack = new PackageInfo(); - - pack.setPackageId(UUID.randomUUID().toString()); - pack.setSize(containers[random.nextInt(containers.length)]); - - Delivery delivery = new Delivery(); - - delivery.setDeliveryId(UUID.randomUUID().toString()); - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(new Date()); - delivery.setDropOffLocation(Locations[random.nextInt(Locations.length)]); - delivery.setPickupLocation(Locations[random.nextInt(Locations.length)]); - delivery.setConfirmationRequired(confirmations[random.nextInt(confirmations.length)]); - delivery.setDeadline("LineOfDeadPeople"); - delivery.setExpedited(true); - - delivery.setPackageInfo(pack); - - return delivery; - } -} - diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java new file mode 100644 index 00000000..39aab754 --- /dev/null +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java @@ -0,0 +1,140 @@ +package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers; + +import java.util.Date; +import java.util.Random; +import java.util.UUID; + +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DroneDelivery; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.Location; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.UserAccount; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; +import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.ModelsConverter; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public class ModelsUtils { + private final static Random random = new Random(); + + private final static JsonParser jsonParser = new JsonParser(); + + private final static ContainerSize[] containers = ContainerSize.values(); + + private final static String Locations[] = { "Austin", "Seattle", "Berkley", "Oregon", "Florida", "Blaine", "Renton" }; + + private final static ConfirmationRequired[] confirmations = ConfirmationRequired.values(); + + private final static double leftLimit = 10D; + + private final static double rightLimit = 100D; + + + public static Location getRandomLocation(double... arguments) { + Location location = null; + if (arguments != null && arguments.length >= 1) { + double x = arguments[0]; + location = new Location(x, x, x); + } else { + location = new Location(); + location.setAltitude(random.nextDouble()); + location.setLatitude(random.nextDouble()); + location.setLongitude(random.nextDouble()); + } + + return location; + } + + public static PackageGen getPackageGen(String jsonStr) { + PackageGen pack = new PackageGen(); + JsonElement jsonElem = jsonParser.parse(jsonStr); + JsonObject jObject = jsonElem.getAsJsonObject(); + pack.setId(jObject.get("id").getAsString()); + pack.setSize(ContainerSize.valueOf(jObject.get("size").getAsString())); + pack.setTag(jObject.get("tag").getAsString()); + + JsonElement weight = jObject.get("weight"); + pack.setWeight(weight.isJsonNull() ? 0.0 : weight.getAsDouble()); + + return pack; + } + + public static PackageInfo getPackageInfo(String randomTag){ + PackageInfo packInfo = new PackageInfo(); + + packInfo.setPackageId(UUID.randomUUID().toString()); + packInfo.setSize(containers[random.nextInt(containers.length)]); + packInfo.setTag(randomTag); + packInfo.setWeight(1.0); + + return packInfo; + } + + public static PackageGen createPackageGen(String tag) + { + PackageGen pack = new PackageGen(); + pack.setId(UUID.randomUUID().toString()); + pack.setSize(containers[random.nextInt(containers.length)]); + pack.setTag(tag); + pack.setWeight(leftLimit + new Random().nextDouble() * (rightLimit - leftLimit)); + return pack; + } + + public static Delivery createDeliveryRequest() { + PackageInfo pack = new PackageInfo(); + + pack.setPackageId(UUID.randomUUID().toString()); + pack.setSize(containers[random.nextInt(containers.length)]); + pack.setWeight(leftLimit + new Random().nextDouble() * (rightLimit - leftLimit)); + + Delivery delivery = new Delivery(); + + delivery.setDeliveryId(UUID.randomUUID().toString()); + delivery.setOwnerId(UUID.randomUUID().toString()); + delivery.setPickupTime(new Date()); + delivery.setDropOffLocation(Locations[random.nextInt(Locations.length)]); + delivery.setPickupLocation(Locations[random.nextInt(Locations.length)]); + delivery.setConfirmationRequired(confirmations[random.nextInt(confirmations.length)]); + delivery.setDeadline("ByTheEndOfDay"); + delivery.setExpedited(true); + + delivery.setPackageInfo(pack); + + return delivery; + } + + public static DroneDelivery createDroneDelivery(Delivery deliveryRequest) { + DroneDelivery delivery = new DroneDelivery(); + delivery.setDeliveryId(deliveryRequest.getDeliveryId()); + + delivery.setDropoff(ModelsUtils.getRandomLocation()); + delivery.setPickup(ModelsUtils.getRandomLocation()); + + delivery.setExpedited(delivery.getExpedited()); + delivery.setPackageDetail(ModelsConverter.getPackageDetail(deliveryRequest.getPackageInfo())); + + return delivery; + } + + public static DeliverySchedule createDeliverySchedule(Delivery deliveryRequest, String droneId) { + UserAccount account = new UserAccount(UUID.randomUUID().toString(), deliveryRequest.getOwnerId()); + + DeliverySchedule scheduleDelivery = new DeliverySchedule(); + scheduleDelivery.setId(deliveryRequest.getDeliveryId()); + scheduleDelivery.setOwner(account); + scheduleDelivery.setPickup(ModelsUtils.getRandomLocation()); + scheduleDelivery.setDropoff(ModelsUtils.getRandomLocation()); + scheduleDelivery.setDeadline(deliveryRequest.getDeadline()); + scheduleDelivery.setExpedited(deliveryRequest.isExpedited()); + scheduleDelivery + .setConfirmationRequired(ModelsConverter.getConfirmationType(deliveryRequest.getConfirmationRequired())); + scheduleDelivery.setDroneId(droneId); + + return scheduleDelivery; + } + +} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java index f349b372..af2b2d32 100644 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java +++ b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java @@ -29,6 +29,7 @@ public void run() { } } catch (InterruptedException ex) { // terminate + shutdown(); } } From 8c8aa7819cd0534df0fd06e49722c7e833cae3ab Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Thu, 16 Aug 2018 05:49:01 -0700 Subject: [PATCH 032/246] Streamline deployment steps (#105) * Streamline deployment steps - Use `-o tsv` to avoid needing to strip quotes - Pipe sed commands to a new file, to the operation is non-destructive of the source files (idempotent) - Use CLI to create event hub and get access key - Use 8 partitions and 8 scheduler replicas, to reduce the footprint for F5 experience - Use CLI to get Storage Account access key and connection string - Clarify which steps are optional for a test deployment * Use 4 event hub partitions instead of 8 (or 32) * Update partition property to 8 --- deployment.md | 209 ++++++++++++++++++++++++++------------------- k8s/scheduler.yaml | 4 +- 2 files changed, 123 insertions(+), 90 deletions(-) diff --git a/deployment.md b/deployment.md index 27bac8c3..d9cd39cf 100644 --- a/deployment.md +++ b/deployment.md @@ -1,7 +1,5 @@ # Deploying the Reference Implementation - - ## Prerequisites - Azure suscription @@ -28,6 +26,8 @@ export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" + +export K8S=./microservices-reference-implementation/k8s ``` Provision a Kubernetes cluster in ACS @@ -69,10 +69,7 @@ az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic az acr login --name $ACR_NAME # Get the ACR login server name -export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer") - -# Strip quotes -export ACR_SERVER=("${ACR_SERVER[@]//\"/}") +export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer" -o tsv) ``` ## Deploy the Delivery service @@ -125,12 +122,12 @@ Create Kubernetes secrets ```bash export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] -export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ -export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) +export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) +export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ - --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ - --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ + --from-literal=CosmosDB_Key=${COSMOSDB_KEY} \ + --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT} \ --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ --from-literal=EH_ConnectionString= ``` @@ -138,16 +135,17 @@ kubectl --namespace shipping create --save-config=true secret generic delivery-s Deploy the Delivery service: ```bash -# Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" ./microservices-reference-implementation/k8s/delivery.yaml - -## Update config values in the deployment YAML -sed -i "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" +# Update the image tag and config values in the deployment YAML +sed "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" $K8S/delivery.yaml | \ + sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ + sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ + sed "s/value: \"EH_EntityPath\"/value:/g" > $K8S/delivery-0.yaml # Deploy the service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml +kubectl --namespace shipping apply -f $K8S/delivery-0.yaml + +# Verify the pod is created +kubectl get pods -n shipping ``` ## Deploy the Package service @@ -179,14 +177,17 @@ Deploy the Package service ```bash # Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml +sed "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml # Create secret -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") -kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv) +kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION # Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/package.yml +kubectl --namespace shipping apply -f $K8S/package-0.yml + +# Verify the pod is created +kubectl get pods -n shipping ``` ## Deploy the Ingestion service @@ -197,16 +198,33 @@ export INGESTION_EH_NS=[INGESTION_EVENT_HUB_NAMESPACE_HERE] export INGESTION_EH_NAME=[INGESTION_EVENT_HUB_NAME_HERE] export INGESTION_EH_CONSUMERGROUP_NAME=[INGESTION_EVENT_HUB_CONSUMERGROUP_NAME_HERE] -wget https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/201-event-hubs-create-event-hub-and-consumer-group/azuredeploy.json && \ -sed -i 's#"partitionCount": "4"#"partitionCount": "32"#g' azuredeploy.json && \ -az group deployment create -g $RESOURCE_GROUP --template-file azuredeploy.json --parameters \ -'{ \ - "namespaceName": {"value": "'${INGESTION_EH_NS}'"}, \ - "eventHubName": {"value": "'${INGESTION_EH_NAME}'"}, \ - "consumerGroupName": {"value": "'${INGESTION_EH_CONSUMERGROUP_NAME}'"} \ -}' +# Create an Event Hubs namespace +az eventhubs namespace create --name $INGESTION_EH_NS \ + --resource-group $RESOURCE_GROUP \ + --location $LOCATION + +# Create an event hub +az eventhubs eventhub create --name $INGESTION_EH_NAME \ + --resource-group $RESOURCE_GROUP \ + --namespace-name $INGESTION_EH_NS \ + --partition-count 4 + +# Create consumer group +az eventhubs eventhub consumer-group create --eventhub-name $INGESTION_EH_NAME \ + --name $INGESTION_EH_CONSUMERGROUP_NAME \ + --namespace-name $INGESTION_EH_NS \ + --resource-group $RESOURCE_GROUP + +# Create authorization rule +az eventhubs eventhub authorization-rule create --eventhub-name $INGESTION_EH_NAME \ + --name IngestionServiceAccessKey \ + --namespace-name $INGESTION_EH_NS \ + --resource-group $RESOURCE_GROUP \ + --rights Listen Send + +# Get access key +export EH_ACCESS_KEY_VALUE=$(az eventhubs eventhub authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_EH_NS --name IngestionServiceAccessKey --eventhub-name $INGESTION_EH_NAME --query primaryKey -o tsv) ``` -Note: you could also create this from [the Azure Portal](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create) Build the Ingestion service @@ -214,7 +232,7 @@ Build the Ingestion service export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion # Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ +docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk # Build the docker image @@ -229,20 +247,19 @@ Deploy the Ingestion service ```bash # Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" ./microservices-reference-implementation/k8s/ingestion.yaml - -# Get the EventHub shared access policy name and key from the Azure Portal -export EH_ACCESS_KEY_NAME=[YOUR_SHARED_ACCESS_POLICY_NAME_HERE] -export EH_ACCESS_KEY_VALUE=[YOUR_SHARED_ACCESS_POLICY_VALUE_HERE] +sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml # Create secret kubectl -n shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ --from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ +--from-literal=eventhub_keyname=IngestionServiceAccessKey \ --from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} # Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml +kubectl --namespace shipping apply -f $K8S/ingestion-0.yaml + +# Verify the pod is created +kubectl get pods -n shipping ``` ## Deploy the Scheduler service @@ -260,7 +277,7 @@ Build the Scheduler service export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler # Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ +docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk # Build the docker image @@ -275,12 +292,20 @@ Deploy the Scheduler service ```bash # Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" ./microservices-reference-implementation/k8s/scheduler.yaml +sed "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" $K8S/scheduler.yaml > $K8S/scheduler-0.yaml + +export EH_CONNECTION_STRING=$(az eventhubs eventhub authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_EH_NS --name IngestionServiceAccessKey --eventhub-name $INGESTION_EH_NAME --query primaryConnectionString -o tsv) -# Get the following values from the Azure Portal -export EH_CONNECTION_STRING="[YOUR_EVENT_HUB_CONNECTION_STRING_HERE]" -export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] -export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" +export STORAGE_ACCOUNT_CONNECTION_STRING=$(az storage account show-connection-string \ + --name $SCHEDULER_STORAGE_ACCOUNT_NAME \ + --resource-group $RESOURCE_GROUP \ + --output tsv) + +export STORAGE_ACCOUNT_ACCESS_KEY=$(az storage account keys list \ + --account-name $SCHEDULER_STORAGE_ACCOUNT_NAME \ + --resource-group $RESOURCE_GROUP \ + --query "[0].value" \ + --output tsv) # Create secrets kubectl -n shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ @@ -290,7 +315,10 @@ kubectl -n shipping create secret generic scheduler-secrets --from-literal=event --from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} # Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml +kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler-0.yaml + +# Verify all pods are created +kubectl get pods -n shipping ``` ## Deploy mock services @@ -305,12 +333,12 @@ docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up Build and publish the container image ```bash -# Build the Docker image +# Build the Docker images docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. -# Push the image to ACR +# Push the images to ACR az acr login --name $ACR_NAME docker push $ACR_SERVER/account:0.1.0 && \ docker push $ACR_SERVER/dronescheduler:0.1.0 && \ @@ -321,22 +349,59 @@ Deploy the mock services: ```bash # Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml +sed "s#image:#image: $ACR_SERVER/account:0.1.0#g" $K8S/account.yaml > $K8S/account-0.yaml && \ +sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" $K8S/dronescheduler.yaml > $K8S/dronescheduler-0.yaml && \ +sed "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" $K8S/thirdparty.yaml > $K8S/thirdparty-0.yaml # Deploy the service -kubectl --namespace accounts apply -f ./microservices-reference-implementation/k8s/account.yaml && \ -kubectl --namespace dronemgmt apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -kubectl --namespace 3rdparty apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml -``` +kubectl --namespace accounts apply -f $K8S/account-0.yaml && \ +kubectl --namespace dronemgmt apply -f $K8S/dronescheduler-0.yaml && \ +kubectl --namespace 3rdparty apply -f $K8S/thirdparty-0.yaml ## Verify all services are running: +kubectl get all --all-namespaces -l co=fabrikam +``` + +## Deploy linkerd + ```bash -kubectl get all --all-namespaces -l co=fabrikam +kubectl create ns linkerd +wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ +sed -i "s#/default#/shipping#g" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ +kubectl apply -f servicemesh.yml +``` + +For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) + +> Note: +> The service mesh configuration linked above uses the default namespace for service discovery. +> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified as shown. This change modifies the dtab rules. + +## Validate the application is running + +You can send delivery requests to the ingestion service using the Swagger UI. + +Get the public IP address of the Ingestion Service: + +```bash +export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") ``` +Use a web browser to navigate to `http://EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. + +> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. + +## Optional steps + +Follow these steps to add logging and monitoring capabilities to the solution. + Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch ```bash @@ -361,36 +426,4 @@ sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticse kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml ``` -#### Deploy linkerd - -For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) - -> Note: -> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. -> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified. This will consist of a small change in the dtab rules. - -Deploy linkerd defaulting the namespace to shipping instead: - -```bash -wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ -sed -i "s#/default#/shipping#g" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -kubectl apply -f servicemesh.yml -``` - Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy - -It is recommended to put an API Gateway in front of all APIs you want exposed to the public, -however for convenience, we exposed the Ingestion service with a public IP address. - -You can send delivery requests to the ingestion service using the swagger ui. - -```bash -export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") -curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` diff --git a/k8s/scheduler.yaml b/k8s/scheduler.yaml index 246b88c5..1d42794f 100644 --- a/k8s/scheduler.yaml +++ b/k8s/scheduler.yaml @@ -16,7 +16,7 @@ metadata: co: fabrikam spec: serviceName: scheduler - replicas: 32 + replicas: 4 selector: matchLabels: app: scheduler @@ -84,7 +84,7 @@ spec: name: scheduler-secrets key: queueconstring - name: IOTHUB_EVENTHUB_PARTITIONS - value: "32" + value: "4" # Integration with l5d Daemon Sets - name: NODE_NAME valueFrom: From 39e6556184e6949ca77c22dcf06cb3b27250c731 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Thu, 16 Aug 2018 11:46:47 -0700 Subject: [PATCH 033/246] Use ConfigurationOptions class to build connection string (#106) --- deployment.md | 6 ++++-- deploymentARM.md | 6 ++++-- k8s/delivery.yaml | 9 +++++++-- .../Services/RedisCache.cs | 14 ++++++++++---- .../Startup.cs | 4 ++-- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/deployment.md b/deployment.md index d9cd39cf..44fe8306 100644 --- a/deployment.md +++ b/deployment.md @@ -120,7 +120,8 @@ docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 Create Kubernetes secrets ```bash -export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] +export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) +export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query primaryKey -o tsv) export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) @@ -128,7 +129,8 @@ export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-gro kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ --from-literal=CosmosDB_Key=${COSMOSDB_KEY} \ --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT} \ - --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ + --from-literal=Redis_Endpoint=${REDIS_ENDPOINT} \ + --from-literal=Redis_AccessKey=${REDIS_KEY} \ --from-literal=EH_ConnectionString= ``` diff --git a/deploymentARM.md b/deploymentARM.md index 73621f9f..dc3fb123 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -119,7 +119,8 @@ docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 Create Kubernetes secrets ```bash -export REDIS_CONNECTION_STRING=[YOUR_REDIS_CONNECTION_STRING] +export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) +export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query primaryKey -o tsv) export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value | sed -e 's/^"//' -e 's/"$//') && \ export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ @@ -128,7 +129,8 @@ export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-gro kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ - --from-literal=Redis_ConnectionString=${REDIS_CONNECTION_STRING} \ + --from-literal=Redis_Endpoint=${REDIS_ENDPOINT} \ + --from-literal=Redis_AccessKey=${REDIS_KEY} \ --from-literal=EH_ConnectionString= ``` diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index d35da49f..7bf0a40f 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -66,11 +66,16 @@ spec: value: "CosmosDB_DatabaseId" - name: DOCDB_COLLECTIONID value: "CosmosDB_CollectionId" - - name: REDIS_CONNSTR + - name: REDIS_ENDPOINT valueFrom: secretKeyRef: name: delivery-storageconf - key: Redis_ConnectionString + key: Redis_Endpoint + - name: REDIS_KEY + valueFrom: + secretKeyRef: + name: delivery-storageconf + key: Redis_AccessKey - name: EH_CONNSTR valueFrom: secretKeyRef: diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs index dbfad577..a14a2682 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/RedisCache.cs @@ -16,14 +16,14 @@ namespace Fabrikam.DroneDelivery.DeliveryService.Services { public static class RedisCache where T : BaseCache { - private static string ConnectionString; + private static ConfigurationOptions ConfigOptions; private static int DB; private static ILogger logger; private static Lazy lazyConnection = new Lazy(() => { // automatically disposed when the AppDomain is torn down - var connection = ConnectionMultiplexer.Connect(ConnectionString); + var connection = ConnectionMultiplexer.Connect(ConfigOptions); return connection; }); @@ -49,10 +49,16 @@ private static IDatabase cache } } - public static void Configure(int db, string connectionString, ILoggerFactory loggerFactory) + public static void Configure(int db, string endPoint, string key, ILoggerFactory loggerFactory) { DB = db; - ConnectionString = connectionString; + ConfigOptions = new ConfigurationOptions + { + Password = key, + EndPoints = { { endPoint } }, + Ssl = true, + AbortOnConnectFail = false + }; logger = loggerFactory.CreateLogger(nameof(RedisCache)); } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 592e7597..ca017da8 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -87,8 +87,8 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF //TODO look into creating a factory of DocDBRepos/RedisCache/EventHubMessenger DocumentDBRepository.Configure(Configuration["DOCDB_ENDPOINT"], Configuration["DOCDB_KEY"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["REDIS_CONNSTR"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["REDIS_CONNSTR"], loggerFactory); + RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["REDIS_ENDPOINT"], Configuration["REDIS_KEY"], loggerFactory); + RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["REDIS_ENDPOINT"], Configuration["REDIS_KEY"], loggerFactory); EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); } } From 1f7daf8f8c88bd44ef11a609ed4e066a2ebdb238 Mon Sep 17 00:00:00 2001 From: fsimonazzi Date: Sun, 21 Oct 2018 00:58:06 -0300 Subject: [PATCH 034/246] update instructions to deploy the RI to AKS instead of ACS (#107) --- deployment.md | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/deployment.md b/deployment.md index 44fe8306..e36a2b6f 100644 --- a/deployment.md +++ b/deployment.md @@ -30,23 +30,23 @@ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" export K8S=./microservices-reference-implementation/k8s ``` -Provision a Kubernetes cluster in ACS +Provision a Kubernetes cluster in AKS ```bash # Log in to Azure az login -# Create a resource group for ACS +# Create a resource group for AKS az group create --name $RESOURCE_GROUP --location $LOCATION -# Create the ACS cluster -az acs create --orchestrator-type kubernetes --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --generate-ssh-keys +# Create the AKS cluster +az aks create --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --node-count 4 --enable-addons monitoring --generate-ssh-keys # Install kubectl -sudo az acs kubernetes install-cli +sudo az aks install-cli # Get the Kubernetes cluster credentials -az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create the BC namespaces kubectl create namespace shipping && \ @@ -72,6 +72,17 @@ az acr login --name $ACR_NAME export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer" -o tsv) ``` +Grant the cluster access to the registry. + +```bash +# Acquire the necessary IDs +export CLUSTER_CLIENT_ID=$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "servicePrincipalProfile.clientId" --output tsv) +export ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP --query "id" --output tsv) + +# Grant the cluster read access to the registry +az role assignment create --assignee $CLUSTER_CLIENT_ID --role Reader --scope $ACR_ID +``` + ## Deploy the Delivery service Provision Azure resources @@ -386,6 +397,16 @@ For more information, see [https://linkerd.io/getting-started/k8s/](https://link > The service mesh configuration linked above uses the default namespace for service discovery. > Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified as shown. This change modifies the dtab rules. +The linkerd accounts need to be granted permissions to query the cluster for resources, as RBAC is enabled by default on AKS. + +```bash +wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/linkerd-rbac.yml && \ +sed -i "s#namespace: default#namespace: linkerd#g" linkerd-rbac.yml && \ +kubectl apply -f linkerd-rbac.yml +``` + +For more information on using linkerd with an RBAC-enabled cluster see [https://blog.buoyant.io/2017/07/24/using-linkerd-kubernetes-rbac/](https://blog.buoyant.io/2017/07/24/using-linkerd-kubernetes-rbac/) + ## Validate the application is running You can send delivery requests to the ingestion service using the Swagger UI. @@ -396,7 +417,11 @@ Get the public IP address of the Ingestion Service: export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") ``` -Use a web browser to navigate to `http://EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. +Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. + +```bash +open "http://$EXTERNAL_IP_ADDRESS/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +``` > We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. @@ -412,7 +437,7 @@ kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kuber kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml ``` -Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd +Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd ```bash # The example elasticsearch yaml files deploy a service named "elasticsearch" From 17fbd7ddc1060e4bed3a4e9cf704a1b5725ccff9 Mon Sep 17 00:00:00 2001 From: Gaurav Madaan Date: Mon, 18 Feb 2019 16:56:16 +0530 Subject: [PATCH 035/246] Update deployment.md (#116) thanks for contributing @codestellar, it looks good to me! --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index e36a2b6f..94852a2c 100644 --- a/deployment.md +++ b/deployment.md @@ -2,7 +2,7 @@ ## Prerequisites -- Azure suscription +- Azure subscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) - [Docker Compose](https://docs.docker.com/compose/install/) From be306fa166932e60c1e3e53cdd5f29ca7ff561a2 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Mon, 25 Feb 2019 14:49:45 -0800 Subject: [PATCH 036/246] Update Microsoft.AspNetCore.All version due to vulnerability report --- .../Fabrikam.DroneDelivery.Common.csproj | 2 +- .../Fabrikam.DroneDelivery.DeliveryService.csproj | 2 +- .../delivery/MockAccountService/MockAccountService.csproj | 2 +- .../MockDeliveryScheduler/MockDeliveryScheduler.csproj | 3 +-- .../delivery/MockDroneScheduler/MockDroneScheduler.csproj | 2 +- .../MockThirdPartyService/MockThirdPartyService.csproj | 2 +- src/shipping/delivery/docker-compose.ci.build.yml | 2 +- src/shipping/delivery/docker-compose.dcproj | 2 +- src/shipping/delivery/docker-compose.override.yml | 2 +- src/shipping/delivery/docker-compose.yml | 2 +- 10 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj index 96305d20..a86b0660 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index 7b1d4e88..a6b935d0 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/shipping/delivery/MockAccountService/MockAccountService.csproj b/src/shipping/delivery/MockAccountService/MockAccountService.csproj index 1fce6be8..b8c05b71 100644 --- a/src/shipping/delivery/MockAccountService/MockAccountService.csproj +++ b/src/shipping/delivery/MockAccountService/MockAccountService.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj b/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj index ab929bab..2f790fa5 100644 --- a/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj +++ b/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj @@ -8,8 +8,7 @@ ..\docker-compose.dcproj - - + diff --git a/src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj b/src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj index 1fce6be8..b8c05b71 100644 --- a/src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj +++ b/src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj b/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj index 1fce6be8..b8c05b71 100644 --- a/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj +++ b/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/shipping/delivery/docker-compose.ci.build.yml b/src/shipping/delivery/docker-compose.ci.build.yml index 4bf45581..b6034a2d 100644 --- a/src/shipping/delivery/docker-compose.ci.build.yml +++ b/src/shipping/delivery/docker-compose.ci.build.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.4' services: ci-build: diff --git a/src/shipping/delivery/docker-compose.dcproj b/src/shipping/delivery/docker-compose.dcproj index 6f92515a..791f1205 100644 --- a/src/shipping/delivery/docker-compose.dcproj +++ b/src/shipping/delivery/docker-compose.dcproj @@ -6,7 +6,7 @@ http://localhost:{ServicePort}/api/values fabrikam.dronedelivery.deliveryservice Linux - 2.0 + 2.1 diff --git a/src/shipping/delivery/docker-compose.override.yml b/src/shipping/delivery/docker-compose.override.yml index 9bb326a3..b58ab7d7 100644 --- a/src/shipping/delivery/docker-compose.override.yml +++ b/src/shipping/delivery/docker-compose.override.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.4' services: delivery: diff --git a/src/shipping/delivery/docker-compose.yml b/src/shipping/delivery/docker-compose.yml index e1824ea8..46c89110 100644 --- a/src/shipping/delivery/docker-compose.yml +++ b/src/shipping/delivery/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.4' services: delivery: From 6a966186ad191d527ac91a3781ea19543fc3a080 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 6 Nov 2018 17:54:58 +0000 Subject: [PATCH 037/246] Merged PRs 586-590-591-595-597: Create Azure Pipeline CI/CD solved: #7722, #7723, #7724, #7888, #7950 --- charts/delivery/Chart.yaml | 4 + charts/delivery/templates/NOTES.txt | 10 ++ charts/delivery/templates/_helpers.tpl | 32 ++++ .../delivery/templates/delivery-deploy.yaml | 70 ++++++++ .../delivery/templates/delivery-service.yaml | 21 +++ charts/delivery/values.yaml | 10 ++ charts/package/Chart.yaml | 4 + charts/package/templates/NOTES.txt | 10 ++ charts/package/templates/_helpers.tpl | 32 ++++ charts/package/templates/package-deploy.yaml | 57 +++++++ charts/package/templates/package-service.yaml | 21 +++ charts/package/values.yaml | 10 ++ deployment.md | 34 ++-- deploymentARM.md | 52 +++--- deploymentCICD.md | 115 +++++++++++++ src/shipping/delivery/Dockerfile | 62 +++++++ .../Dockerfile | 6 - src/shipping/delivery/azure-pipelines-ci.yml | 157 ++++++++++++++++++ .../delivery/azure-pipelines-validation.yml | 61 +++++++ src/shipping/delivery/azure-pipelines.yml | 42 +++++ src/shipping/delivery/scripts/run.sh | 2 + src/shipping/package/.dockerignore | 16 ++ src/shipping/package/Dockerfile | 34 ++++ src/shipping/package/azure-pipelines-cd.yml | 64 +++++++ src/shipping/package/azure-pipelines-ci.yml | 118 +++++++++++++ .../package/azure-pipelines-validation.yml | 58 +++++++ src/shipping/package/azure-pipelines.yml | 42 +++++ src/shipping/package/build/prod.dockerfile | 11 -- src/shipping/package/gulpfile.js | 17 +- src/shipping/package/package-service.md | 5 +- src/shipping/package/package.json | 57 +++---- src/shipping/package/readme.md | 0 32 files changed, 1134 insertions(+), 100 deletions(-) create mode 100644 charts/delivery/Chart.yaml create mode 100644 charts/delivery/templates/NOTES.txt create mode 100644 charts/delivery/templates/_helpers.tpl create mode 100644 charts/delivery/templates/delivery-deploy.yaml create mode 100644 charts/delivery/templates/delivery-service.yaml create mode 100644 charts/delivery/values.yaml create mode 100644 charts/package/Chart.yaml create mode 100644 charts/package/templates/NOTES.txt create mode 100644 charts/package/templates/_helpers.tpl create mode 100644 charts/package/templates/package-deploy.yaml create mode 100644 charts/package/templates/package-service.yaml create mode 100644 charts/package/values.yaml create mode 100644 deploymentCICD.md create mode 100644 src/shipping/delivery/Dockerfile delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile create mode 100644 src/shipping/delivery/azure-pipelines-ci.yml create mode 100644 src/shipping/delivery/azure-pipelines-validation.yml create mode 100644 src/shipping/delivery/azure-pipelines.yml create mode 100644 src/shipping/delivery/scripts/run.sh create mode 100644 src/shipping/package/.dockerignore create mode 100644 src/shipping/package/Dockerfile create mode 100644 src/shipping/package/azure-pipelines-cd.yml create mode 100644 src/shipping/package/azure-pipelines-ci.yml create mode 100644 src/shipping/package/azure-pipelines-validation.yml create mode 100644 src/shipping/package/azure-pipelines.yml delete mode 100644 src/shipping/package/build/prod.dockerfile create mode 100644 src/shipping/package/readme.md diff --git a/charts/delivery/Chart.yaml b/charts/delivery/Chart.yaml new file mode 100644 index 00000000..dd231e35 --- /dev/null +++ b/charts/delivery/Chart.yaml @@ -0,0 +1,4 @@ +name: delivery +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Service diff --git a/charts/delivery/templates/NOTES.txt b/charts/delivery/templates/NOTES.txt new file mode 100644 index 00000000..c5dbfc98 --- /dev/null +++ b/charts/delivery/templates/NOTES.txt @@ -0,0 +1,10 @@ +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +All the objects were created in the namespace {{ .Values.namespace }} + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} + $ helm get {{ .Release.Name }} diff --git a/charts/delivery/templates/_helpers.tpl b/charts/delivery/templates/_helpers.tpl new file mode 100644 index 00000000..44000f85 --- /dev/null +++ b/charts/delivery/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "delivery.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "delivery.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "delivery.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml new file mode 100644 index 00000000..14875480 --- /dev/null +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "delivery.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "delivery.chart" . }} + annotations: + kubernetes.io/change-cause: {{ .Values.reason }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "delivery.chart" . }} + spec: + containers: + - name: deliveryservice + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + env: + - name: DOCDB_KEY + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf + key: CosmosDB_Key + - name: DOCDB_ENDPOINT + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf + key: CosmosDB_Endpoint + - name: DOCDB_DATABASEID + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf + key: CosmosDB_DatabaseId + - name: DOCDB_COLLECTIONID + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf + key: CosmosDB_CollectionId + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: http_proxy + value: $(NODE_NAME):4140 + - name: CORRELATION_HEADER + value: {{ .Values.correlationheader }} + ports: + - name: service + containerPort: 80 diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml new file mode 100644 index 00000000..b6d704ec --- /dev/null +++ b/charts/delivery/templates/delivery-service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "delivery.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + clusterIP: None diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml new file mode 100644 index 00000000..d6d3dabe --- /dev/null +++ b/charts/delivery/values.yaml @@ -0,0 +1,10 @@ +# Default values for dronedelivery. +replicaCount: 1 +image: + repository: + tag: + pullPolicy: IfNotPresent +service: + type: ClusterIP +dockerregistry: +reason: unknown diff --git a/charts/package/Chart.yaml b/charts/package/Chart.yaml new file mode 100644 index 00000000..d4d20804 --- /dev/null +++ b/charts/package/Chart.yaml @@ -0,0 +1,4 @@ +name: package +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Package Service diff --git a/charts/package/templates/NOTES.txt b/charts/package/templates/NOTES.txt new file mode 100644 index 00000000..c5dbfc98 --- /dev/null +++ b/charts/package/templates/NOTES.txt @@ -0,0 +1,10 @@ +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +All the objects were created in the namespace {{ .Values.namespace }} + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} + $ helm get {{ .Release.Name }} diff --git a/charts/package/templates/_helpers.tpl b/charts/package/templates/_helpers.tpl new file mode 100644 index 00000000..166a742e --- /dev/null +++ b/charts/package/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "package.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "package.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "package.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml new file mode 100644 index 00000000..dbedadfa --- /dev/null +++ b/charts/package/templates/package-deploy.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "package.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronepackage + helm.sh/chart: {{ include "package.chart" . }} + annotations: + kubernetes.io/change-cause: {{ .Values.reason }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "package.chart" . }} + spec: + containers: + - name: packageservice + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + env: + - name: CONNECTION_STRING + valueFrom: + secretKeyRef: + name: package-secrets + key: {{ .Release.Name }}-mongodb-pwd + - name: COLLECTION_NAME + value: packages + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: http_proxy + value: $(NODE_NAME):4140 + - name: CORRELATION_HEADER + value: {{ .Values.correlationheader }} + ports: + - name: service + containerPort: 80 diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml new file mode 100644 index 00000000..07eedb12 --- /dev/null +++ b/charts/package/templates/package-service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "package.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronepackage + helm.sh/chart: {{ include "package.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + clusterIP: None diff --git a/charts/package/values.yaml b/charts/package/values.yaml new file mode 100644 index 00000000..1c8e550b --- /dev/null +++ b/charts/package/values.yaml @@ -0,0 +1,10 @@ +# Default values for package. +replicaCount: 1 +image: + repository: + tag: + pullPolicy: IfNotPresent +service: + type: ClusterIP +dockerregistry: +reason: unknown diff --git a/deployment.md b/deployment.md index 94852a2c..98034de5 100644 --- a/deployment.md +++ b/deployment.md @@ -52,10 +52,10 @@ az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME kubectl create namespace shipping && \ kubectl create namespace accounts && \ kubectl create namespace dronemgmt && \ -kubectl create namespace 3rdparty +kubectl create namespace 3rdparty ``` -Create an Azure Container Registry instance. +Create an Azure Container Registry instance. > Note: Azure Container Registory is not required. If you prefer, you can store the Docker images for this solution in another container registry. @@ -106,7 +106,7 @@ az cosmosdb create \ --kind GlobalDocumentDB \ --resource-group $RESOURCE_GROUP \ --max-interval 10 \ - --max-staleness-prefix 200 + --max-staleness-prefix 200 ``` Build the Delivery service @@ -116,11 +116,11 @@ export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/deliv docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up ``` -Build and publish the container image +Build and publish the container image ```bash # Build the Docker image -docker build -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/Fabrikam.DroneDelivery.DeliveryService/. +docker build --pull --compress -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/. # Push the image to ACR az acr login --name $ACR_NAME @@ -203,7 +203,7 @@ kubectl --namespace shipping apply -f $K8S/package-0.yml kubectl get pods -n shipping ``` -## Deploy the Ingestion service +## Deploy the Ingestion service Provision Azure resources ```bash @@ -244,7 +244,7 @@ Build the Ingestion service ```bash export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion -# Build the app +# Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk @@ -275,7 +275,7 @@ kubectl --namespace shipping apply -f $K8S/ingestion-0.yaml kubectl get pods -n shipping ``` -## Deploy the Scheduler service +## Deploy the Scheduler service Provision Azure resources ```bash @@ -289,7 +289,7 @@ Build the Scheduler service ```bash export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler -# Build the app +# Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk @@ -343,13 +343,13 @@ export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up ``` -Build and publish the container image +Build and publish the container image ```bash # Build the Docker images docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ -docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. +docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. # Push the images to ACR az acr login --name $ACR_NAME @@ -382,19 +382,19 @@ kubectl get all --all-namespaces -l co=fabrikam kubectl create ns linkerd wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ sed -i "s#/default#/shipping#g" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ kubectl apply -f servicemesh.yml -``` +``` For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) -> Note: -> The service mesh configuration linked above uses the default namespace for service discovery. +> Note: +> The service mesh configuration linked above uses the default namespace for service discovery. > Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified as shown. This change modifies the dtab rules. The linkerd accounts need to be granted permissions to query the cluster for resources, as RBAC is enabled by default on AKS. @@ -444,13 +444,13 @@ Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/article wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml -# Commenting out X-Pack credentials for demo purposes. +# Commenting out X-Pack credentials for demo purposes. # Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml -``` +``` Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy diff --git a/deploymentARM.md b/deploymentARM.md index dc3fb123..44c64ed6 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -17,10 +17,10 @@ git clone https://github.com/mspnp/microservices-reference-implementation.git > The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -## Generate a SSH rsa public/private key pair +## Generate a SSH rsa public/private key pair -the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). -> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. +the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). +> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. ## Azure Resources Provisioning @@ -50,7 +50,7 @@ export BEARER_TOKEN=$(az account get-access-token --query accessToken | sed -e ' export SUBS_ID=$(az account show --query id | sed -e 's/^"//' -e 's/"$//') ``` -Deployment +Deployment > Note: this deployment might take up to 20 minutes @@ -66,17 +66,17 @@ Deployment * from Azure Portal [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Fmaster%2Fazuredeploy.json) - > Note: - > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group + > Note: + > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group > 2. paste the content of your ssh-rsa public key file in the Ssh RSA Plublic Key field. > 3. paste the $SP_APP_ID and $SP_CLIENT_SECRET in the Client Id and Secret fields. Get outputs from Azure Deploy ```bash -# Shared +# Shared export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value | sed -e 's/^"//' -e 's/"$//') && \ export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value | sed -e 's/^"//' -e 's/"$//') && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acsK8sClusterName.value | sed -e 's/^"//' -e 's/"$//') +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acsK8sClusterName.value | sed -e 's/^"//' -e 's/"$//') ``` Download kubectl and create a k8s namespace @@ -104,11 +104,11 @@ export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/deliv docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up ``` -Build and publish the container image +Build and publish the container image ```bash # Build the Docker image -docker build -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/Fabrikam.DroneDelivery.DeliveryService/. +docker build --pull --compress -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/. # Push the image to ACR az acr login --name $ACR_NAME @@ -182,14 +182,14 @@ kubectl -n shipping create secret generic package-secrets --from-literal=mongodb kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/package.yml ``` -## Deploy the Ingestion service +## Deploy the Ingestion service Build the Ingestion service ```bash export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion -# Build the app +# Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk @@ -225,14 +225,14 @@ kubectl -n shipping create secret generic ingestion-secrets --from-literal=event kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml ``` -## Deploy the Scheduler service +## Deploy the Scheduler service Build the Scheduler service ```bash export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler -# Build the app +# Build the app docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk @@ -278,13 +278,13 @@ export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up ``` -Build and publish the container image +Build and publish the container image ```bash # Build the Docker image docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ -docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. +docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. # Push the image to ACR az acr login --name $ACR_NAME @@ -299,7 +299,7 @@ Deploy the mock services: # Update the image tag in the deployment YAML sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml +sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml # Deploy the service kubectl --namespace accounts apply -f ./microservices-reference-implementation/k8s/account.yaml && \ @@ -328,40 +328,40 @@ Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml -# Commenting out X-Pack credentials for demo purposes. +# Commenting out X-Pack credentials for demo purposes. # Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml -``` +``` -#### Deploy linkerd +#### Deploy linkerd For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) -> Note: -> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. +> Note: +> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. > Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified. This will consist of a small change in the dtab rules. -Deploy linkerd defaulting the namespace to shipping instead: +Deploy linkerd defaulting the namespace to shipping instead: ```bash wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ sed -i "s#/default#/shipping#g" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ kubectl apply -f servicemesh.yml -``` +``` Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy -It is recommended to put an API Gateway in front of all APIs you want exposed to the public, +It is recommended to put an API Gateway in front of all APIs you want exposed to the public, however for convenience, we exposed the Ingestion service with a public IP address. You can send delivery requests to the ingestion service using the swagger ui. diff --git a/deploymentCICD.md b/deploymentCICD.md new file mode 100644 index 00000000..248e5212 --- /dev/null +++ b/deploymentCICD.md @@ -0,0 +1,115 @@ +# Deploying the Reference Implementation + +## Prerequisites +TODO: use azure shell +- Azure suscription +- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Helm 2.11.1](#) +- Azure Pipelines CLI: See the [install steps](https://docs.microsoft.com/en-us/cli/vsts/install?view=vsts-cli-latest) for instructions on installing the VSTS CLI on Windows, Linux, or Mac. +1. [create Azure DevOps account](https://azure.microsoft.com/en-us/services/devops) +2. create a new Azure DevOps organization and a [personal access token](https://docs.microsoft.com/vsts/accounts/use-personal-access-tokens-to-authenticate) +2. [add Azure subscription as service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=vsts#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal) +3. [optionally, assign service connection application to role, so it is allowed to create new azure resources](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal#assign-application-to-role) + +Create a new Azure Repo + +```bash +# login in Azure DevOps +vsts login --token # use the token created in prerequisite step th + +# configure as default the new organization create in prerequisite step th +vsts configure --defaults instance=https://dev.azure.com// + +# create repo +vsts code repo create +``` + +Clone or download this repo locally + +```bash +git clone https://github.com/mspnp/.git && \ +cd && \ +git remote add # this remote url corresponds to the prerequisite step 4th +``` +Export the following environment variables + +``` +export SERVICECONNECTION= # use the name configured in the 2nd prerequisite step +export LOCATION= +export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] + +export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ +export ACR_NAME="${UNIQUE_APP_NAME_PREFIX}-acr" && \ +export AKS_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" +``` + +The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. + +## Create the Kubernetes cluster + +Provision a Kubernetes cluster in AKS +TODO: use arm and deploy aks and other infrastructure needed +```bash +# Log in to Azure +az login + +# Create a resource group for AKS +az group create --name $RESOURCE_GROUP --location $LOCATION + +# Create the AKS cluster +az aks create --resource-group $RESOURCE_GROUP --name $AKS_NAME --node-count 4 --enable-addons monitoring --generate-ssh-keys + +# Install kubectl +sudo az aks install-cli + +# Get the Kubernetes cluster credentials +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$AKS_NAME + +# Create the ACR instance +az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic + +# Add a repository to Helm client +az acr helm repo add -n $ACR_NAME + +# Install Helm +helm init --upgrade +``` + +## Setup CICD Pipeline using Azure Pipelines + +Replace azure pipeline place holders + +``` +sed -i "s#AzureSubscription: #AzureSubscription: $SERVICECONNECTION#g" azure-pipelines.yml && \ +sed -i "s#Location: #Location: $LOCATION#g" azure-pipelines.yml && \ +sed -i "s#ResourceGroup: #ResourceGroup: $RESOURCEGROUP#g" azure-pipelines.yml && \ +sed -i "s#AzureContainerRegistry: #AzureContainerRegistry: $ACR_NAME#g" azure-pipelines.yml && \ +sed -i "s#AzureKubernetesService: #AzureKubernetesService: $AKS_NAME#g" azure-pipelines.yml +``` + +Push changes to azure repos or github + +```bash +git push master +``` + +Follow instructions below to configure your first Azure Pipeline + +[Get your first build with Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started-yaml?view=vsts#get-your-first-build) + +> Note: this first build will attemp to execute the azurepipeline.yml against master + +Trigger the CICD pipeline by pushing to staging + +``` +git checkout -b staging && \ +git push staging +``` + +> Note: also feature branches are going through the CI pipeline. + +Follow CICD from Azure Pipelines + +``` +open https://dev.azure.com///_build +``` diff --git a/src/shipping/delivery/Dockerfile b/src/shipping/delivery/Dockerfile new file mode 100644 index 00000000..4b7bf3e3 --- /dev/null +++ b/src/shipping/delivery/Dockerfile @@ -0,0 +1,62 @@ +FROM microsoft/dotnet:2-aspnetcore-runtime as base +WORKDIR /app +EXPOSE 80 + +FROM microsoft/dotnet:2-sdk AS build + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app +COPY Fabrikam.DroneDelivery.Common/*.csproj ./Fabrikam.DroneDelivery.Common/ +COPY Fabrikam.DroneDelivery.DeliveryService/*.csproj ./Fabrikam.DroneDelivery.DeliveryService/ +WORKDIR /app +RUN dotnet restore /app/Fabrikam.DroneDelivery.Common/ +RUN dotnet restore /app/Fabrikam.DroneDelivery.DeliveryService/ + +WORKDIR /app +COPY Fabrikam.DroneDelivery.Common/. ./Fabrikam.DroneDelivery.Common/ +COPY Fabrikam.DroneDelivery.DeliveryService/. ./Fabrikam.DroneDelivery.DeliveryService/ + +FROM build AS testrunner + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app/tests +COPY Fabrikam.DroneDelivery.DeliveryService.Tests/*.csproj . +WORKDIR /app/tests +RUN dotnet restore /app/tests/ + +WORKDIR /app/tests +COPY Fabrikam.DroneDelivery.DeliveryService.Tests/. . +ENTRYPOINT ["dotnet", "test", "--logger:trx"] + +FROM build AS publish + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app +RUN dotnet publish /app/Fabrikam.DroneDelivery.DeliveryService/ -c Release -o ../out + +FROM base AS runtime + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +LABEL Tags="Azure,AKS,DroneDelivery" + +ARG user=deliveryuser + +RUN useradd -m -s /bin/bash -U $user + +WORKDIR /app +COPY --from=publish /app/out ./ +COPY scripts/. ./ +RUN \ + # Ensures the entry point is executable + chmod ugo+x /app/run.sh + +RUN chown -R $user.$user /app + +# Set it for subsequent commands +USER $user + +ENTRYPOINT ["/bin/bash", "/app/run.sh"] diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile deleted file mode 100644 index 5aa81526..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM microsoft/aspnetcore:2.0 -ARG source -WORKDIR /app -EXPOSE 80 -COPY ${source:-obj/Docker/publish} . -ENTRYPOINT ["dotnet", "Fabrikam.DroneDelivery.DeliveryService.dll"] diff --git a/src/shipping/delivery/azure-pipelines-ci.yml b/src/shipping/delivery/azure-pipelines-ci.yml new file mode 100644 index 00000000..58db4e9d --- /dev/null +++ b/src/shipping/delivery/azure-pipelines-ci.yml @@ -0,0 +1,157 @@ +steps: +- script: echo '##vso[task.setvariable variable=repositoryName]delivery' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- script: echo '##vso[task.setvariable variable=chartPath]charts/delivery' + name: setvarChartPath +- script: echo $(chartPath) + name: echovarChartPath + +- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/delivery/Dockerfile' + name: setvarDockerFileName +- script: echo $(dockerFileName) + name: echovarDockerFileName + +- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' + name: setvarReleaseVersion +- script: echo $(releaseVersion) + name: echovarReleaseVersion + +- script: echo '##vso[task.setvariable variable=aksNamespace]backend' + name: setvarAKSNamespace +- script: echo $(aksNamespace) + name: echovarAKSNamespace + +- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' + name: setvarImageName +- script: echo $(imageName) + name: echovarImageName + +- task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + +- task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/app/tests/TestResults' + + imageName: '$(imageName)-test' + + runInBackground: false + +- task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/*.trx' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + +- task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + +- task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + +- task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.9.1' + inputs: + kubectlVersion: 1.10.3 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + +- task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(AzureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + force='--force' + packageExists="$(az acr helm list -n pnpdhacr --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + + export force + export packageExists + + if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then + force='' + fi + + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name pnpdhacr $force; diff --git a/src/shipping/delivery/azure-pipelines-validation.yml b/src/shipping/delivery/azure-pipelines-validation.yml new file mode 100644 index 00000000..41929866 --- /dev/null +++ b/src/shipping/delivery/azure-pipelines-validation.yml @@ -0,0 +1,61 @@ +variables: + ResourceGroup: 'pnpdh-rg' + AzureKubernetesService: 'pnpdh-cluster' + BuildConfiguration: "Release" + AzureSubscription: arm_aks-ra + AzureContainerRegistry: pnpdhacr.azurecr.io + AzureContainerRegistryHelmRepo: pnpdhacr + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + branches: + include: + + - master + + - release/delivery/v* # for bug fixes + + paths: + include: + + - /src/shipping/delivery/ + +trigger: + batch: true + branches: + include: + + - master + + - feature/* + + - topic/* + + exclude: + + - feature/experimental/* + + - topic/experimental/* + + paths: + include: + + - /src/shipping/delivery/ + +resources: +- repo: self + +jobs: + +# CI +- job: deliverycivalidationjob + displayName: "Delivery CI Validation (build & test)" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: False + buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] + steps: + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml new file mode 100644 index 00000000..5c2d2e93 --- /dev/null +++ b/src/shipping/delivery/azure-pipelines.yml @@ -0,0 +1,42 @@ +variables: + ResourceGroup: 'pnpdh-rg' + AzureKubernetesService: 'pnpdh-cluster' + BuildConfiguration: "Release" + AzureSubscription: arm_aks-ra + AzureContainerRegistry: pnpdhacr.azurecr.io + AzureContainerRegistryHelmRepo: pnpdhacr + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +trigger: + batch: true + branches: + include: + + # for new release to production: release flow strategy + - release/delivery/v* + + - refs/relelase/delivery/v* + + paths: + include: + + - /src/shipping/delivery/ + +resources: +- repo: self + +jobs: + +# CI +- job: deliveryjobci + displayName: "Delivery CI" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/delivery/v') ] + steps: + - script: echo $(fullCI) + name: echovar + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/delivery/scripts/run.sh b/src/shipping/delivery/scripts/run.sh new file mode 100644 index 00000000..fd583e96 --- /dev/null +++ b/src/shipping/delivery/scripts/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash +dotnet Fabrikam.DroneDelivery.DeliveryService.dll diff --git a/src/shipping/package/.dockerignore b/src/shipping/package/.dockerignore new file mode 100644 index 00000000..7c21ce5e --- /dev/null +++ b/src/shipping/package/.dockerignore @@ -0,0 +1,16 @@ +# env # +.git +.github +.vscode + +# others +.DS_Store +ehthumbs.db +Icon? +Thumbs.db + +# Node files +node_modules +npm-debug.log +npm-debug.log.* +logs/* diff --git a/src/shipping/package/Dockerfile b/src/shipping/package/Dockerfile new file mode 100644 index 00000000..787a27bd --- /dev/null +++ b/src/shipping/package/Dockerfile @@ -0,0 +1,34 @@ +FROM node:8.12.0-alpine as base + +WORKDIR /app + +EXPOSE 80 + +# ---- install dependencies ---- +FROM base AS dependencies + +WORKDIR /app +COPY package.json . +COPY gulpfile.js . +RUN npm set progress=false && npm config set depth 0 +RUN npm install --only=production +RUN cp -R node_modules prod_node_modules + +# ---- build ---- +FROM dependencies AS build +WORKDIR /app +RUN npm install +COPY tsconfig.json . +COPY app app/. +RUN npm run build + +# ---- runtime ---- +FROM base AS runtime + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app +COPY --from=dependencies /app/prod_node_modules ./node_modules +COPY --from=build /app/.bin/app . + +ENTRYPOINT ["node", "main.js"] diff --git a/src/shipping/package/azure-pipelines-cd.yml b/src/shipping/package/azure-pipelines-cd.yml new file mode 100644 index 00000000..701b4d08 --- /dev/null +++ b/src/shipping/package/azure-pipelines-cd.yml @@ -0,0 +1,64 @@ +steps: + +- script: echo '##vso[task.setvariable variable=repositoryName]package' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- task: HelmInstaller@0 + displayName: 'Install Helm 2.9.1' + inputs: + kubectlVersion: 1.10.3 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + namespace: prod + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: AzureCLI@1 + displayName: 'Add repository to Helm client' + inputs: + azureSubscription: 'arm_service_connection' + + scriptLocation: inlineScript + + inlineScript: 'az acr helm repo add --name $(AzureContainerRegistryHelmRepo)' + +- task: HelmDeploy@0 + displayName: 'helm upgrade ' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + namespace: backend + + command: 'upgrade ' + + chartType: Name + + chartName: $(AzureContainerRegistryHelmRepo)/$(repositoryName) + + arguments: '--version $(Build.SourceBranchName)' + + releaseName: $(repositoryName)-$(Build.SourceBranchName) + + overrideValues: 'image.tag=$(Build.SourceBranchName),image.repository=$(repositoryName),dockerregistry=$(AzureContainerRegistry)' + + waitForExecution: false diff --git a/src/shipping/package/azure-pipelines-ci.yml b/src/shipping/package/azure-pipelines-ci.yml new file mode 100644 index 00000000..bb7597bc --- /dev/null +++ b/src/shipping/package/azure-pipelines-ci.yml @@ -0,0 +1,118 @@ +steps: +- script: echo '##vso[task.setvariable variable=repositoryName]package' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- script: echo '##vso[task.setvariable variable=chartPath]charts/package' + name: setvarChartPath +- script: echo $(chartPath) + name: echovarChartPath + +- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/package/Dockerfile' + name: setvarDockerFileName +- script: echo $(dockerFileName) + name: echovarDockerFileName + +- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' + name: setvarReleaseVersion +- script: echo $(releaseVersion) + name: echovarReleaseVersion + +- script: echo '##vso[task.setvariable variable=aksNamespace]backend' + name: setvarAKSNamespace +- script: echo $(aksNamespace) + name: echovarAKSNamespace + +- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' + name: setvarImageName +- script: echo $(imageName) + name: echovarImageName + +- task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: true + + imageName: '$(imageName)' + +- task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + +- task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.9.1' + inputs: + kubectlVersion: 1.10.3 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + namespace: prod + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + +- task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(AzureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + force='--force' + packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + + export force + export packageExists + + if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then + force='' + fi + + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; diff --git a/src/shipping/package/azure-pipelines-validation.yml b/src/shipping/package/azure-pipelines-validation.yml new file mode 100644 index 00000000..6b183002 --- /dev/null +++ b/src/shipping/package/azure-pipelines-validation.yml @@ -0,0 +1,58 @@ +variables: + ResourceGroup: 'pnpdh-rg' + AzureKubernetesService: 'pnpdh-cluster' + BuildConfiguration: "Release" + AzureSubscription: arm_aks-ra + AzureContainerRegistry: pnpdhacr.azurecr.io + AzureContainerRegistryHelmRepo: pnpdhacr + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + branches: + include: + + - master + - release/package/v* # for bug fixes + + paths: + include: + - /src/shipping/package/ + +trigger: + batch: true + branches: + include: + - master + + - feature/* + + - topic/* + + exclude: + + - feature/experimental/* + + - topic/experimental/* + + paths: + include: + + - /src/shipping/package/ + +resources: +- repo: self + +jobs: + +# CI +- job: packagecivalidationjob + displayName: "Package CI Validation (build & test)" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: False + buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] + steps: + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/package/azure-pipelines.yml b/src/shipping/package/azure-pipelines.yml new file mode 100644 index 00000000..1aca0789 --- /dev/null +++ b/src/shipping/package/azure-pipelines.yml @@ -0,0 +1,42 @@ +variables: + ResourceGroup: 'pnpdh-rg' + AzureKubernetesService: 'pnpdh-cluster' + BuildConfiguration: "Release" + AzureSubscription: arm_aks-ra + AzureContainerRegistry: pnpdhacr.azurecr.io + AzureContainerRegistryHelmRepo: pnpdhacr + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +trigger: + batch: true + branches: + include: + + # for new release to production: release flow strategy + - release/package/v* + + - refs/relelase/package/v* + + paths: + include: + + - /src/shipping/package/ + +resources: +- repo: self + +jobs: + +# CI +- job: packagejobci + displayName: "Package CI" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/package/v') ] + steps: + - script: echo $(fullCI) + name: echovar + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/package/build/prod.dockerfile b/src/shipping/package/build/prod.dockerfile deleted file mode 100644 index fb0fca46..00000000 --- a/src/shipping/package/build/prod.dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM node:8.0.0 - -WORKDIR /app -EXPOSE 80 - -COPY package.json /app/ -RUN npm install --production - -COPY .bin/app /app - -ENTRYPOINT ["node", "main.js"] diff --git a/src/shipping/package/gulpfile.js b/src/shipping/package/gulpfile.js index 37e2e334..162f5acf 100644 --- a/src/shipping/package/gulpfile.js +++ b/src/shipping/package/gulpfile.js @@ -5,7 +5,6 @@ var del = require('del'); var mocha = require('gulp-mocha'); gulp.task('set-env', function() { - process.env.CONNECTION_STRING = "mongodb://packagedb:27017/local"; process.env.COLLECTION_NAME = "packages"; process.env.CORRELATION_HEADER = "x-b3-traceid"; @@ -26,13 +25,13 @@ gulp.task ('create-api-classes', function() { for (var gprop in definition.properties) { let propType = definition.properties[gprop]['type']; - + // If this is a reference to a class we will use that if (propType == null) { propType = definition.properties[gprop]['$ref'].split('/').splice(-1); } - // convert swagger integer to + // convert swagger integer to if (propType == "integer") propType = "number"; @@ -41,13 +40,13 @@ gulp.task ('create-api-classes', function() { } generated += "}\n\n"; } - + var fs = require('fs'); fs.writeFile("./app/models/api-models.ts", generated, function(err) { if(err) { return console.log(err); } - }); + }); }); gulp.task('build', ['clean'], function () { @@ -78,16 +77,16 @@ gulp.task('clean', function (done) { return del(['.bin/'], done); }); -gulp.task('build-test', ['build'], function () { +gulp.task('build-int-test', ['build'], function () { var tsProject = ts.createProject('tsconfig.json'); - + var tsResult = gulp.src("test/**/*.ts", {'base':'.'}) .pipe(tsProject()); return tsResult.js.pipe(gulp.dest('.bin')); }); -gulp.task('test', ['build-test', 'set-env'], function() { +gulp.task('int-test', ['build-int-test', 'set-env'], function() { gulp.src("test/**/*.js", { cwd: '.bin' }) .pipe(mocha({ reporter: 'list' })); -}); \ No newline at end of file +}); diff --git a/src/shipping/package/package-service.md b/src/shipping/package/package-service.md index bb68baa9..62bcdf91 100644 --- a/src/shipping/package/package-service.md +++ b/src/shipping/package/package-service.md @@ -4,15 +4,14 @@ 1. Install Docker 2. From a bash CLI navigate to the project root and type `./up.sh` -3. From the app environment type `npm install` +3. From the app environment type `npm install` 4. To run the app in the local dev environment, type `npm start` 5. The app listens on port 80. In the dev environment, this is mapped to localhost:7080 ## Build docker image ``` -npm run build -docker build -f ./build/prod.dockerfile -t /package-service . +docker build -f ./Dockerfile -t /package-service . ``` ## Provision database and create secrets diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 8fe5e8fb..1c04bd9b 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -7,13 +7,13 @@ "build": "gulp build", "clean": "gulp clean", "api": "gulp create-api-classes", - "test": "gulp test", + "int-test": "gulp int-test", "ncu": "ncu", "ncu:update": "ncu -u" }, "repository": { "type": "git", - "url": "ssh://pnp@pnp.visualstudio.com:22/DefaultCollection/_git/DroneDelivery-PackageService" + "url": "" }, "keywords": [ "Drone", @@ -23,34 +23,35 @@ "author": "Microsoft Patterns and Practices", "license": "MIT", "dependencies": { - "fleek-context": "1.0.6", - "fleek-router": "2.0.2", - "fleek-validator": "2.0.0", - "koa": "2.2.0", - "koa-body": "2.1.0", - "koa-bodyparser": "4.2.0", - "koa-compress": "2.0.0", - "koa-convert": "1.2.0", + "fleek-context": "^1.0.6", + "fleek-router": "^2.1.0", + "fleek-validator": "^2.0.1", + "koa": "^2.6.1", + "koa-body": "^4.0.4", + "koa-bodyparser": "^4.2.1", + "koa-compress": "^3.0.0", + "koa-convert": "^1.2.0", "koa-cors": "0.0.16", - "koa-route": "3.2.0", - "mongodb": "2.2.28", - "winston": "^2.4.0" + "koa-route": "^3.2.0", + "mongodb": "^3.1.8", + "winston": "^3.1.0" }, "devDependencies": { - "@types/koa": "2.0.39", - "@types/koa-bodyparser": "3.0.23", - "@types/mocha": "2.2.41", - "@types/node": "7.0.29", - "@types/winston": "^2.3.6", - "del": "3.0.0", - "gulp": "3.9.1", - "gulp-cli": "1.3.0", - "gulp-mocha": "^4.3.1", - "gulp-nodemon": "2.2.1", - "gulp-typescript": "3.1.7", - "mocha": "3.4.2", - "nodemon": "1.11.0", - "npm-check-updates": "2.11.3", - "typescript": "2.3.4" + "@types/koa": "^2.0.46", + "@types/koa-bodyparser": "^5.0.1", + "@types/mocha": "^5.2.5", + "@types/node": "^10.12.2", + "@types/winston": "^2.4.4", + "del": "^3.0.0", + "gulp": "^3.9.1", + "gulp-cli": "^2.0.1", + "gulp-mocha": "^6.0.0", + "gulp-nodemon": "^2.4.1", + "gulp-typescript": "^4.0.2", + "gulp-util": "^3.0.8", + "mocha": "^5.2.0", + "nodemon": "^1.18.6", + "npm-check-updates": "^2.14.2", + "typescript": "^3.1.6" } } diff --git a/src/shipping/package/readme.md b/src/shipping/package/readme.md new file mode 100644 index 00000000..e69de29b From 1d8d44ba0a409ee5c07c7bd47472f6e10e812e5a Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 14 Dec 2018 20:58:59 +0000 Subject: [PATCH 038/246] Merged PR 603: Publish secrets through the key vault flex volume Creates a KeyVault to hold the secrets for the delivery service, and exposes them using the KeyVault flex volume using AAD Pod identity. These secrets exist SxS with the k8s secrets, and the app does not consume them yet. Updated the deployment instructions with all the necessary details. Also adds yaml files for setting up pod identity and updates the delivery yaml to use pod identity, mount the volumes and sets the fsGroup to allow the delivery user to access the mounted volumes. The change to the service's port is due to an issue encountered recently where the app wouldn't be able to bind port 80. This is not caused by the changes, as I've rolled them back and it would still fail. Not sure what this is about but will need to be investigated. Related work items: #8117, #8148, #8150 --- deployment.md | 80 ++++++++++++++++++- k8s/delivery-identity.yaml | 24 ++++++ k8s/delivery.yaml | 29 ++++++- .../ConfigurationBuilderExtensions.cs | 23 ++++++ .../Fabrikam.DroneDelivery.Common.csproj | 3 +- ...rikam.DroneDelivery.DeliveryService.csproj | 2 +- .../Program.cs | 1 + .../Startup.cs | 8 ++ 8 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 k8s/delivery-identity.yaml create mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs diff --git a/deployment.md b/deployment.md index 98034de5..323f565b 100644 --- a/deployment.md +++ b/deployment.md @@ -27,6 +27,9 @@ export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" +export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export TENANT_ID=$(az account show --query tenantId --output tsv) + export K8S=./microservices-reference-implementation/k8s ``` @@ -48,6 +51,9 @@ sudo az aks install-cli # Get the Kubernetes cluster credentials az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +# Get the cluster's principal +export CLUSTER_SERVICE_PRINCIPAL=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query servicePrincipalProfile.clientId --output tsv) + # Create the BC namespaces kubectl create namespace shipping && \ kubectl create namespace accounts && \ @@ -83,6 +89,17 @@ export ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP -- az role assignment create --assignee $CLUSTER_CLIENT_ID --role Reader --scope $ACR_ID ``` +## Setup AAD pod identity and key vault flexvol infrastructure + +Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol + +```bash +# setup AAD pod identity +kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml + +kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml +``` + ## Deploy the Delivery service Provision Azure resources @@ -125,7 +142,6 @@ docker build --pull --compress -t $ACR_SERVER/fabrikam.dronedelivery.deliveryser # Push the image to ACR az acr login --name $ACR_NAME docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 - ``` Create Kubernetes secrets @@ -145,6 +161,45 @@ kubectl --namespace shipping create --save-config=true secret generic delivery-s --from-literal=EH_ConnectionString= ``` +Create KeyVault and secrets + +```bash +export DELIVERY_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-kv" +az keyvault create --name $DELIVERY_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Key --value ${COSMOSDB_KEY} # (consider using encoding base64 to keep the actual values) +az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Endpoint --value ${COSMOSDB_ENDPOINT} +az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-Endpoint --value ${REDIS_ENDPOINT} +az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-AccessKey --value ${REDIS_KEY} +# az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name EH-ConnectionString # cannot create a secret without a value or with an empty value + +export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) +export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) +``` + +Create and set up pod identity + +```bash +# Create the identity and extract properties +export DELIVERY_PRINCIPAL_NAME=delivery +az identity create --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "id" --output tsv) +export DELIVERY_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "principalId" --output tsv) +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "clientId" --output tsv) + +# Grant the identity access to the KeyVault +az role assignment create --role Reader --assignee $DELIVERY_PRINCIPAL_ID --scope $DELIVERY_KEYVAULT_ID +az keyvault set-policy --name $DELIVERY_KEYVAULT_NAME --secret-permissions get, list --spn $DELIVERY_PRINCIPAL_CLIENT_ID + +# Allow the cluster to manage the identity to assign to pods +az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DELIVERY_PRINCIPAL_RESOURCE_ID + +# Deploy the identity resources +cat $K8S/delivery-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $DELIVERY_PRINCIPAL_CLIENT_ID#g" > $K8S/delivery-identity-0.yaml +kubectl apply -f $K8S/delivery-identity-0.yaml +``` + Deploy the Delivery service: ```bash @@ -152,7 +207,12 @@ Deploy the Delivery service: sed "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" $K8S/delivery.yaml | \ sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ - sed "s/value: \"EH_EntityPath\"/value:/g" > $K8S/delivery-0.yaml + sed "s/value: \"EH_EntityPath\"/value:/g" | \ + sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" | \ + sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ + sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ + sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ + sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $DELIVERY_KEYVAULT_NAME#g" > $K8S/delivery-0.yaml # Deploy the service kubectl --namespace shipping apply -f $K8S/delivery-0.yaml @@ -196,6 +256,11 @@ sed "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" $K8S/package.yml > $K8 export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv) kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION +# Create KeyVault secret +export PACKAGE_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-package-kv" +az keyvault create --name $PACKAGE_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +az keyvault secret set --vault-name $PACKAGE_KEYVAULT_NAME --name mongodb-pwd --value $COSMOSDB_CONNECTION + # Deploy service kubectl --namespace shipping apply -f $K8S/package-0.yml @@ -263,11 +328,20 @@ Deploy the Ingestion service sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml # Create secret -kubectl -n shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ +kubectl -n shipping create secret generic ingestion-secrets \ +--from-literal=eventhub_namespace=${INGESTION_EH_NS} \ --from-literal=eventhub_name=${INGESTION_EH_NAME} \ --from-literal=eventhub_keyname=IngestionServiceAccessKey \ --from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} +# Create KeyVault secrets +export INGESTION_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-ingestion-kv" +az keyvault create --name $INGESTION_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-namespace --value ${INGESTION_EH_NS} +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-name --value ${INGESTION_EH_NAME} +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyname --value IngestionServiceAccessKey +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyvalue --value ${EH_ACCESS_KEY_VALUE} + # Deploy service kubectl --namespace shipping apply -f $K8S/ingestion-0.yaml diff --git a/k8s/delivery-identity.yaml b/k8s/delivery-identity.yaml new file mode 100644 index 00000000..f9ded1e7 --- /dev/null +++ b/k8s/delivery-identity.yaml @@ -0,0 +1,24 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service identityß +################################################################################################### +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentity +metadata: + name: delivery-identity +spec: + type: 0 + ResourceID: "identityResourceId" + ClientID: "identityClientid" +--- +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentityBinding +metadata: + name: delivery-identity-binding +spec: + AzureIdentity: delivery-identity + Selector: delivery \ No newline at end of file diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index 7bf0a40f..b72a7217 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -18,7 +18,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: 8080 selector: app: delivery clusterIP: None @@ -32,6 +32,7 @@ metadata: version: 0.1.0 bc: shipping co: fabrikam + aadpodidbinding: delivery spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -44,12 +45,19 @@ spec: version: 0.1.0 bc: shipping co: fabrikam + aadpodidbinding: delivery annotations: team: deliveryservice spec: + securityContext: + fsGroup: 1 containers: - name: delivery image: + volumeMounts: + - name: delivery + mountPath: /kvmnt + readOnly: true env: # Export K8s secrets - name: DOCDB_KEY @@ -66,6 +74,8 @@ spec: value: "CosmosDB_DatabaseId" - name: DOCDB_COLLECTIONID value: "CosmosDB_CollectionId" + - name: KEY_VAULT_URI + value: "KeyVault_Name" - name: REDIS_ENDPOINT valueFrom: secretKeyRef: @@ -94,11 +104,13 @@ spec: fieldPath: status.podIP - name: http_proxy value: $(NODE_NAME):4140 + - name: no_proxy + value: 169.254.169.254 - name: CORRELATION_HEADER value: l5d-ctx-trace ports: - name: service - containerPort: 80 + containerPort: 8080 resources: requests: cpu: 1 @@ -106,3 +118,16 @@ spec: limits: cpu: 1 memory: 3G + volumes: + - name: delivery + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "keyVaultName" + keyvaultobjectnames: CosmosDB-Key;CosmosDB-Endpoint;Redis-Endpoint;Redis-AccessKey + keyvaultobjecttypes: secret;secret;secret;secret + keyvaultobjectversions: "" + resourcegroup: "keyVaultResourceGroup" + subscriptionid: "keyVaultSubscriptionId" + tenantid: "keyVaultTenantId" \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs new file mode 100644 index 00000000..19c88803 --- /dev/null +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.Azure.KeyVault; +using Microsoft.Azure.Services.AppAuthentication; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration.AzureKeyVault; + +namespace Fabrikam.DroneDelivery.Common +{ + public static class ConfigurationBuilderExtensions + { + public static IConfigurationBuilder AddAzureKeyVault(this IConfigurationBuilder configurationBuilder, string vault) + { + var tokenProvider = new AzureServiceTokenProvider(); + var keyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(tokenProvider.KeyVaultTokenCallback)); + + return configurationBuilder.AddAzureKeyVault(vault, keyVaultClient, new DefaultKeyVaultSecretManager()); + } + } +} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj index a86b0660..2a0abb6f 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj @@ -6,7 +6,8 @@ + - \ No newline at end of file + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index a6b935d0..2c9982fd 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs index 9af68a79..b9909ee2 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs @@ -18,6 +18,7 @@ public static void Main(string[] args) public static IWebHost BuildWebHost(string[] args) => WebHost.CreateDefaultBuilder(args) .UseStartup() + .UseUrls("http://0.0.0.0:8080") .Build(); } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index ca017da8..42c140e1 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -28,6 +28,14 @@ public Startup(IHostingEnvironment env) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) .AddEnvironmentVariables(); + + var buildConfig = builder.Build(); + + if(buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) + { + builder.AddAzureKeyVault(keyVaultUri); + } + Configuration = builder.Build(); } From 361ef5f202075cc50ab9477c4779b19dfb3a6784 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 26 Dec 2018 19:27:16 +0000 Subject: [PATCH 039/246] Merged PR 588: cleanup phase 1 cleanup phase 1 - get the repo aligned with the RI we are targeting in this new RA. removes the following services: - scheduler - mocks (except dronescheduler) update main readme to use new namespaces and remove some other related stuff. Related work items: #7953 --- deployment.md | 137 +++------- k8s/account.yaml | 76 ------ k8s/scheduler.yaml | 102 ------- k8s/thirdparty.yaml | 76 ------ .../Controllers/AccountController.cs | 32 --- .../delivery/MockAccountService/Dockerfile | 6 - .../delivery/MockAccountService/Program.cs | 24 -- .../delivery/MockAccountService/Startup.cs | 72 ----- .../MockAccountService/appsettings.json | 14 - .../MockDeliveryScheduler/.dockerignore | 3 - .../Controllers/HomeController.cs | 75 ------ .../delivery/MockDeliveryScheduler/Dockerfile | 6 - .../MockDeliveryScheduler.csproj | 30 --- .../delivery/MockDeliveryScheduler/Program.cs | 28 -- .../delivery/MockDeliveryScheduler/Startup.cs | 74 ----- .../Views/Home/Index.cshtml | 5 - .../Views/Shared/Error.cshtml | 14 - .../Views/Shared/_Layout.cshtml | 12 - .../Shared/_ValidationScriptsPartial.cshtml | 18 -- .../Views/_ViewImports.cshtml | 2 - .../Views/_ViewStart.cshtml | 3 - .../appsettings.Development.json | 10 - .../MockDeliveryScheduler/wwwroot/favicon.ico | Bin 32038 -> 0 bytes .../delivery/MockDroneScheduler/.dockerignore | 3 - .../delivery/MockDroneScheduler/Dockerfile | 6 - .../MockDroneScheduler.csproj | 27 -- .../appsettings.Development.json | 10 - .../MockDroneScheduler/appsettings.json | 14 - .../MockThirdPartyService/.dockerignore | 3 - .../ThirdPartyDeliveriesController.cs | 31 --- .../delivery/MockThirdPartyService/Dockerfile | 6 - .../MockThirdPartyService.csproj | 27 -- .../delivery/MockThirdPartyService/Program.cs | 28 -- .../delivery/MockThirdPartyService/Startup.cs | 72 ----- .../appsettings.Development.json | 10 - .../MockThirdPartyService/appsettings.json | 14 - src/shipping/deliveryhistory/Model.csx | 184 ------------- src/shipping/deliveryhistory/function.json | 24 -- .../deliveryhistory/historydocument.csx | 17 -- src/shipping/deliveryhistory/project.json | 9 - src/shipping/deliveryhistory/run.csx | 23 -- src/shipping/dronescheduler/Dockerfile | 49 ++++ .../.dockerignore | 0 .../Controllers/DroneDeliveriesController.cs | 0 ...rikam.DroneDelivery.DroneScheduler.csproj} | 0 .../Models/DroneDelivery.cs | 0 .../Models/PackageDetail.cs | 0 .../Models/PackageSize.cs | 0 .../Program.cs | 0 .../Startup.cs | 0 .../appsettings.Development.json | 0 .../appsettings.json | 0 src/shipping/dronescheduler/scripts/run.sh | 2 + src/shipping/scheduler/.classpath | 25 -- src/shipping/scheduler/.gitignore | 24 -- src/shipping/scheduler/.project | 29 -- src/shipping/scheduler/Dockerfile | 12 - src/shipping/scheduler/Dockerfilemaven | 25 -- src/shipping/scheduler/Readme.txt | 7 - src/shipping/scheduler/conf/Config.properties | 14 - src/shipping/scheduler/conf/application.conf | 85 ------ src/shipping/scheduler/conf/log4j2.xml | 14 - src/shipping/scheduler/mvn-entrypoint.sh | 41 --- src/shipping/scheduler/pom.xml | 203 -------------- src/shipping/scheduler/settings-docker.xml | 6 - .../akkareader/AkkaDelivery.java | 29 -- .../deliveryscheduler/akkareader/Main.java | 114 -------- .../akkareader/ReactiveStreamingApp.java | 10 - .../DeliveryRequestEventProcessor.java | 254 ------------------ .../scheduler/SchedulerSettings.java | 23 -- .../scheduler/ServiceFailureMetadata.java | 5 - .../scheduler/ServiceName.java | 5 - .../compensation/RetryableDelivery.java | 45 ---- .../StorageQueueClientFactory.java | 27 -- .../compensation/tests/StorageQueueTest.java | 55 ---- .../models/invoker/ConfirmationType.java | 9 - .../models/invoker/DeliverySchedule.java | 83 ------ .../models/invoker/DroneDelivery.java | 55 ---- .../scheduler/models/invoker/Location.java | 45 ---- .../models/invoker/PackageDetail.java | 31 --- .../scheduler/models/invoker/PackageGen.java | 45 ---- .../scheduler/models/invoker/PackageSize.java | 5 - .../scheduler/models/invoker/UserAccount.java | 37 --- .../models/receiver/ConfirmationRequired.java | 9 - .../models/receiver/ContainerSize.java | 5 - .../scheduler/models/receiver/Delivery.java | 97 ------- .../models/receiver/PackageInfo.java | 45 ---- .../services/AccountServiceCallerImpl.java | 63 ----- .../BackendServiceCallFailedException.java | 17 -- .../services/DeliveryServiceCallerImpl.java | 107 -------- .../DroneSchedulerServiceCallerImpl.java | 99 ------- .../services/PackageServiceCallerImpl.java | 113 -------- .../scheduler/services/ServiceCaller.java | 11 - .../scheduler/services/ServiceCallerImpl.java | 146 ---------- .../ServiceCallerResponseErrorHandler.java | 35 --- .../services/ThirdPartyServiceCallerImpl.java | 84 ------ .../tests/InvokeMockBackendServicesTest.java | 157 ----------- .../services/tests/ModelsAndUtilsTest.java | 86 ------ .../services/tests/PostDeliveryRequests.java | 55 ---- .../services/tests/helpers/ModelsUtils.java | 140 ---------- .../scheduler/utils/ConfigReader.java | 43 --- .../utils/CustomDateTimeDeserializer.java | 28 -- .../utils/IdleConnectionMonitorThread.java | 43 --- .../scheduler/utils/KeepAliveStrategy.java | 36 --- .../scheduler/utils/LocationRandomizer.java | 19 -- .../scheduler/utils/ModelsConverter.java | 40 --- 106 files changed, 84 insertions(+), 4134 deletions(-) delete mode 100644 k8s/account.yaml delete mode 100644 k8s/scheduler.yaml delete mode 100644 k8s/thirdparty.yaml delete mode 100644 src/shipping/delivery/MockAccountService/Controllers/AccountController.cs delete mode 100644 src/shipping/delivery/MockAccountService/Dockerfile delete mode 100644 src/shipping/delivery/MockAccountService/Program.cs delete mode 100644 src/shipping/delivery/MockAccountService/Startup.cs delete mode 100644 src/shipping/delivery/MockAccountService/appsettings.json delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/.dockerignore delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Dockerfile delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Program.cs delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Startup.cs delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json delete mode 100644 src/shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico delete mode 100644 src/shipping/delivery/MockDroneScheduler/.dockerignore delete mode 100644 src/shipping/delivery/MockDroneScheduler/Dockerfile delete mode 100644 src/shipping/delivery/MockDroneScheduler/MockDroneScheduler.csproj delete mode 100644 src/shipping/delivery/MockDroneScheduler/appsettings.Development.json delete mode 100644 src/shipping/delivery/MockDroneScheduler/appsettings.json delete mode 100644 src/shipping/delivery/MockThirdPartyService/.dockerignore delete mode 100644 src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs delete mode 100644 src/shipping/delivery/MockThirdPartyService/Dockerfile delete mode 100644 src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj delete mode 100644 src/shipping/delivery/MockThirdPartyService/Program.cs delete mode 100644 src/shipping/delivery/MockThirdPartyService/Startup.cs delete mode 100644 src/shipping/delivery/MockThirdPartyService/appsettings.Development.json delete mode 100644 src/shipping/delivery/MockThirdPartyService/appsettings.json delete mode 100644 src/shipping/deliveryhistory/Model.csx delete mode 100644 src/shipping/deliveryhistory/function.json delete mode 100644 src/shipping/deliveryhistory/historydocument.csx delete mode 100644 src/shipping/deliveryhistory/project.json delete mode 100644 src/shipping/deliveryhistory/run.csx create mode 100644 src/shipping/dronescheduler/Dockerfile rename src/shipping/{delivery/MockAccountService => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/.dockerignore (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Controllers/DroneDeliveriesController.cs (100%) rename src/shipping/{delivery/MockAccountService/MockAccountService.csproj => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj} (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Models/DroneDelivery.cs (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Models/PackageDetail.cs (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Models/PackageSize.cs (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Program.cs (100%) rename src/shipping/{delivery/MockDroneScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/Startup.cs (100%) rename src/shipping/{delivery/MockAccountService => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/appsettings.Development.json (100%) rename src/shipping/{delivery/MockDeliveryScheduler => dronescheduler/Fabrikam.DroneDelivery.DroneScheduler}/appsettings.json (100%) create mode 100644 src/shipping/dronescheduler/scripts/run.sh delete mode 100644 src/shipping/scheduler/.classpath delete mode 100644 src/shipping/scheduler/.gitignore delete mode 100644 src/shipping/scheduler/.project delete mode 100644 src/shipping/scheduler/Dockerfile delete mode 100644 src/shipping/scheduler/Dockerfilemaven delete mode 100644 src/shipping/scheduler/Readme.txt delete mode 100644 src/shipping/scheduler/conf/Config.properties delete mode 100644 src/shipping/scheduler/conf/application.conf delete mode 100644 src/shipping/scheduler/conf/log4j2.xml delete mode 100644 src/shipping/scheduler/mvn-entrypoint.sh delete mode 100644 src/shipping/scheduler/pom.xml delete mode 100644 src/shipping/scheduler/settings-docker.xml delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java delete mode 100644 src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java diff --git a/deployment.md b/deployment.md index 323f565b..212e3a80 100644 --- a/deployment.md +++ b/deployment.md @@ -5,7 +5,10 @@ - Azure subscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) -- [Docker Compose](https://docs.docker.com/compose/install/) + +> Note: in linux systems, it is possible to run the docker command without prefacing +> with sudo. For more information, please refer to [the Post-installation steps +> for linux](https://docs.docker.com/install/linux/linux-postinstall/) Clone or download this repo locally. @@ -54,11 +57,9 @@ az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Get the cluster's principal export CLUSTER_SERVICE_PRINCIPAL=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query servicePrincipalProfile.clientId --output tsv) -# Create the BC namespaces -kubectl create namespace shipping && \ -kubectl create namespace accounts && \ -kubectl create namespace dronemgmt && \ -kubectl create namespace 3rdparty +# Create namespaces +kubectl create namespace backend && \ +kubectl create namespace frontend ``` Create an Azure Container Registry instance. @@ -130,18 +131,17 @@ Build the Delivery service ```bash export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery -docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up ``` Build and publish the container image ```bash # Build the Docker image -docker build --pull --compress -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/. +docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/. # Push the image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 +docker push $ACR_SERVER/delivery:0.1.0 ``` Create Kubernetes secrets @@ -153,7 +153,7 @@ export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOU export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) -kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ +kubectl --namespace backend create --save-config=true secret generic delivery-storageconf \ --from-literal=CosmosDB_Key=${COSMOSDB_KEY} \ --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT} \ --from-literal=Redis_Endpoint=${REDIS_ENDPOINT} \ @@ -204,7 +204,7 @@ Deploy the Delivery service: ```bash # Update the image tag and config values in the deployment YAML -sed "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" $K8S/delivery.yaml | \ +sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ sed "s/value: \"EH_EntityPath\"/value:/g" | \ @@ -215,10 +215,10 @@ sed "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $DELIVERY_KEYVAULT_NAME#g" > $K8S/delivery-0.yaml # Deploy the service -kubectl --namespace shipping apply -f $K8S/delivery-0.yaml +kubectl --namespace backend apply -f $K8S/delivery-0.yaml # Verify the pod is created -kubectl get pods -n shipping +kubectl get pods -n backend ``` ## Deploy the Package service @@ -235,26 +235,23 @@ Build the Package service ```bash export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package -# Build the app -docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up - # Build the docker image -sudo docker build -f $PACKAGE_PATH/build/prod.dockerfile -t $ACR_SERVER/package-service:0.1.0 $PACKAGE_PATH +docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH # Push the docker image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/package-service:0.1.0 +docker push $ACR_SERVER/package:0.1.0 ``` Deploy the Package service ```bash # Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml +sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml # Create secret export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv) -kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION +kubectl -n backend create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION # Create KeyVault secret export PACKAGE_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-package-kv" @@ -262,10 +259,10 @@ az keyvault create --name $PACKAGE_KEYVAULT_NAME --resource-group $RESOURCE_GROU az keyvault secret set --vault-name $PACKAGE_KEYVAULT_NAME --name mongodb-pwd --value $COSMOSDB_CONNECTION # Deploy service -kubectl --namespace shipping apply -f $K8S/package-0.yml +kubectl --namespace backend apply -f $K8S/package-0.yml # Verify the pod is created -kubectl get pods -n shipping +kubectl get pods -n backend ``` ## Deploy the Ingestion service @@ -328,7 +325,7 @@ Deploy the Ingestion service sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml # Create secret -kubectl -n shipping create secret generic ingestion-secrets \ +kubectl -n backend create secret generic ingestion-secrets \ --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ --from-literal=eventhub_name=${INGESTION_EH_NAME} \ --from-literal=eventhub_keyname=IngestionServiceAccessKey \ @@ -343,110 +340,42 @@ az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-key az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyvalue --value ${EH_ACCESS_KEY_VALUE} # Deploy service -kubectl --namespace shipping apply -f $K8S/ingestion-0.yaml +kubectl --namespace backend apply -f $K8S/ingestion-0.yaml # Verify the pod is created -kubectl get pods -n shipping -``` - -## Deploy the Scheduler service - -Provision Azure resources -```bash -export SCHEDULER_STORAGE_ACCOUNT_NAME=[SCHEDULER_STORAGE_ACCOUNT_NAME_HERE] - -az storage account create --resource-group $RESOURCE_GROUP --name $SCHEDULER_STORAGE_ACCOUNT_NAME --sku Standard_LRS -``` - -Build the Scheduler service - -```bash -export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler - -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH -docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk - -# Build the docker image -docker build -f $SCHEDULER_PATH/Dockerfile -t $ACR_SERVER/scheduler:0.1.0 $SCHEDULER_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/scheduler:0.1.0 -``` - -Deploy the Scheduler service - -```bash -# Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" $K8S/scheduler.yaml > $K8S/scheduler-0.yaml - -export EH_CONNECTION_STRING=$(az eventhubs eventhub authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_EH_NS --name IngestionServiceAccessKey --eventhub-name $INGESTION_EH_NAME --query primaryConnectionString -o tsv) - -export STORAGE_ACCOUNT_CONNECTION_STRING=$(az storage account show-connection-string \ - --name $SCHEDULER_STORAGE_ACCOUNT_NAME \ - --resource-group $RESOURCE_GROUP \ - --output tsv) - -export STORAGE_ACCOUNT_ACCESS_KEY=$(az storage account keys list \ - --account-name $SCHEDULER_STORAGE_ACCOUNT_NAME \ - --resource-group $RESOURCE_GROUP \ - --query "[0].value" \ - --output tsv) - -# Create secrets -kubectl -n shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ ---from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ ---from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ ---from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} - -# Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler-0.yaml - -# Verify all pods are created -kubectl get pods -n shipping +kubectl get pods -n backend ``` -## Deploy mock services +## Deploy DroneScheduler service -Build the mock services +Build the dronescheduler services ```bash -export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery -docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up +export DRONE_PATH=microservices-reference-implementation/src/shipping/dronescheduler ``` Build and publish the container image ```bash # Build the Docker images -docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ -docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ -docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. +docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ../$DRONE_PATH # Push the images to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/account:0.1.0 && \ -docker push $ACR_SERVER/dronescheduler:0.1.0 && \ -docker push $ACR_SERVER/thirdparty:0.1.0 +docker push $ACR_SERVER/dronescheduler:0.1.0 ``` -Deploy the mock services: +Deploy the dronescheduler services: ```bash # Update the image tag in the deployment YAML -sed "s#image:#image: $ACR_SERVER/account:0.1.0#g" $K8S/account.yaml > $K8S/account-0.yaml && \ -sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" $K8S/dronescheduler.yaml > $K8S/dronescheduler-0.yaml && \ -sed "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" $K8S/thirdparty.yaml > $K8S/thirdparty-0.yaml +sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" $K8S/dronescheduler.yaml > $K8S/dronescheduler-0.yaml # Deploy the service -kubectl --namespace accounts apply -f $K8S/account-0.yaml && \ -kubectl --namespace dronemgmt apply -f $K8S/dronescheduler-0.yaml && \ -kubectl --namespace 3rdparty apply -f $K8S/thirdparty-0.yaml +kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml ## Verify all services are running: -kubectl get all --all-namespaces -l co=fabrikam +kubectl get pods -n backend ``` ## Deploy linkerd @@ -457,10 +386,10 @@ kubectl create ns linkerd wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ sed -i "s#/default#/shipping#g" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.backend ;" servicemesh.yml && \ sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ +sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.backend ;" servicemesh.yml && \ sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ kubectl apply -f servicemesh.yml ``` diff --git a/k8s/account.yaml b/k8s/account.yaml deleted file mode 100644 index 0274b3b9..00000000 --- a/k8s/account.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Account service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: account - labels: - app: account - bc: accounts - co: fabrikam -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app: account - clusterIP: None ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: account - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: account - version: 0.1.0 - bc: accounts - co: fabrikam -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: account - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: account - version: 0.1.0 - bc: accounts - co: fabrikam - annotations: - team: accountservice - spec: - containers: - - name: account - image: - env: - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: CORRELATION_HEADER - value: l5d-ctx-trace - ports: - - name: service - containerPort: 80 - resources: - requests: - cpu: 1 - memory: 1G - limits: - cpu: 1 - memory: 1G diff --git a/k8s/scheduler.yaml b/k8s/scheduler.yaml deleted file mode 100644 index 1d42794f..00000000 --- a/k8s/scheduler.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Scheduler service -################################################################################################### -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: scheduler - labels: - app: scheduler - bc: shipping - co: fabrikam -spec: - serviceName: scheduler - replicas: 4 - selector: - matchLabels: - app: scheduler - template: - metadata: - labels: - app: scheduler - version: 0.1.0 - bc: shipping - co: fabrikam - spec: - containers: - - image: - name: scheduler - imagePullPolicy: IfNotPresent - env: - - name: HOST_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: SERVICE_URI_ACCOUNT - value: http://account/api/Account - - name: SERVICE_URI_DELIVERY - value: http://delivery/api/Deliveries - - name: SERVICE_URI_DRONE - value: http://dronescheduler/api/DroneDeliveries - - name: SERVICE_URI_PACKAGE - value: http://package/api/packages - - name: SERVICE_URI_THIRDPARTY - value: http://thirdparty/api/ThirdPartyDeliveries - ## set this env to read from period of time - ## otherwise is from last known checkpoint - - name: CHECKPOINT_TIME_MINUTES - value: "0" - - name: IOTHUB_EVENTHUB_NAME - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: eventhub_name - - name: IOTHUB_EVENTHUB_ENDPOINT - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: eventhub_sas_connection_string - - name: IOTHUB_ACCESS_CONNSTRING - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: eventhub_sas_connection_string - - name: IOTHUB_CHECKPOINT_AZSTORAGE_ACCOUNT - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: storageaccount_name - - name: IOTHUB_CHECKPOINT_AZSTORAGE_KEY - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: storageaccount_key - - name: STORAGE_QUEUE_NAME - value: "compensationqueue" - - name: STORAGE_QUEUE_CONNECTION_STRING - valueFrom: - secretKeyRef: - name: scheduler-secrets - key: queueconstring - - name: IOTHUB_EVENTHUB_PARTITIONS - value: "4" - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: SERVICEMESH_HEADERS - value: l5d-ctx-deadline,l5d-ctx-trace - - name: SERVICEMESH_CORRELATION_HEADER - value: l5d-ctx-trace diff --git a/k8s/thirdparty.yaml b/k8s/thirdparty.yaml deleted file mode 100644 index 45933415..00000000 --- a/k8s/thirdparty.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ThirdParty service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: thirdparty - labels: - app: thirdparty - bc: 3rdparty - co: fabrikam -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app: thirdparty - clusterIP: None ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: thirdparty - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: thirdparty - version: 0.1.0 - bc: 3rdparty - co: fabrikam -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: thirdparty - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: thirdparty - bc: 3rdparty - version: 0.1.0 - co: fabrikam - annotations: - team: thirdpartyservice - spec: - containers: - - name: thirdparty - image: - env: - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: CORRELATION_HEADER - value: l5d-ctx-trace - ports: - - name: service - containerPort: 80 - resources: - requests: - cpu: 1 - memory: 1G - limits: - cpu: 1 - memory: 1G diff --git a/src/shipping/delivery/MockAccountService/Controllers/AccountController.cs b/src/shipping/delivery/MockAccountService/Controllers/AccountController.cs deleted file mode 100644 index d85c3bf6..00000000 --- a/src/shipping/delivery/MockAccountService/Controllers/AccountController.cs +++ /dev/null @@ -1,32 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; - -namespace MockAccountService.Controllers -{ - [Route("api/[controller]")] - public class AccountController : Controller - { - private readonly ILogger logger; - - public AccountController(ILoggerFactory loggerFactory) - { - this.logger = loggerFactory.CreateLogger(); - } - - // GET api/account/{accountId} - [HttpGet("{accountId}")] - //TODO: [Authorize] - public bool Get(string accountId) - { - logger.LogInformation("In Get action with accountId: {AccountId}", accountId); - - return true; - } - } -} diff --git a/src/shipping/delivery/MockAccountService/Dockerfile b/src/shipping/delivery/MockAccountService/Dockerfile deleted file mode 100644 index b6096abe..00000000 --- a/src/shipping/delivery/MockAccountService/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM microsoft/aspnetcore:2.0 -ARG source -WORKDIR /app -EXPOSE 80 -COPY ${source:-obj/Docker/publish} . -ENTRYPOINT ["dotnet", "MockAccountService.dll"] diff --git a/src/shipping/delivery/MockAccountService/Program.cs b/src/shipping/delivery/MockAccountService/Program.cs deleted file mode 100644 index 16c24515..00000000 --- a/src/shipping/delivery/MockAccountService/Program.cs +++ /dev/null @@ -1,24 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System.IO; -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; - -namespace MockAccountService -{ - public class Program - { - public static void Main(string[] args) - { - BuildWebHost(args).Run(); - } - - public static IWebHost BuildWebHost(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup() - .Build(); - } -} diff --git a/src/shipping/delivery/MockAccountService/Startup.cs b/src/shipping/delivery/MockAccountService/Startup.cs deleted file mode 100644 index 0de176b0..00000000 --- a/src/shipping/delivery/MockAccountService/Startup.cs +++ /dev/null @@ -1,72 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Swashbuckle.AspNetCore.Swagger; -using Serilog; -using Serilog.Formatting.Compact; -using Fabrikam.DroneDelivery.Common; - -namespace MockAccountService -{ - public class Startup - { - public Startup(IHostingEnvironment env) - { - var builder = new ConfigurationBuilder() - .SetBasePath(env.ContentRootPath) - .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) - .AddEnvironmentVariables(); - Configuration = builder.Build(); - } - - public IConfigurationRoot Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. - public void ConfigureServices(IServiceCollection services) - { - services.AddSingleton(); - - services.AddLogging(loggingBuilder => - loggingBuilder.AddSerilog(dispose: true)); - - // Add framework services. - services.AddMvc(); - - // Register the Swagger generator, defining one or more Swagger documents - services.AddSwaggerGen(c => - { - c.SwaggerDoc("v1", new Info { Title = "Mock AccountService API", Version = "v1" }); - }); - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) - { - Log.Logger = new LoggerConfiguration() - .WriteTo.Console(new CompactJsonFormatter()) - .ReadFrom.Configuration(Configuration) - .Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) - .CreateLogger(); - - app.UseMvc(); - - // Enable middleware to serve generated Swagger as a JSON endpoint. - app.UseSwagger(); - - // Enable middleware to serve swagger-ui (HTML, JS, CSS etc.), specifying the Swagger JSON endpoint. - app.UseSwaggerUI(c => - { - c.SwaggerEndpoint("/swagger/v1/swagger.json", "Mock AccountService API V1"); - }); - } - } -} diff --git a/src/shipping/delivery/MockAccountService/appsettings.json b/src/shipping/delivery/MockAccountService/appsettings.json deleted file mode 100644 index 5d345bf0..00000000 --- a/src/shipping/delivery/MockAccountService/appsettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" - }, - "Serilog": { - "MinimumLevel": "Verbose", - "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], - "WriteTo": [] - } -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/.dockerignore b/src/shipping/delivery/MockDeliveryScheduler/.dockerignore deleted file mode 100644 index d8f8175f..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!obj/Docker/publish/* -!obj/Docker/empty/ diff --git a/src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs b/src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs deleted file mode 100644 index dfb37d69..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Controllers/HomeController.cs +++ /dev/null @@ -1,75 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Configuration; -using System.Text; -using System; -using Microsoft.Extensions.Logging; - -namespace MockDeliveryScheduler.Controllers -{ - public class HomeController : Controller - { - private readonly IConfigurationRoot _configuration; - private readonly ILogger _logger; - - public HomeController(IConfigurationRoot configuration, ILoggerFactory loggerFactory) - { - _configuration = configuration; - _logger = loggerFactory.CreateLogger(); - } - - public async Task Index() - { - _logger.LogInformation("In Index action!!!"); - - var guid = Guid.NewGuid(); - var client = new HttpClient(); - client.DefaultRequestHeaders.Add("l5d-ctx-trace", $"TestCorrelationId{guid}"); - - var deliveryId = $"deliveryId{guid}"; - var droneDelivery = @"{""deliveryId"": """ + deliveryId + @""", ""pickup"": {""altitude"": 1, ""latitude"": 2, ""longitude"": 3}, ""dropoff"": {""altitude"": 3, ""latitude"": 2, ""longitude"": 1}, ""packageDetails"": [{""id"": ""packageId"", ""size"": 0}], ""expedited"": true}"; - var response = await client.PutAsync($"http://dronescheduler/api/dronedeliveries/{deliveryId}" , new StringContent(droneDelivery, Encoding.UTF8, "application/json")); - var droneId = await response.Content.ReadAsStringAsync(); - ViewBag.Message1 = droneId; - - var delivery = @"{ - ""id"": """ + deliveryId + @""", - ""owner"": { - ""userId"": ""user123"", - ""accountId"": ""account123"" - }, - ""pickup"": { - ""altitude"": 1, - ""latitude"": 2, - ""longitude"": 3 - }, - ""dropoff"": { - ""altitude"": 3, - ""latitude"": 2, - ""longitude"": 1 - }, - ""packageIds"": [ - ""packageId1"", ""packageId2"" - ], - ""deadline"": ""deadline"", - ""expedited"": true, - ""droneId"": """ + droneId + @""" - }"; - response = await client.PutAsync($"http://delivery/api/deliveries/{deliveryId}", new StringContent(delivery, Encoding.UTF8, "application/json")); - ViewBag.Message2 = await response.Content.ReadAsStringAsync(); - - return View(); - } - - public IActionResult Error() - { - return View(); - } - } -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/Dockerfile b/src/shipping/delivery/MockDeliveryScheduler/Dockerfile deleted file mode 100644 index 9f8c901a..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM microsoft/aspnetcore:2.0 -ARG source -WORKDIR /app -EXPOSE 80 -COPY ${source:-obj/Docker/publish} . -ENTRYPOINT ["dotnet", "MockDeliveryScheduler.dll"] diff --git a/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj b/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj deleted file mode 100644 index 2f790fa5..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/MockDeliveryScheduler.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - - netcoreapp2.0 - - - - ..\docker-compose.dcproj - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/shipping/delivery/MockDeliveryScheduler/Program.cs b/src/shipping/delivery/MockDeliveryScheduler/Program.cs deleted file mode 100644 index 94d3abc7..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Program.cs +++ /dev/null @@ -1,28 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; - -namespace MockDeliveryScheduler -{ - public class Program - { - public static void Main(string[] args) - { - BuildWebHost(args).Run(); - } - - public static IWebHost BuildWebHost(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup() - .Build(); - } -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/Startup.cs b/src/shipping/delivery/MockDeliveryScheduler/Startup.cs deleted file mode 100644 index 0143adc9..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Startup.cs +++ /dev/null @@ -1,74 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.AspNetCore.Http; -using Serilog; -using Serilog.Formatting.Compact; -using Fabrikam.DroneDelivery.Common; - -namespace MockDeliveryScheduler -{ - public class Startup - { - public Startup(IHostingEnvironment env) - { - var builder = new ConfigurationBuilder() - .SetBasePath(env.ContentRootPath) - .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) - .AddEnvironmentVariables(); - Configuration = builder.Build(); - } - - public IConfigurationRoot Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. - public void ConfigureServices(IServiceCollection services) - { - services.AddSingleton(); - - services.AddLogging(loggingBuilder => - loggingBuilder.AddSerilog(dispose: true)); - - // Add framework services. - services.AddMvc(); - services.AddSingleton(Configuration); - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) - { - Log.Logger = new LoggerConfiguration() - .WriteTo.Console(new CompactJsonFormatter()) - .ReadFrom.Configuration(Configuration) - .Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) - .CreateLogger(); - - if (env.IsDevelopment()) - { - app.UseDeveloperExceptionPage(); - app.UseBrowserLink(); - } - else - { - app.UseExceptionHandler("/Home/Error"); - } - - app.UseStaticFiles(); - - app.UseMvc(routes => - { - routes.MapRoute( - name: "default", - template: "{controller=Home}/{action=Index}/{id?}"); - }); - } - } -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml deleted file mode 100644 index f148e52b..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/Home/Index.cshtml +++ /dev/null @@ -1,5 +0,0 @@ -@{ - ViewData["Title"] = "Home Page"; -} -

DroneScheduler PUT: @ViewBag.Message1

-

DeliveryService PUT: @ViewBag.Message2

\ No newline at end of file diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml deleted file mode 100644 index e514139c..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/Error.cshtml +++ /dev/null @@ -1,14 +0,0 @@ -@{ - ViewData["Title"] = "Error"; -} - -

Error.

-

An error occurred while processing your request.

- -

Development Mode

-

- Swapping to Development environment will display more detailed information about the error that occurred. -

-

- Development environment should not be enabled in deployed applications, as it can result in sensitive information from exceptions being displayed to end users. For local debugging, development environment can be enabled by setting the ASPNETCORE_ENVIRONMENT environment variable to Development, and restarting the application. -

diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml deleted file mode 100644 index 1e4ac33b..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_Layout.cshtml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - @ViewData["Title"] - MockDeliveryScheduler - - - - @RenderBody() - - diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml deleted file mode 100644 index 27e0ea7c..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/Shared/_ValidationScriptsPartial.cshtml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml deleted file mode 100644 index 7c55c2e3..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewImports.cshtml +++ /dev/null @@ -1,2 +0,0 @@ -@using MockDeliveryScheduler -@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers diff --git a/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml b/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml deleted file mode 100644 index a5f10045..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/Views/_ViewStart.cshtml +++ /dev/null @@ -1,3 +0,0 @@ -@{ - Layout = "_Layout"; -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json b/src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json deleted file mode 100644 index fa8ce71a..00000000 --- a/src/shipping/delivery/MockDeliveryScheduler/appsettings.Development.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - } -} diff --git a/src/shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico b/src/shipping/delivery/MockDeliveryScheduler/wwwroot/favicon.ico deleted file mode 100644 index a3a799985c43bc7309d701b2cad129023377dc71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32038 zcmeHwX>eTEbtY7aYbrGrkNjgie?1jXjZ#zP%3n{}GObKv$BxI7Sl;Bwl5E+Qtj&t8 z*p|m4DO#HoJC-FyvNnp8NP<{Na0LMnTtO21(rBP}?EAiNjWgeO?z`{3ZoURUQlV2d zY1Pqv{m|X_oO91|?^z!6@@~od!@OH>&BN;>c@O+yUfy5w>LccTKJJ&`-k<%M^Zvi( z<$dKp=jCnNX5Qa+M_%6g|IEv~4R84q9|7E=|Ho(Wz3f-0wPjaRL;W*N^>q%^KGRr7 zxbjSORb_c&eO;oV_DZ7ua!sPH=0c+W;`vzJ#j~-x3uj};50#vqo*0w4!LUqs*UCh9 zvy2S%$#8$K4EOa&e@~aBS65_hc~Mpu=454VT2^KzWqEpBA=ME|O;1cn?8p<+{MKJf zbK#@1wzL44m$k(?85=Obido7=C|xWKe%66$z)NrzRwR>?hK?_bbwT z@Da?lBrBL}Zemo1@!9pYRau&!ld17h{f+UV0sY(R{ET$PBB|-=Nr@l-nY6w8HEAw* zRMIQU`24Jl_IFEPcS=_HdrOP5yf81z_?@M>83Vv65$QFr9nPg(wr`Ke8 zaY4ogdnMA*F7a4Q1_uXadTLUpCk;$ZPRRJ^sMOch;rlbvUGc1R9=u;dr9YANbQ<4Z z#P|Cp9BP$FXNPolgyr1XGt$^lFPF}rmBF5rj1Kh5%dforrP8W}_qJL$2qMBS-#%-|s#BPZBSETsn_EBYcr(W5dq( z@f%}C|iN7)YN`^)h7R?Cg}Do*w-!zwZb9=BMp%Wsh@nb22hA zA{`wa8Q;yz6S)zfo%sl08^GF`9csI9BlGnEy#0^Y3b);M+n<(}6jziM7nhe57a1rj zC@(2ISYBL^UtWChKzVWgf%4LW2Tqg_^7jMw`C$KvU+mcakFjV(BGAW9g%CzSyM;Df z143=mq0oxaK-H;o>F3~zJ<(3-j&?|QBn)WJfP#JR zRuA;`N?L83wQt78QIA$(Z)lGQY9r^SFal;LB^qi`8%8@y+mwcGsf~nv)bBy2S7z~9 z=;X@Gglk)^jpbNz?1;`!J3QUfAOp4U$Uxm5>92iT`mek#$>s`)M>;e4{#%HAAcb^8_Ax%ersk|}# z0bd;ZPu|2}18KtvmIo8`1@H~@2ejwo(5rFS`Z4&O{$$+ch2hC0=06Jh`@p+p8LZzY z&2M~8T6X^*X?yQ$3N5EzRv$(FtSxhW>>ABUyp!{484f8(%C1_y)3D%Qgfl_!sz`LTXOjR&L!zPA0qH_iNS!tY{!^2WfD%uT}P zI<~&?@&))5&hPPHVRl9);TPO>@UI2d!^ksb!$9T96V(F){puTsn(}qt_WXNw4VvHj zf;6A_XCvE`Z@}E-IOaG0rs>K>^=Sr&OgT_p;F@v0VCN0Y$r|Lw1?Wjt`AKK~RT*kJ z2>QPuVgLNcF+XKno;WBv$yj@d_WFJbl*#*V_Cwzo@%3n5%z4g21G*PVZ)wM5$A{klYozmGlB zT@u2+s}=f}25%IA!yNcXUr!!1)z(Nqbhojg0lv@7@0UlvUMT)*r;M$d0-t)Z?B1@qQk()o!4fqvfr_I0r7 zy1(NdkHEj#Yu{K>T#We#b#FD=c1XhS{hdTh9+8gy-vkcdkk*QS@y(xxEMb1w6z<^~ zYcETGfB#ibR#ql0EiD;PR$L&Vrh2uRv5t_$;NxC;>7_S5_OXxsi8udY3BUUdi55Sk zcyKM+PQ9YMA%D1kH1q48OFG(Gbl=FmV;yk8o>k%0$rJ8%-IYsHclnYuTskkaiCGkUlkMY~mx&K}XRlKIW;odWIeuKjtbc^8bBOTqK zjj(ot`_j?A6y_h%vxE9o*ntx#PGrnK7AljD_r58ylE*oy@{IY%+mA^!|2vW_`>`aC{#3`#3;D_$^S^cM zRcF+uTO2sICledvFgNMU@A%M)%8JbSLq{dD|2|2Sg8vvh_uV6*Q?F&rKaV{v_qz&y z`f;stIb?Cb2!Cg7CG91Bhu@D@RaIrq-+o+T2fwFu#|j>lD6ZS9-t^5cx>p|?flqUA z;Cgs#V)O#`Aw4$Kr)L5?|7f4izl!;n0jux}tEW$&&YBXz9o{+~HhoiYDJ`w5BVTl&ARya=M7zdy$FEe}iGBur8XE>rhLj&_yDk5D4n2GJZ07u7%zyAfNtOLn;)M?h*Py-Xtql5aJOtL4U8e|!t? z((sc6&OJXrPdVef^wZV&x=Z&~uA7^ix8rly^rEj?#d&~pQ{HN8Yq|fZ#*bXn-26P^ z5!)xRzYO9{u6vx5@q_{FE4#7BipS#{&J7*>y}lTyV94}dfE%Yk>@@pDe&F7J09(-0|wuI|$of-MRfK51#t@t2+U|*s=W; z!Y&t{dS%!4VEEi$efA!#<<7&04?kB}Soprd8*jYv;-Qj~h~4v>{XX~kjF+@Z7<t?^|i z#>_ag2i-CRAM8Ret^rZt*^K?`G|o>1o(mLkewxyA)38k93`<~4VFI?5VB!kBh%NNU zxb8K(^-MU1ImWQxG~nFB-Un;6n{lQz_FfsW9^H$Xcn{;+W^ZcG$0qLM#eNV=vGE@# z1~k&!h4@T|IiI<47@pS|i?Qcl=XZJL#$JKve;booMqDUYY{(xcdj6STDE=n?;fsS1 ze`h~Q{CT$K{+{t+#*I1=&&-UU8M&}AwAxD-rMa=e!{0gQXP@6azBq9(ji11uJF%@5 zCvV`#*?;ZguQ7o|nH%bm*s&jLej#@B35gy32ZAE0`Pz@#j6R&kN5w{O4~1rhDoU zEBdU)%Nl?8zi|DR((u|gg~r$aLYmGMyK%FO*qLvwxK5+cn*`;O`16c!&&XT{$j~5k zXb^fbh1GT-CI*Nj{-?r7HNg=e3E{6rxuluPXY z5Nm8ktc$o4-^SO0|Es_sp!A$8GVwOX+%)cH<;=u#R#nz;7QsHl;J@a{5NUAmAHq4D zIU5@jT!h?kUp|g~iN*!>jM6K!W5ar0v~fWrSHK@})@6Lh#h)C6F6@)&-+C3(zO! z8+kV|B7LctM3DpI*~EYo>vCj>_?x&H;>y0*vKwE0?vi$CLt zfSJB##P|M2dEUDBPKW=9cY-F;L;h3Fs4E2ERdN#NSL7ctAC z?-}_a{*L@GA7JHJudxtDVA{K5Yh*k(%#x4W7w+^ zcb-+ofbT5ieG+@QG2lx&7!MyE2JWDP@$k`M;0`*d+oQmJ2A^de!3c53HFcfW_Wtv< zKghQ;*FifmI}kE4dc@1y-u;@qs|V75Z^|Q0l0?teobTE8tGl@EB?k#q_wUjypJ*R zyEI=DJ^Z+d*&}B_xoWvs27LtH7972qqMxVFcX9}c&JbeNCXUZM0`nQIkf&C}&skSt z^9fw@b^Hb)!^hE2IJq~~GktG#ZWwWG<`@V&ckVR&r=JAO4YniJewVcG`HF;59}=bf zLyz0uxf6MhuSyH#-^!ZbHxYl^mmBVrx) zyrb8sQ*qBd_WXm9c~Of$&ZP$b^)<~0%nt#7y$1Jg$e}WCK>TeUB{P>|b1FAB?%K7>;XiOfd}JQ`|IP#Vf%kVy zXa4;XFZ+>n;F>uX&3|4zqWK2u3c<>q;tzjsb1;d{u;L$-hq3qe@82(ob<3qom#%`+ z;vzYAs7TIMl_O75BXu|r`Qhc4UT*vN$3Oo0kAC!{f2#HexDy|qUpgTF;k{o6|L>7l z=?`=*LXaow1o;oNNLXsGTrvC)$R&{m=94Tf+2iTT3Y_Or z-!;^0a{kyWtO4vksG_3cyc7HQ0~detf0+2+qxq(e1NS251N}w5iTSrM)`0p8rem!j zZ56hGD=pHI*B+dd)2B`%|9f0goozCSeXPw3 z+58k~sI02Yz#lOneJzYcG)EB0|F+ggC6D|B`6}d0khAK-gz7U3EGT|M_9$ZINqZjwf>P zJCZ=ogSoE`=yV5YXrcTQZx@Un(64*AlLiyxWnCJ9I<5Nc*eK6eV1Mk}ci0*NrJ=t| zCXuJG`#7GBbPceFtFEpl{(lTm`LX=B_!H+& z>$*Hf}}y zkt@nLXFG9%v**s{z&{H4e?aqp%&l#oU8lxUxk2o%K+?aAe6jLojA& z_|J0<-%u^<;NT*%4)n2-OdqfctSl6iCHE?W_Q2zpJken#_xUJlidzs249H=b#g z?}L4-Tnp6)t_5X?_$v)vz`s9@^BME2X@w<>sKZ3=B{%*B$T5Nj%6!-Hr;I!Scj`lH z&2dHFlOISwWJ&S2vf~@I4i~(0*T%OFiuX|eD*nd2utS4$1_JM?zmp>a#CsVy6Er^z zeNNZZDE?R3pM?>~e?H_N`C`hy%m4jb;6L#8=a7l>3eJS2LGgEUxsau-Yh9l~o7=Yh z2mYg3`m5*3Ik|lKQf~euzZlCWzaN&=vHuHtOwK!2@W6)hqq$Zm|7`Nmu%9^F6UH?+ z@2ii+=iJ;ZzhiUKu$QB()nKk3FooI>Jr_IjzY6=qxYy;&mvi7BlQ?t4kRjIhb|2q? zd^K~{-^cxjVSj?!Xs=Da5IHmFzRj!Kzh~b!?`P7c&T9s77VLYB?8_?F zauM^)p;qFG!9PHLfIsnt43UnmV?Wn?Ki7aXSosgq;f?MYUuSIYwOn(5vWhb{f%$pn z4ySN-z}_%7|B);A@PA5k*7kkdr4xZ@s{e9j+9w;*RFm;XPDQwx%~;8iBzSKTIGKO z{53ZZU*OLr@S5=k;?CM^i#zkxs3Sj%z0U`L%q`qM+tP zX$aL;*^g$7UyM2Go+_4A+f)IQcy^G$h2E zb?nT$XlgTEFJI8GN6NQf%-eVn9mPilRqUbT$pN-|;FEjq@Ao&TxpZg=mEgBHB zU@grU;&sfmqlO=6|G3sU;7t8rbK$?X0y_v9$^{X`m4jZ_BR|B|@?ZCLSPPEzz`w1n zP5nA;4(kQFKm%$enjkkBxM%Y}2si&d|62L)U(dCzCGn56HN+i#6|nV-TGIo0;W;`( zW-y=1KF4dp$$mC_|6}pbb>IHoKQeZajXQB>jVR?u`R>%l1o54?6NnS*arpVopdEF; zeC5J3*M0p`*8lif;!irrcjC?(uExejsi~>4wKYwstGY^N@KY}TujLx`S=Cu+T=!dx zKWlPm->I**E{A*q-Z^FFT5$G%7Ij0_*Mo4-y6~RmyTzUB&lfae(WZfO>um}mnsDXPEbau-!13!!xd!qh*{C)6&bz0j1I{>y$D-S)b*)JMCPk!=~KL&6Ngin0p6MCOxF2L_R9t8N!$2Wpced<#`y!F;w zKTi5V_kX&X09wAIJ#anfg9Dhn0s7(C6Nj3S-mVn(i|C6ZAVq0$hE)874co};g z^hR7pe4lU$P;*ggYc4o&UTQC%liCXooIfkI3TNaBV%t~FRr}yHu7kjQ2J*3;e%;iW zvDVCh8=G80KAeyhCuY2LjrC!Od1rvF7h}zszxGV)&!)6ChP5WAjv-zQAMNJIG!JHS zwl?pLxC-V5II#(hQ`l)ZAp&M0xd4%cxmco*MIk?{BD=BK`1vpc}D39|XlV z{c&0oGdDa~TL2FT4lh=~1NL5O-P~0?V2#ie`v^CnANfGUM!b4F=JkCwd7Q`c8Na2q zJGQQk^?6w}Vg9-{|2047((lAV84uN%sK!N2?V(!_1{{v6rdgZl56f0zDMQ+q)jKzzu^ztsVken;=DjAh6G`Cw`Q4G+BjS+n*=KI~^K{W=%t zbD-rN)O4|*Q~@<#@1Vx$E!0W9`B~IZeFn87sHMXD>$M%|Bh93rdGf1lKoX3K651t&nhsl= zXxG|%@8}Bbrlp_u#t*DZX<}_0Yb{A9*1Pd_)LtqNwy6xT4pZrOY{s?N4)pPwT(i#y zT%`lRi8U#Ken4fw>H+N`{f#FF?ZxFlLZg7z7#cr4X>id z{9kUD`d2=w_Zlb{^c`5IOxWCZ1k<0T1D1Z31IU0Q2edsZ1K0xv$pQVYq2KEp&#v#Z z?{m@Lin;*Str(C2sfF^L>{R3cjY`~#)m>Wm$Y|1fzeS0-$(Q^z@} zEO*vlb-^XK9>w&Ef^=Zzo-1AFSP#9zb~X5_+){$(eB4K z8gtW+nl{q+CTh+>v(gWrsP^DB*ge(~Q$AGxJ-eYc1isti%$%nM<_&Ev?%|??PK`$p z{f-PM{Ym8k<$$)(F9)tqzFJ?h&Dk@D?Dt{4CHKJWLs8$zy6+(R)pr@0ur)xY{=uXFFzH_> z-F^tN1y(2hG8V)GpDg%wW0Px_ep~nIjD~*HCSxDi0y`H!`V*~RHs^uQsb1*bK1qGpmd zB1m`Cjw0`nLBF2|umz+a#2X$c?Lj;M?Lj;MUp*d>7j~ayNAyj@SLpeH`)BgRH}byy zyQSat!;U{@O(<<2fp&oQkIy$z`_CQ-)O@RN;QD9T4y|wIJ^%U#(BF%=`i49}j!D-) zkOwPSJaG03SMkE~BzW}b_v>LA&y)EEYO6sbdnTX*$>UF|JhZ&^MSb4}Tgbne_4n+C zwI8U4i~PI>7a3{kVa8|))*%C0|K+bIbmV~a`|G#+`TU#g zXW;bWIcWsQi9c4X*RUDpIfyoPY)2bI-r9)xulm1CJDkQd6u+f)_N=w1ElgEBjprPF z3o?Ly0RVeY_{3~fPVckRMxe2lM8hj!B8F)JO z!`AP6>u>5Y&3o9t0QxBpNE=lJx#NyIbp1gD zzUYBIPYHIv9ngk-Zt~<)62^1Zs1LLYMh@_tP^I7EX-9)Ed0^@y{k65Gp0KRcTmMWw zU|+)qx{#q0SL+4q?Q`i0>COIIF8a0Cf&C`hbMj?LmG9K&iW-?PJt*u)38tTXAP>@R zZL6uH^!RYNq$p>PKz7f-zvg>OKXcZ8h!%Vo@{VUZp|+iUD_xb(N~G|6c#oQK^nHZU zKg#F6<)+`rf~k*Xjjye+syV{bwU2glMMMs-^ss4`bYaVroXzn`YQUd__UlZL_mLs z(vO}k!~(mi|L+(5&;>r<;|OHnbXBE78LruP;{yBxZ6y7K3)nMo-{6PCI7gQi6+rF_ zkPod!Z8n}q46ykrlQS|hVB(}(2Kf7BCZ>Vc;V>ccbk2~NGaf6wGQH@W9&?Zt3v(h*P4xDrN>ex7+jH*+Qg z%^jH$&+*!v{sQ!xkWN4+>|b}qGvEd6ANzgqoVy5Qfws}ef2QqF{iiR5{pT}PS&yjo z>lron#va-p=v;m>WB+XVz|o;UJFdjo5_!RRD|6W{4}A2a#bZv)gS_`b|KsSH)Sd_JIr%<%n06TX&t{&!H#{)?4W9hlJ`R1>FyugOh3=D_{einr zu(Wf`qTkvED+gEULO0I*Hs%f;&=`=X4;N8Ovf28x$A*11`dmfy2=$+PNqX>XcG`h% zJY&A6@&)*WT^rC(Caj}2+|X|6cICm5h0OK0cGB_!wEKFZJU)OQ+TZ1q2bTx9hxnq& z$9ee|f9|0M^)#E&Pr4)f?o&DMM4w>Ksb{hF(0|wh+5_{vPow{V%TFzU2za&gjttNi zIyR9qA56dX52Qbv2aY^g`U7R43-p`#sO1A=KS2aKgfR+Yu^bQ*i-qu z%0mP;Ap)B~zZgO9lG^`325gOf?iUHF{~7jyGC)3L(eL(SQ70VzR~wLN18tnx(Cz2~ zctBl1kI)wAe+cxWHw*NW-d;=pd+>+wd$a@GBju*wFvabSaPtHiT!o#QFC+wBVwYo3s=y;z1jM+M=Fj!FZM>UzpL-eZzOT( zhmZmEfWa=%KE#V3-ZK5#v!Hzd{zc^{ctF~- z>DT-U`}5!fk$aj24`#uGdB7r`>oX5tU|d*b|N3V1lXmv%MGrvE(dXG)^-J*LA>$LE z7kut4`zE)v{@Op|(|@i#c>tM!12FQh?}PfA0`Bp%=%*RiXVzLDXnXtE@4B)5uR}a> zbNU}q+712pIrM`k^odG8dKtG$zwHmQI^c}tfjx5?egx3!e%JRm_64e+>`Ra1IRfLb z1KQ`SxmH{cZfyVS5m(&`{V}Y4j6J{b17`h6KWqZ&hfc(oR zxM%w!$F(mKy05kY&lco3%zvLCxBW+t*rxO+i=qGMvobx0-<7`VUu)ka`){=ew+Ovt zg%52_{&UbkUA8aJPWsk)gYWV4`dnxI%s?7^fGpq{ZQuu=VH{-t7w~K%_E<8`zS;V- zKTho*>;UQQul^1GT^HCt@I-q?)&4!QDgBndn?3sNKYKCQFU4LGKJ$n@Je$&w9@E$X z^p@iJ(v&`1(tq~1zc>0Vow-KR&vm!GUzT?Eqgnc)leZ9p)-Z*C!zqb=-$XG0 z^!8RfuQs5s>Q~qcz92(a_Q+KH?C*vCTr~UdTiR`JGuNH8v(J|FTiSEcPrBpmHRtmd zI2Jng0J=bXK);YY^rM?jzn?~X-Pe`GbAy{D)Y6D&1GY-EBcy%Bq?bKh?A>DD9DD!p z?{q02wno2sraGUkZv5dx+J8)&K$)No43Zr(*S`FEdL!4C)}WE}vJd%{S6-3VUw>Wp z?Aasv`T0^%P$2vE?L+Qhj~qB~K%eW)xH(=b_jU}TLD&BP*Pc9hz@Z=e0nkpLkWl}> z_5J^i(9Z7$(XG9~I3sY)`OGZ#_L06+Dy4E>UstcP-rU@xJ$&rxvo!n1Ao`P~KLU-8 z{zDgN4-&A6N!kPSYbQ&7sLufi`YtE2uN$S?e&5n>Y4(q#|KP!cc1j)T^QrUXMPFaP z_SoYO8S8G}Z$?AL4`;pE?7J5K8yWqy23>cCT2{=-)+A$X^-I9=e!@J@A&-;Ufc)`H}c(VI&;0x zrrGv()5mjP%jXzS{^|29?bLNXS0bC%p!YXI!;O457rjCEEzMkGf~B3$T}dXBO23tP z+Ci>;5UoM?C@bU@f9G1^X3=ly&ZeFH<@|RnOG--A&)fd)AUgjw?%izq{p(KJ`EP0v z2mU)P!+3t@X14DA=E2RR-|p${GZ9ETX=d+kJRZL$nSa0daI@&oUUxnZg0xd_xu>Vz lzF#z5%kSKX?YLH3ll^(hI(_`L*t#Iva2Ede*Z;>H_ - - - netcoreapp2.0 - ..\docker-compose.dcproj - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/shipping/delivery/MockDroneScheduler/appsettings.Development.json b/src/shipping/delivery/MockDroneScheduler/appsettings.Development.json deleted file mode 100644 index fa8ce71a..00000000 --- a/src/shipping/delivery/MockDroneScheduler/appsettings.Development.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - } -} diff --git a/src/shipping/delivery/MockDroneScheduler/appsettings.json b/src/shipping/delivery/MockDroneScheduler/appsettings.json deleted file mode 100644 index 815fe795..00000000 --- a/src/shipping/delivery/MockDroneScheduler/appsettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" - }, - "Serilog": { - "MinimumLevel": "Verbose", - "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], - "WriteTo": [] - } -} \ No newline at end of file diff --git a/src/shipping/delivery/MockThirdPartyService/.dockerignore b/src/shipping/delivery/MockThirdPartyService/.dockerignore deleted file mode 100644 index d8f8175f..00000000 --- a/src/shipping/delivery/MockThirdPartyService/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!obj/Docker/publish/* -!obj/Docker/empty/ diff --git a/src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs b/src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs deleted file mode 100644 index e9589eaa..00000000 --- a/src/shipping/delivery/MockThirdPartyService/Controllers/ThirdPartyDeliveriesController.cs +++ /dev/null @@ -1,31 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Fabrikam.DroneDelivery.Common; - -namespace MockThirdPartyService.Controllers -{ - [Route("api/[controller]")] - public class ThirdPartyDeliveriesController : Controller - { - private readonly ILogger logger; - - public ThirdPartyDeliveriesController(ILoggerFactory loggerFactory) - { - this.logger = loggerFactory.CreateLogger(); - } - - // PUT api/thirdpartydeliveries/5 - [HttpPut("{id}")] - public IActionResult Put([FromBody]Location pickup, [FromBody]Location dropoff, string id) - { - logger.LogInformation("In Put action with DeliveryId: {DeliveryId}", id); - - return Ok(false); - } - } -} diff --git a/src/shipping/delivery/MockThirdPartyService/Dockerfile b/src/shipping/delivery/MockThirdPartyService/Dockerfile deleted file mode 100644 index 7a7d7017..00000000 --- a/src/shipping/delivery/MockThirdPartyService/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM microsoft/aspnetcore:2.0 -ARG source -WORKDIR /app -EXPOSE 80 -COPY ${source:-obj/Docker/publish} . -ENTRYPOINT ["dotnet", "MockThirdPartyService.dll"] diff --git a/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj b/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj deleted file mode 100644 index b8c05b71..00000000 --- a/src/shipping/delivery/MockThirdPartyService/MockThirdPartyService.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - netcoreapp2.0 - ..\docker-compose.dcproj - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/shipping/delivery/MockThirdPartyService/Program.cs b/src/shipping/delivery/MockThirdPartyService/Program.cs deleted file mode 100644 index dc276e40..00000000 --- a/src/shipping/delivery/MockThirdPartyService/Program.cs +++ /dev/null @@ -1,28 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; - -namespace MockThirdPartyService -{ - public class Program - { - public static void Main(string[] args) - { - BuildWebHost(args).Run(); - } - - public static IWebHost BuildWebHost(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup() - .Build(); - } -} diff --git a/src/shipping/delivery/MockThirdPartyService/Startup.cs b/src/shipping/delivery/MockThirdPartyService/Startup.cs deleted file mode 100644 index ecc259df..00000000 --- a/src/shipping/delivery/MockThirdPartyService/Startup.cs +++ /dev/null @@ -1,72 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Swashbuckle.AspNetCore.Swagger; -using Serilog; -using Serilog.Formatting.Compact; -using Fabrikam.DroneDelivery.Common; - -namespace MockThirdPartyService -{ - public class Startup - { - public Startup(IHostingEnvironment env) - { - var builder = new ConfigurationBuilder() - .SetBasePath(env.ContentRootPath) - .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) - .AddEnvironmentVariables(); - Configuration = builder.Build(); - } - - public IConfigurationRoot Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. - public void ConfigureServices(IServiceCollection services) - { - services.AddSingleton(); - - services.AddLogging(loggingBuilder => - loggingBuilder.AddSerilog(dispose: true)); - - // Add framework services. - services.AddMvc(); - - // Register the Swagger generator, defining one or more Swagger documents - services.AddSwaggerGen(c => - { - c.SwaggerDoc("v1", new Info { Title = "Mock ThirdPartyService API", Version = "v1" }); - }); - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) - { - Log.Logger = new LoggerConfiguration() - .WriteTo.Console(new CompactJsonFormatter()) - .ReadFrom.Configuration(Configuration) - .Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) - .CreateLogger(); - - app.UseMvc(); - - // Enable middleware to serve generated Swagger as a JSON endpoint. - app.UseSwagger(); - - // Enable middleware to serve swagger-ui (HTML, JS, CSS etc.), specifying the Swagger JSON endpoint. - app.UseSwaggerUI(c => - { - c.SwaggerEndpoint("/swagger/v1/swagger.json", "Mock ThirdPartyService API V1"); - }); - } - } -} diff --git a/src/shipping/delivery/MockThirdPartyService/appsettings.Development.json b/src/shipping/delivery/MockThirdPartyService/appsettings.Development.json deleted file mode 100644 index fa8ce71a..00000000 --- a/src/shipping/delivery/MockThirdPartyService/appsettings.Development.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - } -} diff --git a/src/shipping/delivery/MockThirdPartyService/appsettings.json b/src/shipping/delivery/MockThirdPartyService/appsettings.json deleted file mode 100644 index 815fe795..00000000 --- a/src/shipping/delivery/MockThirdPartyService/appsettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" - }, - "Serilog": { - "MinimumLevel": "Verbose", - "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], - "WriteTo": [] - } -} \ No newline at end of file diff --git a/src/shipping/deliveryhistory/Model.csx b/src/shipping/deliveryhistory/Model.csx deleted file mode 100644 index d7d9b72d..00000000 --- a/src/shipping/deliveryhistory/Model.csx +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Collections.ObjectModel; -using Newtonsoft.Json; - - public abstract class BaseCache - { - public abstract string Key { get; } - } - public class DateTimeStamp - { - public DateTimeStamp(string dateTimeValue) - { - DateTimeValue = dateTimeValue; - } - - public string DateTimeValue { get; set; } - } - public class Location - { - public Location(double altitude, double latitude, double longitude) - { - Altitude = altitude; - Latitude = latitude; - Longitude = longitude; - } - public double Altitude { get; } - public double Latitude { get; } - public double Longitude { get; } - } - public enum ConfirmationType - { - FingerPrint, - Picture, - Voice, - None - } - public class Confirmation - { - public Confirmation(DateTimeStamp dateTime, Location geoCoordinates, ConfirmationType confirmationType, string confirmationBlob) - { - DateTime = dateTime; - GeoCoordinates = geoCoordinates; - ConfirmationType = confirmationType; - ConfirmationBlob = confirmationBlob; - } - public DateTimeStamp DateTime { get; } - public Location GeoCoordinates { get; } - public ConfirmationType ConfirmationType { get; } - public string ConfirmationBlob { get; } - } - public enum DeliveryStage - { - Created, - Rescheduled, - HeadedToPickup, - HeadedToDropoff, - Completed, - Cancelled - } - public class DeliveryTrackingEvent : BaseCache - { - public string DeliveryId { get; set; } - public DeliveryStage Stage { get; set; } - public Location Location { get; set; } - public override string Key => $"{this.DeliveryId}_{this.Stage.ToString()}"; - } - - public class InternalDelivery : BaseCache - { - public InternalDelivery(string id, - UserAccount owner, - Location pickup, - Location dropoff, - ReadOnlyCollection packageids, - string deadline, - bool expedited, - ConfirmationType confirmationRequired, - string droneid) - { - Id = id; - Owner = owner; - Pickup = pickup; - Dropoff = dropoff; - PackageIds = packageids; - Deadline = deadline; - Expedited = expedited; - ConfirmationRequired = confirmationRequired; - DroneId = droneid; - } - - [JsonProperty(PropertyName = "id")] - public string Id { get; } - public UserAccount Owner { get; } - public Location Pickup { get; } - public Location Dropoff { get; } - public ReadOnlyCollection PackageIds { get; } - public string Deadline { get; } - public bool Expedited { get; } - public ConfirmationType ConfirmationRequired { get; } - public string DroneId { get; } - public override string Key => this.Id; - } - public class BaseMessage - { - [JsonProperty(PropertyName = "partitionKey")] - public string PartitionKey { get; set; } - public string MessageType { get; set; } - - } - - public class UserAccount - { - public UserAccount(string userid, string accountid) - { - UserId = userid; - AccountId = accountid; - } - public string UserId { get; } - public string AccountId { get; } - } - public class DeliveryStatus - { - public DeliveryStatus(DeliveryStage deliveryStage, Location lastKnownLocation, string pickupeta, string deliveryeta) - { - Stage = deliveryStage; - LastKnownLocation = lastKnownLocation; - PickupETA = pickupeta; - DeliveryETA = deliveryeta; - } - public DeliveryStage Stage { get; } - public Location LastKnownLocation { get; } - public string PickupETA { get; } - public string DeliveryETA { get; } - } - public class Delivery - { - public Delivery(string id, - UserAccount owner, - Location pickup, - Location dropoff, - ReadOnlyCollection packageids, - string deadline, - bool expedited, - ConfirmationType confirmationRequired, - string droneid) - { - Id = id; - Owner = owner; - Pickup = pickup; - Dropoff = dropoff; - PackageIds = packageids; - Deadline = deadline; - Expedited = expedited; - ConfirmationRequired = confirmationRequired; - DroneId = droneid; - } - - public string Id { get; } - public UserAccount Owner { get; } - public Location Pickup { get; } - public Location Dropoff { get; } - public ReadOnlyCollection PackageIds { get; } - public string Deadline { get; } - public bool Expedited { get; } - public ConfirmationType ConfirmationRequired { get; } - public string DroneId { get; } - } - public class DeliveryHistory : BaseMessage - { - public DeliveryHistory(string id, - InternalDelivery delivery, - params DeliveryTrackingEvent[] deliveryTrackingEvents) - { - Id = id; - Delivery = delivery; - DeliveryTrackingEvents = deliveryTrackingEvents; - } - - [JsonProperty(PropertyName = "id")] - public string Id { get; } - public InternalDelivery Delivery { get; } - public DeliveryTrackingEvent[] DeliveryTrackingEvents { get; } - } - diff --git a/src/shipping/deliveryhistory/function.json b/src/shipping/deliveryhistory/function.json deleted file mode 100644 index 28f17dc7..00000000 --- a/src/shipping/deliveryhistory/function.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "bindings": [ - { - "type": "eventHubTrigger", - "name": "EventHubMessages", - "direction": "in", - "path": "deliveryeventstream", - "connection": "Your shared access policy", - "consumerGroup": "Your consumer group", - "cardinality": "many" - }, - { - "type": "documentDB", - "name": "outputDocument", - "databaseName": "outDatabase", - "collectionName": "MyCollection", - "createIfNotExists": false, - "partitionKey": "/PartitionKey", - "connection": "Your DocumentDB", - "direction": "out" - } - ], - "disabled": false -} diff --git a/src/shipping/deliveryhistory/historydocument.csx b/src/shipping/deliveryhistory/historydocument.csx deleted file mode 100644 index ab3e5a7b..00000000 --- a/src/shipping/deliveryhistory/historydocument.csx +++ /dev/null @@ -1,17 +0,0 @@ -#load "Model.csx" -using System; - - -public class HistoryDocument -{ - public HistoryDocument(DeliveryHistory history) - { - //Set the first character of deliveryID as partition key - this.PartitionKey = history.Id.ToCharArray()[0].ToString(); - this.DeliveryTrackingEvents = history.DeliveryTrackingEvents; - - } - public string PartitionKey {get;} - public DeliveryTrackingEvent[] DeliveryTrackingEvents { get; } - -} diff --git a/src/shipping/deliveryhistory/project.json b/src/shipping/deliveryhistory/project.json deleted file mode 100644 index a7d296cd..00000000 --- a/src/shipping/deliveryhistory/project.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "frameworks": { - "net46":{ - "dependencies": { - "Newtonsoft.Json": "6.0.8" - } - } - } -} diff --git a/src/shipping/deliveryhistory/run.csx b/src/shipping/deliveryhistory/run.csx deleted file mode 100644 index a9abe37b..00000000 --- a/src/shipping/deliveryhistory/run.csx +++ /dev/null @@ -1,23 +0,0 @@ -#r "Microsoft.ServiceBus" -#load "Model.csx" -#load "historydocument.csx" - -using System; -using System.Text; -using Microsoft.ServiceBus.Messaging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - - -public static async Task Run(EventData[] EventHubMessages, IAsyncCollector outputDocument, TraceWriter log) -{ - foreach (var message in EventHubMessages) - { - // Save each event in CosmosDB - var sr = new StreamReader(message.GetBodyStream()); - var serializer = new JsonSerializer(); - DeliveryHistory history = (DeliveryHistory)serializer.Deserialize(sr, typeof(DeliveryHistory)); - HistoryDocument historyDocument = new HistoryDocument(history); - await outputDocument.AddAsync(historyDocument); - } -} diff --git a/src/shipping/dronescheduler/Dockerfile b/src/shipping/dronescheduler/Dockerfile new file mode 100644 index 00000000..31662a41 --- /dev/null +++ b/src/shipping/dronescheduler/Dockerfile @@ -0,0 +1,49 @@ +FROM microsoft/dotnet:2-aspnetcore-runtime as base +WORKDIR /app +EXPOSE 80 + +FROM microsoft/dotnet:2-sdk AS build + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app +COPY delivery/Fabrikam.DroneDelivery.Common/*.csproj ./Fabrikam.DroneDelivery.Common/ +COPY dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/*.csproj ./Fabrikam.DroneDelivery.DroneScheduler/ +WORKDIR /app +RUN dotnet restore /app/Fabrikam.DroneDelivery.Common/ +RUN dotnet restore /app/Fabrikam.DroneDelivery.DroneScheduler/ + +WORKDIR /app +COPY delivery/Fabrikam.DroneDelivery.Common/. ./Fabrikam.DroneDelivery.Common/ +COPY dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/. ./Fabrikam.DroneDelivery.DroneScheduler/ + +FROM build AS publish + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +WORKDIR /app +RUN dotnet publish /app/Fabrikam.DroneDelivery.DroneScheduler/ -c Release -o ../out + +FROM base AS runtime + +MAINTAINER Fernando Antivero (https://github.com/ferantivero) + +LABEL Tags="Azure,AKS,DroneDelivery" + +ARG user=deliveryuser + +RUN useradd -m -s /bin/bash -U $user + +WORKDIR /app +COPY --from=publish /app/out ./ +COPY dronescheduler/scripts/. ./ +RUN \ + # Ensures the entry point is executable + chmod ugo+x /app/run.sh + +RUN chown -R $user.$user /app + +# Set it for subsequent commands +USER $user + +ENTRYPOINT ["/bin/bash", "/app/run.sh"] diff --git a/src/shipping/delivery/MockAccountService/.dockerignore b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/.dockerignore similarity index 100% rename from src/shipping/delivery/MockAccountService/.dockerignore rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/.dockerignore diff --git a/src/shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Controllers/DroneDeliveriesController.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs diff --git a/src/shipping/delivery/MockAccountService/MockAccountService.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj similarity index 100% rename from src/shipping/delivery/MockAccountService/MockAccountService.csproj rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj diff --git a/src/shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneDelivery.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Models/DroneDelivery.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneDelivery.cs diff --git a/src/shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageDetail.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Models/PackageDetail.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageDetail.cs diff --git a/src/shipping/delivery/MockDroneScheduler/Models/PackageSize.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageSize.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Models/PackageSize.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageSize.cs diff --git a/src/shipping/delivery/MockDroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Program.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs diff --git a/src/shipping/delivery/MockDroneScheduler/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs similarity index 100% rename from src/shipping/delivery/MockDroneScheduler/Startup.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs diff --git a/src/shipping/delivery/MockAccountService/appsettings.Development.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json similarity index 100% rename from src/shipping/delivery/MockAccountService/appsettings.Development.json rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json diff --git a/src/shipping/delivery/MockDeliveryScheduler/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json similarity index 100% rename from src/shipping/delivery/MockDeliveryScheduler/appsettings.json rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json diff --git a/src/shipping/dronescheduler/scripts/run.sh b/src/shipping/dronescheduler/scripts/run.sh new file mode 100644 index 00000000..c692f57b --- /dev/null +++ b/src/shipping/dronescheduler/scripts/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash +dotnet Fabrikam.DroneDelivery.DroneScheduler.dll diff --git a/src/shipping/scheduler/.classpath b/src/shipping/scheduler/.classpath deleted file mode 100644 index 7a2b4048..00000000 --- a/src/shipping/scheduler/.classpath +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/shipping/scheduler/.gitignore b/src/shipping/scheduler/.gitignore deleted file mode 100644 index 2af7cefb..00000000 --- a/src/shipping/scheduler/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans - -### IntelliJ IDEA ### -.idea -*.iws -*.iml -*.ipr - -### NetBeans ### -nbproject/private/ -build/ -nbbuild/ -dist/ -nbdist/ -.nb-gradle/ \ No newline at end of file diff --git a/src/shipping/scheduler/.project b/src/shipping/scheduler/.project deleted file mode 100644 index fdb9e0c4..00000000 --- a/src/shipping/scheduler/.project +++ /dev/null @@ -1,29 +0,0 @@ - - - deliveryscheduler.delivery-dispatcher-service - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.springframework.ide.eclipse.core.springbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.springframework.ide.eclipse.core.springnature - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/src/shipping/scheduler/Dockerfile b/src/shipping/scheduler/Dockerfile deleted file mode 100644 index 858c8aa7..00000000 --- a/src/shipping/scheduler/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM openjdk:8-jdk-alpine -MAINTAINER Kirpa Singh - -ENTRYPOINT ["/usr/bin/java", "-jar", "/usr/share/dronedelivery/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar"] - -# Add Maven dependencies (not shaded into the artifact; Docker-cached) -ADD target/lib /usr/share/dronedelivery/lib - -# Add the service itself -ADD target/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar /usr/share/dronedelivery/dronedelivery.delivery-scheduler-service-0.0.1-SNAPSHOT.jar - - diff --git a/src/shipping/scheduler/Dockerfilemaven b/src/shipping/scheduler/Dockerfilemaven deleted file mode 100644 index 6fca0b81..00000000 --- a/src/shipping/scheduler/Dockerfilemaven +++ /dev/null @@ -1,25 +0,0 @@ -FROM openjdk:8-jdk - -ARG MAVEN_VERSION=3.5.4 -ARG USER_HOME_DIR="/root" -ARG SHA=ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries - -RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ - && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && echo "${SHA} /tmp/apache-maven.tar.gz" | sha256sum -c - \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn - -ENV MAVEN_HOME /usr/share/maven -ENV MAVEN_CONFIG "$USER_HOME_DIR/.m2" -ENV SOURCEPATH /sln - -COPY mvn-entrypoint.sh /usr/local/bin/mvn-entrypoint.sh -COPY settings-docker.xml /usr/share/maven/ref/ -RUN chmod +x /usr/local/bin/mvn-entrypoint.sh -VOLUME "$USER_HOME_DIR/.m2" - -ENTRYPOINT ["/usr/local/bin/mvn-entrypoint.sh"] -CMD ["mvn","package"] diff --git a/src/shipping/scheduler/Readme.txt b/src/shipping/scheduler/Readme.txt deleted file mode 100644 index cd89ccb1..00000000 --- a/src/shipping/scheduler/Readme.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Instructions for building docker image -1. Make sure maven is installed and in the PATH. -2. Make sure 'Docker for Windows' is installed. -3. Open a command prompt and CD into the project folder. Make sure POM.xml is in the same folder. -4. Run 'mvn clean install'. -5. Run 'docker build -t dispatcher: .' where is the version you want to assign, e.g. v1.0. -6. Run 'docker run -it dispatcher:' where is the version you assigned in the previous step. \ No newline at end of file diff --git a/src/shipping/scheduler/conf/Config.properties b/src/shipping/scheduler/conf/Config.properties deleted file mode 100644 index f9d5e053..00000000 --- a/src/shipping/scheduler/conf/Config.properties +++ /dev/null @@ -1,14 +0,0 @@ -env.account.service.uri.key = SERVICE_URI_ACCOUNT -env.delivery.service.uri.key = SERVICE_URI_DELIVERY -env.dronescheduler.service.uri.key = SERVICE_URI_DRONE -env.package.service.uri.key = SERVICE_URI_PACKAGE -env.thirdparty.service.uri.key = SERVICE_URI_THIRDPARTY - -env.hostname.key = HOST_POD_NAME -env.proxyname.key = http_proxy -env.service.mesh.headers.key = SERVICEMESH_HEADERS -env.service.mesh.correlation.header.key = SERVICEMESH_CORRELATION_HEADER -env.checkpoint.time = CHECKPOINT_TIME_MINUTES - -env.storage.queue.connection.string = STORAGE_QUEUE_CONNECTION_STRING -env.storage.queue.name = STORAGE_QUEUE_NAME \ No newline at end of file diff --git a/src/shipping/scheduler/conf/application.conf b/src/shipping/scheduler/conf/application.conf deleted file mode 100644 index 7786e55a..00000000 --- a/src/shipping/scheduler/conf/application.conf +++ /dev/null @@ -1,85 +0,0 @@ -// Configuration file [HOCON format] - -// @see http://doc.akka.io/docs/akka/2.4.10/scala/logging.html -akka { - # Options: OFF, ERROR, WARNING, INFO, DEBUG - loglevel = "ERROR" -} - -iothub-react { - - // Connection settings can be retrieved from the Azure portal at https://portal.azure.com - // For more information about IoT Hub settings, see: - // https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-create-through-portal#endpoints - // https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-java-java-getstarted - connection { - // see: Endpoints ⇒ Messaging ⇒ Events ⇒ "Event Hub-compatible name" - hubName = ${?IOTHUB_EVENTHUB_NAME} - - // see: Endpoints ⇒ Messaging ⇒ Events ⇒ "Event Hub-compatible endpoint" - hubEndpoint = ${?IOTHUB_EVENTHUB_ENDPOINT} - - // see: Endpoints ⇒ Messaging ⇒ Events ⇒ Partitions - hubPartitions = ${?IOTHUB_EVENTHUB_PARTITIONS} - - // see: Shared access policies ⇒ key name ⇒ Connection string - accessConnString = ${?IOTHUB_ACCESS_CONNSTRING} - } - - streaming { - - // see: "IoT Hub" >> your hub > "Messaging" >> Consumer groups - // "$Default" is predefined and is the typical scenario - consumerGroup = "$Default" - - // Value expressed as a duration, e.g. 3s, 3000ms, "3 seconds", etc. - receiverTimeout = 3s - - // How many messages to retrieve on each pull, max is 900 - receiverBatchSize = 800 - - // Whether to retrieve information about the partitions while streming events from IoT Hub - retrieveRuntimeInfo = true - } - - checkpointing { - - // Checkpoints frequency (best effort), for each IoT hub partition - // Min: 1 second, Max: 1 minute - frequency = 15s - - // How many messages to stream before saving the position, for each IoT hub partition. - // Since the stream position is saved in the Source, before the rest of the - // Graph (Flows/Sinks), this provides a mechanism to replay buffered messages. - // The value should be greater than receiverBatchSize - countThreshold = 1000 - - // Store a position if its value is older than this amount of time, ignoring the threshold. - // For instance when the telemetry stops, this will force to write the last offset after some time. - // Min: 1 second, Max: 1 hour. Value is rounded to seconds. - timeThreshold = 1200s - - storage { - - // Value expressed as a duration, e.g. 3s, 3000ms, "3 seconds", etc. - rwTimeout = 5s - - // Supported types (not case sensitive): Cassandra, AzureBlob - backendType = "AzureBlob" - - // If you use the same storage while processing multiple streams, you'll want - // to use a distinct table/container/path in each job, to to keep state isolated - namespace = "ehubcheckpoint" - azureblob { - // Time allowed for a checkpoint to be written, rounded to seconds (min 15, max 60) - lease = 15s - // Whether to use the Azure Storage Emulator - useEmulator = false - // Storage credentials - protocol = "https" - account = ${?IOTHUB_CHECKPOINT_AZSTORAGE_ACCOUNT} - key = ${?IOTHUB_CHECKPOINT_AZSTORAGE_KEY} - } - } - } -} \ No newline at end of file diff --git a/src/shipping/scheduler/conf/log4j2.xml b/src/shipping/scheduler/conf/log4j2.xml deleted file mode 100644 index bb5c4ba7..00000000 --- a/src/shipping/scheduler/conf/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - diff --git a/src/shipping/scheduler/mvn-entrypoint.sh b/src/shipping/scheduler/mvn-entrypoint.sh deleted file mode 100644 index f8ed7fda..00000000 --- a/src/shipping/scheduler/mvn-entrypoint.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/bash -eu - -set -o pipefail - -# Copy files from /usr/share/maven/ref into ${MAVEN_CONFIG} -# So the initial ~/.m2 is set with expected content. -# Don't override, as this is just a reference setup -copy_reference_file() { - local root="${1}" - local f="${2%/}" - local logfile="${3}" - local rel="${f/${root}/}" # path relative to /usr/share/maven/ref/ - echo "$f" >> "$logfile" - echo " $f -> $rel" >> "$logfile" - if [[ ! -e ${MAVEN_CONFIG}/${rel} || $f = *.override ]] - then - echo "copy $rel to ${MAVEN_CONFIG}" >> "$logfile" - mkdir -p "${MAVEN_CONFIG}/$(dirname "${rel}")" - cp -r "${f}" "${MAVEN_CONFIG}/${rel}"; - fi; -} - -copy_reference_files() { - local log="$MAVEN_CONFIG/copy_reference_file.log" - - if (touch "${log}" > /dev/null 2>&1) - then - echo "--- Copying files at $(date)" >> "$log" - find /usr/share/maven/ref/ -type f -exec bash -eu -c 'copy_reference_file /usr/share/maven/ref/ "$1" "$2"' _ {} "$log" \; - else - echo "Can not write to ${log}. Wrong volume permissions? Carrying on ..." - fi -} - -export -f copy_reference_file -copy_reference_files -unset MAVEN_CONFIG - -cd $SOURCEPATH - -exec "$@" diff --git a/src/shipping/scheduler/pom.xml b/src/shipping/scheduler/pom.xml deleted file mode 100644 index 5c940746..00000000 --- a/src/shipping/scheduler/pom.xml +++ /dev/null @@ -1,203 +0,0 @@ - - 4.0.0 - com.fabrikam.dronedelivery - dronedelivery.delivery-scheduler-service - 0.0.1-SNAPSHOT - Processes the messages dropped to the event hub - - UTF-8 - - - - spring-milestone - Spring Milestone Repository - https://repo.spring.io/milestone - - - alfresco-public - https://artifacts.alfresco.com/nexus/content/groups/public - - - central - Central - http://repo1.maven.org/maven2 - - - toketi-snapshots - https://dl.bintray.com/microsoftazuretoketi/toketi-repo/ - - - - src - - - conf - - - - - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - runtime - ${project.build.directory}/lib - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.6 - - - - true - lib/ - com.fabrikam.dronedelivery.deliveryscheduler.akkareader.Main - - - - - - - - - - com.microsoft.azure - azure - 1.0.0 - - - org.slf4j - slf4j-simple - - - - - - com.fasterxml.jackson.core - jackson-databind - 2.9.0.pr3 - - - - - io.projectreactor - reactor-core - 3.0.7.RELEASE - - - - - org.springframework - spring-web - 5.0.0.M5 - - - - org.springframework - spring-web-reactive - 5.0.0.M4 - - - - org.springframework - spring-webflux - 5.0.0.M5 - - - - junit - junit - 4.12 - - - - - com.microsoft.azure.iot - iothub-react_2.12 - 0.10.0-DEV.170725c - compile - - - - org.apache.httpcomponents - httpasyncclient - 4.1.3 - - - - commons-io - commons-io - 2.6 - - - - net.javacrumbs.future-converter - future-converter-spring-java8 - 1.1.0 - - - - com.github.stefanbirkner - system-rules - 1.16.1 - - - - com.github.tomakehurst - wiremock-standalone - 2.9.0 - - - - - - org.apache.httpcomponents - fluent-hc - 4.5.3 - test - - - - org.apache.logging.log4j - log4j-api - 2.9.1 - - - org.apache.logging.log4j - log4j-core - 2.9.1 - - - - com.microsoft.azure - azure-storage - 6.1.0 - - - - com.opentable - wiremock-body-transformer - 1.1.6 - - - - \ No newline at end of file diff --git a/src/shipping/scheduler/settings-docker.xml b/src/shipping/scheduler/settings-docker.xml deleted file mode 100644 index 586c587c..00000000 --- a/src/shipping/scheduler/settings-docker.xml +++ /dev/null @@ -1,6 +0,0 @@ - - /usr/share/maven/ref/repository - diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java deleted file mode 100644 index 53999fa5..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/AkkaDelivery.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.akkareader; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.microsoft.azure.iot.iothubreact.MessageFromDevice; - -/* - * AkkaDelivery aids in flowing the original message received from event hub - * through the fluent akka api, eventually being used for checkpointing. - */ -public class AkkaDelivery { - private Delivery delivery; - private MessageFromDevice messageFromDevice; - - public Delivery getDelivery() { - return delivery; - } - - public void setDelivery(Delivery delivery) { - this.delivery = delivery; - } - - public MessageFromDevice getMessageFromDevice() { - return messageFromDevice; - } - - public void setMessageFromDevice(MessageFromDevice messageFromDevice) { - this.messageFromDevice = messageFromDevice; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java deleted file mode 100644 index 315c6829..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/Main.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.akkareader; - -import java.time.temporal.ChronoUnit; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - - -import java.util.stream.Collectors; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.DeliveryRequestEventProcessor; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.ConfigReader; - -import com.microsoft.azure.iot.iothubreact.MessageFromDevice; -import com.microsoft.azure.iot.iothubreact.SourceOptions; -import com.microsoft.azure.iot.iothubreact.javadsl.IoTHub; -import com.typesafe.config.ConfigFactory; - -import akka.NotUsed; -import akka.stream.javadsl.Flow; -import akka.stream.javadsl.Source; - -public class Main extends ReactiveStreamingApp { - - private static final Logger Log = LogManager.getLogger(Main.class); - - public static void main(String args[]) { - Map configSet = ConfigReader.readAllConfigurationValues("Config.properties"); - - // Uris for backend services and other env variables - SchedulerSettings.AccountServiceUri = System.getenv(configSet.get("env.account.service.uri.key")); - SchedulerSettings.DeliveryServiceUri = System.getenv(configSet.get("env.delivery.service.uri.key")); - SchedulerSettings.DroneSchedulerServiceUri = System.getenv(configSet.get("env.dronescheduler.service.uri.key")); - SchedulerSettings.PackageServiceUri = System.getenv(configSet.get("env.package.service.uri.key")); - SchedulerSettings.ThirdPartyServiceUri = System.getenv(configSet.get("env.thirdparty.service.uri.key")); - SchedulerSettings.ServiceMeshHeaders = Arrays.asList(System.getenv(configSet.get("env.service.mesh.headers.key")).split("\\s*,\\s*")); - SchedulerSettings.CorrelationHeader= System.getenv(configSet.get("env.service.mesh.correlation.header.key")); - SchedulerSettings.HostNameValue = System.getenv(configSet.get("env.hostname.key")); - SchedulerSettings.HttpProxyValue = System.getenv(configSet.get("env.proxyname.key")); - - SchedulerSettings.StorageQueueConnectionString = System - .getenv(configSet.get("env.storage.queue.connection.string")); - SchedulerSettings.StorageQueueName = System.getenv(configSet.get("env.storage.queue.name")); - SchedulerSettings.CheckpointTimeInMinutes = Integer - .parseInt(System.getenv(configSet.get("env.checkpoint.time"))); - - List partitionsList = getPartitionsList(); - String partitionNumber = partitionsList.stream().map(Object::toString).collect(Collectors.joining(",")); - - Log.info("Reading from partitions: {}", partitionNumber); - - SourceOptions options; - - // Either we read from time or from last known checkpoint - if (SchedulerSettings.CheckpointTimeInMinutes > 0) { - options = new SourceOptions().partitions(partitionsList).fromTime( - java.time.Instant.now().minus(SchedulerSettings.CheckpointTimeInMinutes, ChronoUnit.MINUTES)); - } else { - options = new SourceOptions().partitions(partitionsList).fromCheckpoint(null); - } - - IoTHub iotHub = new IoTHub(); - Source messages = iotHub.source(options); - - messages.map(msg -> DeliveryRequestEventProcessor.parseDeliveryRequest(msg)) - .filter(ad -> ad.getDelivery() != null).via(deliveryProcessor()).to(iotHub.checkpointSink()) - .run(streamMaterializer); - } - - /* - * Retrieves the list of partitions either from env or config - */ - private static List getPartitionsList() { - // Read setting for the partition from environment variable first - Integer partitionId = -1; - if (StringUtils.isNotEmpty(SchedulerSettings.HostNameValue)) { - partitionId = Integer.valueOf(SchedulerSettings.HostNameValue.trim().split("\\s*-\\s*")[1]); - } - - List partitionsList = null; - if (partitionId >= 0) { - partitionsList = (List) Arrays.asList(partitionId); - } else { - // Read using ConfigFactory so as to have custom configuration for - // each node in terms of partition(s) being read - partitionsList = ConfigFactory.load().getIntList("iothub-react.connection.hubPartitions"); - } - - return partitionsList; - } - - /* - * Implementation of Akka workflow in fluent mode - */ - - - private static Flow deliveryProcessor() { - return Flow.of(AkkaDelivery.class).map(delivery -> { - DeliveryRequestEventProcessor.processDeliveryRequestAsync(delivery.getDelivery(), - delivery.getMessageFromDevice().properties()); - - return delivery.getMessageFromDevice(); - }); - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java deleted file mode 100644 index 40a46bad..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/akkareader/ReactiveStreamingApp.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.akkareader; - -import akka.actor.ActorSystem; -import akka.stream.ActorMaterializer; -import akka.stream.Materializer; - -public class ReactiveStreamingApp { - private static ActorSystem system = ActorSystem.create("Dispatcher"); - protected final static Materializer streamMaterializer = ActorMaterializer.create(system); -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java deleted file mode 100644 index 9c53af6d..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/DeliveryRequestEventProcessor.java +++ /dev/null @@ -1,254 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler; - -import com.fabrikam.dronedelivery.deliveryscheduler.akkareader.AkkaDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.RetryableDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.StorageQueueClientFactory; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.*; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonSyntaxException; -import com.microsoft.azure.iot.iothubreact.MessageFromDevice; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.queue.CloudQueue; -import com.microsoft.azure.storage.queue.CloudQueueClient; -import com.microsoft.azure.storage.queue.CloudQueueMessage; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.CloseableThreadContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.http.HttpHeaders; - -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -public class DeliveryRequestEventProcessor { - private static final Logger Log = LogManager.getLogger(DeliveryRequestEventProcessor.class); - - private static final String CorrelationHeaderTag = "CorrelationId"; - - private static final Gson gsonSerializer = new GsonBuilder().setPrettyPrinting().create(); - - private static final EnumMap backendServicesMap = new EnumMap( - ServiceName.class); - - // Ensures that only one instance of each backend service is created with - // it's own connection pool - static { - backendServicesMap.put(ServiceName.AccountService, new AccountServiceCallerImpl()); - backendServicesMap.put(ServiceName.DeliveryService, new DeliveryServiceCallerImpl()); - backendServicesMap.put(ServiceName.DroneSchedulerService, new DroneSchedulerServiceCallerImpl()); - backendServicesMap.put(ServiceName.PackageService, new PackageServiceCallerImpl()); - backendServicesMap.put(ServiceName.ThirdPartyService, new ThirdPartyServiceCallerImpl()); - } - - public static AkkaDelivery parseDeliveryRequest(MessageFromDevice message) { - AkkaDelivery deliveryRequest = null; - - try { - - String dataReceived = message.contentAsString(); - Delivery delivery = gsonSerializer.fromJson(dataReceived, Delivery.class); - deliveryRequest = new AkkaDelivery(); - deliveryRequest.setDelivery(delivery); - deliveryRequest.setMessageFromDevice(message); - } catch (JsonSyntaxException | IllegalStateException e) { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e).toString()); - } - - return deliveryRequest; - } - - public static void processDeliveryRequestAsync(Delivery deliveryRequest, - Map properties) { - DeliverySchedule deliverySchedule = null; - // Extract the correlation id and log it - String correlationId = properties.get(SchedulerSettings.CorrelationHeader); - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put(CorrelationHeaderTag, - correlationId)) { - Log.info("Processing delivery request: Calling backend services for delivery id: {}", - deliveryRequest.getDeliveryId()); - - Boolean isAccountActive = invokeAccountServiceAsync(deliveryRequest, properties); - - if (isAccountActive) { - Log.info("Account is {}", (isAccountActive ? "active." : "suspended.")); - Boolean isThirdPartyRequired = invokeThirdPartyServiceAsync(deliveryRequest, properties); - Log.info("Third party is {}", (isThirdPartyRequired ? "required." : "not required.")); - - if (!isThirdPartyRequired) { - PackageGen packageGen = invokePackageServiceAsync(deliveryRequest, properties); - if (packageGen != null) { - Log.info("Package generated: {}", packageGen.toString()); - String droneId = invokeDroneSchedulerServiceAsync(deliveryRequest, properties); - - if (droneId != null) { - Log.info("Drone assigned: {}", droneId); - deliverySchedule = invokeDeliverySchedulerServiceAsync(deliveryRequest, droneId, properties); - } - } - } - } - } - - CompletableFuture.completedFuture(deliverySchedule).whenCompleteAsync((ds, error) -> { - if (ds == null) { - Log.error("Failed Delivery"); - superviseFailureAsync(deliveryRequest, ServiceName.DeliveryService, - error == null ? "Unknown error" : ExceptionUtils.getStackTrace(error).toString()) - .thenAcceptAsync(result -> Log.debug(result)); - } else { - Log.info("Completed Delivery", ds.toString()); - } - - }); - - } - - public static Boolean invokeAccountServiceAsync(Delivery deliveryRequest, Map properties) { - Boolean accountResult = null; - try { - AccountServiceCallerImpl backendService = (AccountServiceCallerImpl) backendServicesMap.get(ServiceName.AccountService); - appendServiceMeshHeaders(backendService, properties); - accountResult = backendService.isAccountActiveAsync(deliveryRequest.getOwnerId(), SchedulerSettings.AccountServiceUri); - } catch (Exception e) { - // Assume failure of service here - a crude supervisor - // implementation - superviseFailureAsync(deliveryRequest, ServiceName.AccountService, ExceptionUtils.getStackTrace(e)) - .thenAcceptAsync(result -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - Log.debug(result); - }); - } - - return accountResult; - } - - private static Boolean invokeThirdPartyServiceAsync(Delivery deliveryRequest, Map properties) { - Boolean thirdPartyResult = null; - try { - ThirdPartyServiceCallerImpl backendService = (ThirdPartyServiceCallerImpl) backendServicesMap - .get(ServiceName.ThirdPartyService); - appendServiceMeshHeaders(backendService, properties); - thirdPartyResult = backendService.isThirdPartyServiceRequiredAsync(deliveryRequest.getDropOffLocation(), - SchedulerSettings.ThirdPartyServiceUri); - } catch (Exception e) { - // Assume failure of service here - a crude supervisor - // implementation - superviseFailureAsync(deliveryRequest, ServiceName.ThirdPartyService, ExceptionUtils.getStackTrace(e)) - .thenAcceptAsync(result -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - Log.debug(result); - }); - } - - return thirdPartyResult; - } - - private static String invokeDroneSchedulerServiceAsync(Delivery deliveryRequest, Map properties) { - String droneScheduleResult = null; - try { - DroneSchedulerServiceCallerImpl backendService = (DroneSchedulerServiceCallerImpl) backendServicesMap - .get(ServiceName.DroneSchedulerService); - appendServiceMeshHeaders(backendService, properties); - droneScheduleResult = backendService.getDroneIdAsync(deliveryRequest, - SchedulerSettings.DroneSchedulerServiceUri); - } catch (Exception e) { - // Assume failure of service here - a crude supervisor - // implementation - superviseFailureAsync(deliveryRequest, ServiceName.DroneSchedulerService, ExceptionUtils.getStackTrace(e)) - .thenAcceptAsync(result -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - Log.debug(result); - }); - } - - return droneScheduleResult; - } - - public static PackageGen invokePackageServiceAsync(Delivery deliveryRequest, Map properties) { - PackageGen packageResult = null; - try { - PackageInfo packageInfo = deliveryRequest.getPackageInfo(); - PackageServiceCallerImpl backendService = (PackageServiceCallerImpl) backendServicesMap.get(ServiceName.PackageService); - appendServiceMeshHeaders(backendService, properties); - packageResult = backendService.createPackageAsync(packageInfo, SchedulerSettings.PackageServiceUri); - } catch (Exception e) { - // Assume failure of service here - a crude supervisor - // implementation - superviseFailureAsync(deliveryRequest, ServiceName.PackageService, ExceptionUtils.getStackTrace(e)) - .thenAcceptAsync(result -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - Log.debug(result); - }); - } - - return packageResult; - } - - private static DeliverySchedule invokeDeliverySchedulerServiceAsync(Delivery deliveryRequest, - String droneId, Map properties) { - DeliverySchedule deliveryResult = null; - try { - DeliveryServiceCallerImpl backendService = (DeliveryServiceCallerImpl) backendServicesMap.get(ServiceName.DeliveryService); - appendServiceMeshHeaders(backendService, properties); - deliveryResult = backendService.scheduleDeliveryAsync(deliveryRequest, droneId, SchedulerSettings.DeliveryServiceUri); - } catch (Exception e) { - // Assume failure of service here - a crude supervisor - // implementation - superviseFailureAsync(deliveryRequest, ServiceName.DeliveryService, ExceptionUtils.getStackTrace(e)) - .thenAcceptAsync(result -> { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e)); - Log.debug(result); - }); - } - - return deliveryResult; - } - - private static void appendServiceMeshHeaders(ServiceCallerImpl service, Map properties) { - HttpHeaders httpHeaders = service.getRequestHeaders(); - for (String headerName : SchedulerSettings.ServiceMeshHeaders) { - String headerValue = properties.get(headerName); - if (headerValue != null) { - if (httpHeaders.containsKey(headerName)) { - httpHeaders.replace(headerName, Arrays.asList(headerValue)); - } else { - httpHeaders.add(headerName, headerValue); - } - } - } - } - - /* - * Supervisor implementation - */ - private static CompletableFuture superviseFailureAsync(Delivery deliveryRequest, ServiceName serviceName, String errorMessage) { - CloudQueueClient queueClient = StorageQueueClientFactory.INSTANCE.get(SchedulerSettings.StorageQueueConnectionString); - String endResult = "Failed delivery request added to compensation queue!"; - try { - CloudQueue queueReference = queueClient.getQueueReference(SchedulerSettings.StorageQueueName); - queueReference.createIfNotExists(); - - // Store the metadata so that delivery could be retried from failure state if needed - String requestInJson = gsonSerializer.toJson(new RetryableDelivery(deliveryRequest, serviceName, errorMessage)); - byte[] requestInJsonBytes = requestInJson.getBytes(StandardCharsets.UTF_8); - CloudQueueMessage message = new CloudQueueMessage(requestInJsonBytes); - - queueReference.addMessage(message); - - } catch (URISyntaxException | StorageException e) { - endResult = ExceptionUtils.getStackTrace(e); - } - - return CompletableFuture.completedFuture(endResult); - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java deleted file mode 100644 index 12bf603c..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/SchedulerSettings.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler; - -import java.util.ArrayList; -import java.util.List; - -public class SchedulerSettings { - // Service URIs - public static String DeliveryServiceUri; - public static String PackageServiceUri; - public static String DroneSchedulerServiceUri; - public static String AccountServiceUri; - public static String ThirdPartyServiceUri; - - public static List ServiceMeshHeaders = new ArrayList(); - public static String CorrelationHeader; - public static String HttpProxyValue; - public static String HostNameValue; - - //StorageQueue - public static String StorageQueueConnectionString; - public static String StorageQueueName; - public static int CheckpointTimeInMinutes; -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java deleted file mode 100644 index 50291fd8..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceFailureMetadata.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler; - -public class ServiceFailureMetadata { - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java deleted file mode 100644 index 44f0aa4b..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/ServiceName.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler; - -public enum ServiceName { - AccountService, ThirdPartyService, PackageService, DroneSchedulerService, DeliveryService -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java deleted file mode 100644 index 5d23a476..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/RetryableDelivery.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.ServiceName; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; - -public class RetryableDelivery { - private Delivery deliveryRequest; - private ServiceName serviceName; - private String errorMessage; - - /** - * @param deliveryRequest - * @param serviceName - * @param errorMessage - */ - public RetryableDelivery(Delivery deliveryRequest, ServiceName serviceName, String errorMessage) { - this.deliveryRequest = deliveryRequest; - this.serviceName = serviceName; - this.errorMessage = errorMessage; - } - - public Delivery getDeliveryRequest() { - return deliveryRequest; - } - - public void setDeliveryRequest(Delivery deliveryRequest) { - this.deliveryRequest = deliveryRequest; - } - - public ServiceName getServiceName() { - return serviceName; - } - - public void setServiceName(ServiceName serviceName) { - this.serviceName = serviceName; - } - - public String getErrorMessage() { - return errorMessage; - } - - public void setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java deleted file mode 100644 index d75a8e8e..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/StorageQueueClientFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation; - -import java.net.URISyntaxException; -import java.security.InvalidKeyException; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.queue.CloudQueueClient; - -public enum StorageQueueClientFactory { - INSTANCE; - private static final Logger Log = LogManager.getLogger(StorageQueueClientFactory.class); - - public CloudQueueClient get(String connectionString) { - CloudQueueClient queueClient = null; - try { - CloudStorageAccount cloudStorageAccount = CloudStorageAccount.parse(connectionString); - queueClient = cloudStorageAccount.createCloudQueueClient(); - } catch (URISyntaxException | InvalidKeyException e) { - Log.error("throwable: {}", ExceptionUtils.getStackTrace(e).toString()); - } - return queueClient; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java deleted file mode 100644 index fe40e680..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/compensation/tests/StorageQueueTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.tests; - -import java.net.URISyntaxException; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.compensation.StorageQueueClientFactory; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.queue.CloudQueue; -import com.microsoft.azure.storage.queue.CloudQueueClient; -import com.microsoft.azure.storage.queue.CloudQueueMessage; - -public class StorageQueueTest { - @Rule - public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - - private static final String envStorageConnectionString = "STORAGE_QUEUE_CONNECTION_STRING"; - private static final String envStorageQueueName = "STORAGE_QUEUE_NAME"; - - @Before - public void setup() { - environmentVariables.set(envStorageConnectionString, ""); - environmentVariables.set(envStorageQueueName, ""); - } - - /* - * Ignoring for Maven build since storage connection and queue name needs to be set to run this test - */ - @Ignore - @Test - public void can_add_message_to_queue() throws URISyntaxException, StorageException { - // Arrange - String connStr = System.getenv(envStorageConnectionString); - CloudQueueClient storageClient = StorageQueueClientFactory.INSTANCE.get(connStr); - String someStringContent = "This is test message to queue"; - CloudQueueMessage queueMessage = new CloudQueueMessage(someStringContent); - - CloudQueue queueReference = storageClient.getQueueReference(System.getenv(envStorageQueueName)); - queueReference.createIfNotExists(); - - // Act - queueReference.addMessage(queueMessage); - - CloudQueueMessage peekMessage = queueReference.peekMessage(); - - // Assert - Assert.assertTrue(peekMessage.getMessageContentAsString().equalsIgnoreCase(someStringContent)); - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java deleted file mode 100644 index c67fc42c..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/ConfirmationType.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public enum ConfirmationType -{ - FingerPrint, - Picture, - Voice, - None -} \ No newline at end of file diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java deleted file mode 100644 index d91cc7a7..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DeliverySchedule.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public class DeliverySchedule { - private String Id; - private UserAccount owner; - private Location pickup; - private Location dropoff; - private String deadline; - private Boolean expedited; - private ConfirmationType confirmationRequired; - private String droneId; - - public Location getPickup() { - return pickup; - } - - public void setPickup(Location pickup) { - this.pickup = pickup; - } - - public Location getDropoff() { - return dropoff; - } - - public void setDropoff(Location dropoff) { - this.dropoff = dropoff; - } - - public ConfirmationType getConfirmationRequired() { - return confirmationRequired; - } - - public void setConfirmationRequired(ConfirmationType confirmationRequired) { - this.confirmationRequired = confirmationRequired; - } - - public String getDroneId() { - return droneId; - } - - public void setDroneId(String droneId) { - this.droneId = droneId; - } - - public String getDeadline() { - return this.deadline; - } - - public void setDeadline(String deadline) { - this.deadline = deadline; - } - - public Boolean getExpedited() { - return expedited; - } - - public void setExpedited(Boolean expedited) { - this.expedited = expedited; - } - - public UserAccount getOwner() { - return owner; - } - - public void setOwner(UserAccount owner) { - this.owner = owner; - } - - public String getId() { - return Id; - } - - public void setId(String id) { - Id = id; - } - - @Override - public String toString() { - return "DeliverySchedule [Id=" + Id + ", owner=" + owner.toString() + ", pickup=" + pickup + ", dropoff=" + dropoff - + ", deadline=" + deadline + ", expedited=" + expedited + ", confirmationRequired=" - + confirmationRequired.name() + ", droneId=" + droneId + "]"; //, packageId=" + packageId + " - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java deleted file mode 100644 index 1123025e..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/DroneDelivery.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public class DroneDelivery { - - public DroneDelivery() { - // TODO Auto-generated constructor stub - } - - private String deliveryId; - private Location pickup; - private Location dropoff; - private PackageDetail packageDetail; - private boolean expedited; - - public String getDeliveryId() { - return deliveryId; - } - - public void setDeliveryId(String deliveryId) { - this.deliveryId = deliveryId; - } - - public Location getPickup() { - return pickup; - } - - public void setPickup(Location pickup) { - this.pickup = pickup; - } - - public Location getDropoff() { - return dropoff; - } - - public void setDropoff(Location dropoff) { - this.dropoff = dropoff; - } - - public boolean getExpedited() { - return expedited; - } - - public void setExpedited(boolean expedited) { - this.expedited = expedited; - } - - public PackageDetail getPackageDetail() { - return packageDetail; - } - - public void setPackageDetail(PackageDetail packageDetail) { - this.packageDetail = packageDetail; - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java deleted file mode 100644 index 208bf11a..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/Location.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public class Location { - private double altitude; - private double latitude; - private double longitude; - - public Location(){ - super(); - this.altitude = 0.0; - this.latitude = 0.0; - this.longitude = 0.0; - } - - public Location(double altitude, double latitude, double longitude) { - super(); - this.altitude = altitude; - this.latitude = latitude; - this.longitude = longitude; - } - - public double getAltitude() { - return altitude; - } - - public void setAltitude(double altitude) { - this.altitude = altitude; - } - - public double getLatitude() { - return latitude; - } - - public void setLatitude(double latitude) { - this.latitude = latitude; - } - - public double getLongitude() { - return longitude; - } - - public void setLongitude(double longitude) { - this.longitude = longitude; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java deleted file mode 100644 index 16042548..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageDetail.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public class PackageDetail { - private String Id; - private PackageSize Size; - - public PackageDetail() { - - } - - public String getId() { - return Id; - } - - public void setId(String id) { - Id = id; - } - - public PackageSize getSize() { - return Size; - } - - public void setSize(PackageSize size) { - Size = size; - } - - public PackageDetail(String id, PackageSize size){ - this.Id = id; - this.Size = size; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java deleted file mode 100644 index 39daadaa..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageGen.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; - -public class PackageGen { - - public PackageGen() { - // TODO Auto-generated constructor stub - } - - private String id; - private ContainerSize size; - private String tag; - private double weight; - - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public ContainerSize getSize() { - return size; - } - public void setSize(ContainerSize size) { - this.size = size; - } - public String getTag() { - return tag; - } - public void setTag(String tag) { - this.tag = tag; - } - public double getWeight() { - return weight; - } - public void setWeight(double weight) { - this.weight = weight; - } - - @Override - public String toString() { - return "PackageGen [id=" + id + ", size=" + size.name() + ", tag=" + tag + ", weight=" + weight + "]"; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java deleted file mode 100644 index fdcd0fcc..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/PackageSize.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public enum PackageSize { - Small, Medium, Large -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java deleted file mode 100644 index 9e2fb2cc..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/invoker/UserAccount.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker; - -public class UserAccount { - private String userId; - private String accountId; - - public UserAccount(){ - - } - - public UserAccount(String userId, String accountId) { - this.userId = userId; - this.accountId = accountId; - } - - public String getUserId() { - return userId; - } - - public void setUserId(String userId) { - this.userId = userId; - } - - public String getAccountId() { - return accountId; - } - - public void setAccountId(String accountId) { - this.accountId = accountId; - } - - @Override - public String toString() { - return "UserAccount [userId=" + userId + ", accountId=" + accountId + "]"; - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java deleted file mode 100644 index f8562589..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ConfirmationRequired.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver; - -public enum ConfirmationRequired -{ - FingerPrint, - Picture, - Voice, - None -} \ No newline at end of file diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java deleted file mode 100644 index d25dbcfb..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/ContainerSize.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver; - -public enum ContainerSize { - Small, Medium, Large -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java deleted file mode 100644 index 8814ef6a..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/Delivery.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver; - -import java.util.Date; - -public class Delivery { - private String deliveryId; - private String ownerId; - private String pickupLocation; - private String dropoffLocation; - private String deadline; - private Boolean expedited; - private ConfirmationRequired confirmationRequired; - - private Date pickupTime; - - private PackageInfo packageInfo; - - public Date getPickupTime() { - return this.pickupTime; - } - - public void setPickupTime(Date pickupTime){ - this.pickupTime = pickupTime; - } - - public String getDropOffLocation() { - return this.dropoffLocation; - } - - public void setDropOffLocation(String dropoffLocation){ - this.dropoffLocation = dropoffLocation; - } - - public String getPickupLocation() { - return this.pickupLocation; - } - - public void setPickupLocation(String pickupLocation){ - this.pickupLocation = pickupLocation; - } - - public ConfirmationRequired getConfirmationRequired() { - return this.confirmationRequired; - } - - public void setConfirmationRequired(ConfirmationRequired confReq) { - this.confirmationRequired = confReq; - } - - public Boolean isExpedited() { - return expedited; - } - - public void setExpedited(Boolean expedited) { - this.expedited = expedited; - } - - public String getDeadline() { - return this.deadline; - } - - public void setDeadline(String deadline) { - this.deadline = deadline; - } - - public String getDeliveryId() { - return deliveryId; - } - - public void setDeliveryId(String deliveryId) { - this.deliveryId = deliveryId; - } - - public String getOwnerId() { - return ownerId; - } - - public void setOwnerId(String ownerId) { - this.ownerId = ownerId; - } - - @Override - public String toString() { - return String.format( - "DeliveryId:%s, OwnerId:%s, Locations [Pickup:%s, Dropoff:%s], PickupTime:%tc, Package:%s, ConfirmationType:%s, Deadline:%s, Expedited:%s", - this.deliveryId, this.ownerId, this.pickupLocation, this.dropoffLocation, this.pickupTime, - this.packageInfo.toString(), this.confirmationRequired.name(), this.deadline, this.expedited); - } - - public PackageInfo getPackageInfo() { - return packageInfo; - } - - public void setPackageInfo(PackageInfo packageInfo) { - this.packageInfo = packageInfo; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java deleted file mode 100644 index 4f96f0cc..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/models/receiver/PackageInfo.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver; - -public class PackageInfo { - private String packageId; - private ContainerSize size; - private double weight; - private String tag; - - public String getPackageId() { - return this.packageId; - } - - public void setPackageId(String packageId) { - this.packageId = packageId; - } - - public ContainerSize getSize() { - return size; - } - - public void setSize(ContainerSize size) { - this.size = size; - } - - @Override - public String toString() { - return "PackageInfo [packageId=" + packageId + ", size=" + size.name() + ", weight=" + weight + ", tag=" + tag + "]"; - } - - public double getWeight() { - return weight; - } - - public void setWeight(double weight) { - this.weight = weight; - } - - public String getTag() { - return tag; - } - - public void setTag(String tag) { - this.tag = tag; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java deleted file mode 100644 index 4cfec84c..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/AccountServiceCallerImpl.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.apache.commons.lang3.exception.ExceptionUtils; -//import org.apache.http.nio.reactor.IOReactorException; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; - - -import static net.javacrumbs.futureconverter.springjava.FutureConverter.*; - -public final class AccountServiceCallerImpl extends ServiceCallerImpl { - - private Boolean isAccountActive = false; - private String exceptionMsg = null; - - // Calls the super constructor and sets the HTTP context - public AccountServiceCallerImpl() { - super(); - } - - @Override - public ListenableFuture getData(String url, T data) { - String accountId = (String) data; - url = url.concat(accountId); - return getAsyncRestTemplate().getForEntity(url, String.class); - } - - public boolean isAccountActive(String accountId, String uri) throws InterruptedException, ExecutionException { - ListenableFuture accountMock = this.getData(uri + '/', accountId); - return Boolean.valueOf(((ResponseEntity) accountMock.get()).getBody().toString()); - } - - - @SuppressWarnings("unchecked") - public Boolean isAccountActiveAsync(String accountId, String uri) { - // Let's call the backend - ListenableFuture> future = (ListenableFuture>) this - .getData(uri + '/', accountId); - - CompletableFuture> cfuture = toCompletableFuture(future); - - cfuture.thenAcceptAsync(response -> { - if (response.getStatusCode() == HttpStatus.OK) { - isAccountActive = Boolean.valueOf(response.getBody()); - } else { - throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); - } - }).exceptionally(e -> { - exceptionMsg = ExceptionUtils.getStackTrace(e); - return null; - }); - - if(isAccountActive==null){ - throw new BackendServiceCallFailedException(exceptionMsg); - } - - return isAccountActive; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java deleted file mode 100644 index 0a3cd124..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/BackendServiceCallFailedException.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class BackendServiceCallFailedException extends RuntimeException { - - private static final long serialVersionUID = 123456798765045L; - private static final Logger Log = LogManager.getLogger(BackendServiceCallFailedException.class); - - public BackendServiceCallFailedException(String message) { - super(message); - Log.debug("BackendServiceCallFailedException raised: {}", message); - - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java deleted file mode 100644 index 288121de..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DeliveryServiceCallerImpl.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; - -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.apache.commons.lang3.exception.ExceptionUtils; -//import org.apache.http.nio.reactor.IOReactorException; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; - - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.UserAccount; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.LocationRandomizer; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.ModelsConverter; - -public class DeliveryServiceCallerImpl extends ServiceCallerImpl { - - private DeliverySchedule deliverySchedule = null; - private String exceptionMsg = null; - - // Calls the super constructor and sets the HTTP context - public DeliveryServiceCallerImpl() { - super(); - } - - @Override - public ListenableFuture postData(String url, T entity) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().postForEntity(url, - httpEntity, DeliverySchedule.class, entity); - - return response; - } - - @Override - public ListenableFuture putData(String url, T entity, Object... args) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().exchange(url, HttpMethod.PUT, httpEntity, DeliverySchedule.class); - return response; - } - - @SuppressWarnings("unchecked") - public DeliverySchedule scheduleDelivery(Delivery deliveryRequest, String droneId, String uri) - throws InterruptedException, ExecutionException { - DeliverySchedule schedule = this.createDeliverySchedule(deliveryRequest, droneId); - ListenableFuture response = this.postData(uri, schedule); - - ResponseEntity entity = (ResponseEntity) response.get(); - return entity.getStatusCode() == HttpStatus.CREATED ? entity.getBody() : null; - } - - - @SuppressWarnings("unchecked") - public DeliverySchedule scheduleDeliveryAsync(Delivery deliveryRequest, String droneId, - String uri) { - - // Create delivery schedule to post as data - DeliverySchedule schedule = this.createDeliverySchedule(deliveryRequest, droneId); - - // Let's call the backend - ListenableFuture> future = (ListenableFuture>) this - .putData(uri + '/' + schedule.getId(), schedule); - - CompletableFuture> cfuture = toCompletableFuture(future); - - cfuture.thenAcceptAsync(response -> { - deliverySchedule = response.getBody(); - if (response.getStatusCode() != HttpStatus.CREATED) { - throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); - } - }).exceptionally(e -> { - exceptionMsg = ExceptionUtils.getStackTrace(e); - return null; - }); - - if(deliverySchedule==null){ - throw new BackendServiceCallFailedException(exceptionMsg); - } - - return deliverySchedule; - } - - private DeliverySchedule createDeliverySchedule(Delivery deliveryRequest, String droneId) { - UserAccount account = new UserAccount(UUID.randomUUID().toString(), deliveryRequest.getOwnerId()); - - DeliverySchedule scheduleDelivery = new DeliverySchedule(); - scheduleDelivery.setId(deliveryRequest.getDeliveryId()); - scheduleDelivery.setOwner(account); - scheduleDelivery.setPickup(LocationRandomizer.getRandomLocation()); - scheduleDelivery.setDropoff(LocationRandomizer.getRandomLocation()); - scheduleDelivery.setDeadline(deliveryRequest.getDeadline()); - scheduleDelivery.setExpedited(deliveryRequest.isExpedited()); - scheduleDelivery - .setConfirmationRequired(ModelsConverter.getConfirmationType(deliveryRequest.getConfirmationRequired())); - scheduleDelivery.setDroneId(droneId); - - return scheduleDelivery; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java deleted file mode 100644 index 96a0dccd..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/DroneSchedulerServiceCallerImpl.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DroneDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.LocationRandomizer; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.ModelsConverter; - -public class DroneSchedulerServiceCallerImpl extends ServiceCallerImpl { - - private String droneId = null; - private String exceptionMsg = null; - - // Calls the super constructor and sets the HTTP context - public DroneSchedulerServiceCallerImpl() { - super(); - } - - @Override - public ListenableFuture postData(String url, T entity) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().postForEntity(url, httpEntity, - String.class, entity); - - return response; - } - - @Override - public ListenableFuture putData(String url, T entity, Object... args) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().exchange(url, HttpMethod.PUT, httpEntity, String.class); - - return response; - } - - @SuppressWarnings("unchecked") - public String getDroneId(Delivery deliveryRequest, String uri) throws InterruptedException, ExecutionException { - DroneDelivery delivery = createDroneDelivery(deliveryRequest); - ListenableFuture response = this.postData(uri, delivery); - - ResponseEntity entity = (ResponseEntity) response.get(); - return entity.getStatusCode() == HttpStatus.OK ? entity.getBody().toString() : null; - } - - - private DroneDelivery createDroneDelivery(Delivery deliveryRequest) { - DroneDelivery delivery = new DroneDelivery(); - delivery.setDeliveryId(deliveryRequest.getDeliveryId()); - - // TODO: Convert string location to Location instead of using below hack - delivery.setDropoff(LocationRandomizer.getRandomLocation()); - delivery.setPickup(LocationRandomizer.getRandomLocation()); - - delivery.setExpedited(delivery.getExpedited()); - delivery.setPackageDetail(ModelsConverter.getPackageDetail(deliveryRequest.getPackageInfo())); - - return delivery; - } - - @SuppressWarnings("unchecked") - public String getDroneIdAsync(Delivery deliveryRequest, String uri) { - // Create drone delivery to post data - DroneDelivery delivery = createDroneDelivery(deliveryRequest); - - // Let's call the backend - ListenableFuture> future = (ListenableFuture>) this.putData(uri+"/"+delivery.getDeliveryId(), - delivery); - - CompletableFuture> cfuture = toCompletableFuture(future); - - cfuture.thenAcceptAsync(response -> { - droneId = response.getBody(); - if (response.getStatusCode() != HttpStatus.OK) { - throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); - } - }).exceptionally(e -> { - exceptionMsg = ExceptionUtils.getStackTrace(e); - return null; - }); - - if(droneId==null){ - throw new BackendServiceCallFailedException(exceptionMsg); - } - - return droneId; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java deleted file mode 100644 index 7b7900a3..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/PackageServiceCallerImpl.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; - - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -public class PackageServiceCallerImpl extends ServiceCallerImpl { - - private final static JsonParser jsonParser = new JsonParser(); - - private PackageGen packageGen = null; - private String exceptionMsg = null; - - // Calls the super constructor and sets the HTTP context - public PackageServiceCallerImpl() { - super(); - } - - @Override - public ListenableFuture getData(String url, T data) { - // TODO implement get methods of package service - return null; - } - - @Override - public ListenableFuture postData(String url, T entity) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().postForEntity(url, httpEntity, - String.class, entity); - - return response; - } - - @Override - public ListenableFuture putData(String url, T entity, Object... args) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().exchange(url, HttpMethod.PUT, httpEntity, String.class); - - return response; - } - - @SuppressWarnings("unchecked") - public PackageGen createPackage(PackageInfo packageInfo, String uri) - throws InterruptedException, ExecutionException { - JsonObject packObj = new JsonObject(); - - packObj.addProperty("size", packageInfo.getSize().name()); - packObj.addProperty("tag", packageInfo.getTag()); - packObj.addProperty("weight", packageInfo.getWeight()); - - ListenableFuture response = this.putData(uri+'/' + packageInfo.getPackageId(), packObj.toString()); - ResponseEntity entity = (ResponseEntity) response.get(); - - return (entity.getStatusCode() == HttpStatus.CREATED ? getPackageGen(entity.getBody()) : null); - } - - @SuppressWarnings("unchecked") - public PackageGen createPackageAsync(PackageInfo packageInfo, String uri) { - // Let's call the backend - ListenableFuture> future = (ListenableFuture>) this - .putData(uri + '/' + packageInfo.getTag(), packageInfo); - - CompletableFuture> cfuture = toCompletableFuture(future); - - cfuture.thenAcceptAsync(response -> { - if (response.getStatusCode() == HttpStatus.CREATED) { - packageGen = getPackageGen(response.getBody()); - } else { - throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); - } - }).exceptionally(e -> { - exceptionMsg = ExceptionUtils.getStackTrace(e); - return null; - }); - - if(packageGen==null){ - throw new BackendServiceCallFailedException(exceptionMsg); - } - - return packageGen; - } - - private PackageGen getPackageGen(String jsonStr) { - PackageGen pack = new PackageGen(); - JsonElement jsonElem = jsonParser.parse(jsonStr); - JsonObject jObject = jsonElem.getAsJsonObject(); - pack.setId(jObject.get("id").getAsString()); - pack.setSize(ContainerSize.valueOf(jObject.get("size").getAsString())); - pack.setTag(jObject.get("tag").getAsString()); - - JsonElement weight = jObject.get("weight"); - pack.setWeight(weight.isJsonNull() ? 0.0 : weight.getAsDouble()); - - return pack; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java deleted file mode 100644 index 4bfbe0ec..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCaller.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import org.springframework.util.concurrent.ListenableFuture; - -public interface ServiceCaller { - ListenableFuture getData(String url, T data); - - ListenableFuture postData(String url, T entity); - - ListenableFuture putData(String url, T entity, Object... args); -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java deleted file mode 100644 index 265e38e8..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerImpl.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; -import java.util.Collections; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.client.HttpComponentsAsyncClientHttpRequestFactory; -import org.springframework.util.concurrent.ListenableFuture; -import org.springframework.web.client.AsyncRestTemplate; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.SchedulerSettings; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.IdleConnectionMonitorThread; - -import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.apache.http.impl.nio.client.HttpAsyncClients; -import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; -import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; -import org.apache.http.nio.reactor.ConnectingIOReactor; -import org.apache.http.nio.reactor.IOReactorException; -import org.apache.http.protocol.HttpContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.http.ConnectionReuseStrategy; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; - -public abstract class ServiceCallerImpl implements ServiceCaller { - private HttpHeaders requestHeaders; - private final static Logger Log = LogManager.getLogger(ServiceCallerImpl.class); - private final static int Second = 1000; - private HttpComponentsAsyncClientHttpRequestFactory clientHttpRequestFactory; - - public AsyncRestTemplate getAsyncRestTemplate() { - AsyncRestTemplate asyncRestTemplate = new AsyncRestTemplate(this.clientHttpRequestFactory); - asyncRestTemplate.setErrorHandler(new ServiceCallerResponseErrorHandler()); - return asyncRestTemplate; - } - - public HttpHeaders getRequestHeaders() { - return requestHeaders; - } - - public void setRequestHeaders(HttpHeaders requestHeaders) { - this.requestHeaders = requestHeaders; - } - - @Override - public ListenableFuture getData(String url, T data) { - return null; - } - - public ServiceCallerImpl() { - // Initialize http headers - initHttpHeaders(); - - // Initialize factory that is shared by all new instances of AsyncRestTemplate - this.clientHttpRequestFactory = getCustomClientHttpRequestFactory(); - } - - private void initHttpHeaders() { - this.requestHeaders = new HttpHeaders(); - this.requestHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); - this.requestHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); - } - - private HttpComponentsAsyncClientHttpRequestFactory getCustomClientHttpRequestFactory() { - PoolingNHttpClientConnectionManager poolingConnManager = - new PoolingNHttpClientConnectionManager(getIOReactor()); - - HttpHost httpProxy = getHttpProxy(); - - CloseableHttpAsyncClient client = getCustomHttpClient(poolingConnManager, httpProxy); - - // Start idle connection monitor - IdleConnectionMonitorThread staleMonitor = new IdleConnectionMonitorThread(poolingConnManager); - staleMonitor.start(); - - return getCustomClientHttpRequestFactory(client); - } - - private ConnectingIOReactor getIOReactor() { - ConnectingIOReactor ioReactor = null; - - try { - ioReactor = new DefaultConnectingIOReactor(); - } catch (IOReactorException e) { - Log.error(ExceptionUtils.getStackTrace(e)); - } - return ioReactor; - } - - private HttpHost getHttpProxy() { - HttpHost httpProxy = null; - if (StringUtils.isNotEmpty(SchedulerSettings.HttpProxyValue)) { - String[] address = SchedulerSettings.HttpProxyValue.split("\\s*:\\s*"); - httpProxy = new HttpHost(address[0], Integer.parseInt(address[1])); - } - return httpProxy; - } - - private HttpComponentsAsyncClientHttpRequestFactory getCustomClientHttpRequestFactory( - CloseableHttpAsyncClient client) { - - HttpComponentsAsyncClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsAsyncClientHttpRequestFactory(); - clientHttpRequestFactory.setConnectionRequestTimeout(30*Second); - clientHttpRequestFactory.setConnectTimeout(30*Second); - clientHttpRequestFactory.setBufferRequestBody(false); - clientHttpRequestFactory.setReadTimeout(0); - - clientHttpRequestFactory.setHttpAsyncClient(client); - return clientHttpRequestFactory; - } - - private CloseableHttpAsyncClient getCustomHttpClient(PoolingNHttpClientConnectionManager poolingConnManager, - HttpHost httpProxy) { - HttpAsyncClientBuilder builder = HttpAsyncClients.custom().setConnectionManager(poolingConnManager) - .setConnectionReuseStrategy(new ConnectionReuseStrategy() { - - @Override - public boolean keepAlive(HttpResponse response, HttpContext context) { - return true; - } - }) - // .setKeepAliveStrategy(KeepAliveStrategy.getCustomKeepAliveStrategy()) - .setMaxConnPerRoute(Integer.MAX_VALUE).setMaxConnTotal(Integer.MAX_VALUE); - - if (httpProxy != null) { - builder.setProxy(httpProxy); - } - - CloseableHttpAsyncClient client = builder.build(); - return client; - } - - @Override - public ListenableFuture postData(String url, T entity) { - return null; - } - - @Override - public ListenableFuture putData(String url, T entity, Object... args) { - return null; - } -} \ No newline at end of file diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java deleted file mode 100644 index 068ad590..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ServiceCallerResponseErrorHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.springframework.http.client.ClientHttpResponse; -import org.springframework.web.client.ResponseErrorHandler; - -public class ServiceCallerResponseErrorHandler implements ResponseErrorHandler { - - private static final Logger log = LogManager.getLogger(ServiceCallerResponseErrorHandler.class); - - @Override - public void handleError(ClientHttpResponse clienthttpresponse) throws IOException { - if(clienthttpresponse.getStatusCode().value()>=400){ - throw new BackendServiceCallFailedException(IOUtils.toString(clienthttpresponse.getBody(), StandardCharsets.UTF_8.name())); - } - } - - @Override - public boolean hasError(ClientHttpResponse clienthttpresponse) throws IOException { - if (clienthttpresponse.getStatusCode().value() >=400) { - log.error("Status code: {}", clienthttpresponse.getStatusCode()); - log.error("Response: {}", clienthttpresponse.getStatusText()); - log.error(clienthttpresponse.getBody()); - return true; - } - - return false; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java deleted file mode 100644 index 48cd7486..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/ThirdPartyServiceCallerImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services; - -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; - -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.Location; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.LocationRandomizer; - -public class ThirdPartyServiceCallerImpl extends ServiceCallerImpl { - - private Boolean isThirdPartyRequired = true; - private String exceptionMsg = null; - - public ThirdPartyServiceCallerImpl() { - super(); - } - - @Override - public ListenableFuture postData(String url, T entity) { - HttpEntity httpEntity = new HttpEntity(entity); - ListenableFuture> response = getAsyncRestTemplate().postForEntity(url, httpEntity, - String.class, entity); - - return response; - } - - @Override - public ListenableFuture putData(String url, T entity, Object... args) { - HttpEntity httpEntity = new HttpEntity(entity, this.getRequestHeaders()); - ListenableFuture> response = getAsyncRestTemplate().exchange(url, HttpMethod.PUT, httpEntity, String.class); - - return response; - } - - @SuppressWarnings("unchecked") - public boolean isThirdPartyServiceRequired(String pickupLocation, String uri) - throws InterruptedException, ExecutionException { - // TODO: Convert string location to Location instead of using the below - // hack - Location pickup = LocationRandomizer.getRandomLocation(); - ListenableFuture response = this.postData(uri, pickup); - return Boolean.valueOf(((ResponseEntity) response.get()).getBody().toString()); - } - - @SuppressWarnings("unchecked") - public Boolean isThirdPartyServiceRequiredAsync(String dropOffLocation, String uri) { - - Location pickup = LocationRandomizer.getRandomLocation(); - - // Let's call the backend - ListenableFuture> future = (ListenableFuture>) this - .putData(uri + '/' + UUID.randomUUID().toString(), pickup); - - CompletableFuture> cfuture = toCompletableFuture(future); - - cfuture.thenAcceptAsync(response -> { - if (response.getStatusCode() == HttpStatus.OK) { - isThirdPartyRequired = Boolean.valueOf(response.getBody()); - } else { - throw new BackendServiceCallFailedException(response.getStatusCode().getReasonPhrase()); - } - }).exceptionally(e -> { - exceptionMsg = ExceptionUtils.getStackTrace(e); - return null; - }); - - if(isThirdPartyRequired==null){ - throw new BackendServiceCallFailedException(exceptionMsg); - } - - return isThirdPartyRequired; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java deleted file mode 100644 index ef4aed2a..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/InvokeMockBackendServicesTest.java +++ /dev/null @@ -1,157 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.put; -import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; -import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static net.javacrumbs.futureconverter.springjava.FutureConverter.toCompletableFuture; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.AsyncRestTemplate; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DroneDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.opentable.extension.BodyTransformer; - -import wiremock.org.eclipse.jetty.http.HttpStatus; - -public class InvokeMockBackendServicesTest { - @ClassRule - public static WireMockClassRule wiremock = new WireMockClassRule(wireMockConfig().dynamicPort().dynamicHttpsPort().extensions(new BodyTransformer())); - - public final String baseUri = "http://localhost:" + wiremock.port(); - - private final AsyncRestTemplate restTemplate = new AsyncRestTemplate(); - - private final Gson deserializer = new GsonBuilder().create(); - - private static final String DeliveryId = "some-random-delivery-id"; - - private static final String Tag = "some-random-tag"; - - private static final String DroneId = UUID.randomUUID().toString(); - - private final PackageInfo packInfo = ModelsUtils.getPackageInfo(Tag); - - private final PackageGen packGen = ModelsUtils.createPackageGen(Tag); - - private final Delivery delivery = ModelsUtils.createDeliveryRequest(); - - private final DroneDelivery droneDelivery = ModelsUtils.createDroneDelivery(delivery); - - private final DeliverySchedule deliverySchedule = ModelsUtils.createDeliverySchedule(delivery, DroneId); - - @Before - public void setUp() throws Exception { - // Account service stub - wiremock.stubFor(get(urlPathMatching("/api/Account/.*")).willReturn( - aResponse().withStatus(HttpStatus.OK_200).withHeader("Content-Type", "text/plain").withBody("true"))); - - // Third party deliveries service stub - wiremock.stubFor(put(urlPathEqualTo("/api/ThirdPartyDeliveries/" + DeliveryId)) - .withRequestBody(equalToJson(deserializer.toJson(ModelsUtils.getRandomLocation(0.0)))) - .willReturn( - aResponse().withStatus(HttpStatus.CREATED_201) - .withBody("false"))); - - // Package service stub - String json = deserializer.toJson(packInfo); - String jsonReturned = deserializer.toJson(packGen); - wiremock.stubFor(put(urlPathEqualTo("/api/packages/" + Tag)) - .withRequestBody(equalToJson(json)) - .willReturn( - aResponse().withStatus(HttpStatus.CREATED_201) - .withHeader("Content-Type", "application/json") - .withBody(jsonReturned))); - - // Drone service stub - wiremock.stubFor(put(urlPathEqualTo("/api/DroneDeliveries/" + droneDelivery.getDeliveryId())) - .withRequestBody(equalToJson(deserializer.toJson(droneDelivery))) - .willReturn(aResponse().withStatus(HttpStatus.CREATED_201).withHeader("Content-Type", "text/plain") - .withBody("AssignedDroneId$(uuid)").withTransformers("body-transformer"))); - - // Delivery service stub - String json0 = deserializer.toJson(deliverySchedule); - wiremock.stubFor(put(urlPathEqualTo("/api/Deliveries/" + delivery.getDeliveryId())) - .withRequestBody(equalToJson(json0)) - .willReturn(aResponse().withStatus(HttpStatus.CREATED_201).withHeader("Content-Type", "application/json") - .withBody(json0))); - - } - - @Test - public void can_retrieve_account_status() throws IOException, InterruptedException, ExecutionException { - String uri = this.baseUri + "/api/Account/some-random-account-id"; - - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.getForEntity(uri, String.class)); - String result = ((HttpEntity)cfuture.get()).getBody(); - assertEquals(result, "true"); - } - - @Test - public void can_retrieve_thirdparty_status() throws IOException, InterruptedException, ExecutionException { - String uri = this.baseUri + "/api/ThirdPartyDeliveries/" + DeliveryId; - String json = deserializer.toJson(ModelsUtils.getRandomLocation(0.0)); - HttpEntity entity = new HttpEntity(json); - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); - String result = ((HttpEntity)cfuture.get()).getBody(); - assertEquals(result, "false"); - } - - @Test - public void can_retrieve_package() throws IOException, InterruptedException, ExecutionException { - String uri = this.baseUri + "/api/packages/" + Tag; - HttpEntity entity = new HttpEntity(deserializer.toJson(packInfo)); - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); - HttpEntity resultEntity = cfuture.get(); - PackageGen result = (PackageGen)ModelsUtils.getPackageGen(resultEntity.getBody()); - assertEquals(result.getTag(), Tag); - } - - @Test - public void can_assign_drone() throws InterruptedException, ExecutionException{ - String uri = this.baseUri + "api/DroneDeliveries/" + droneDelivery.getDeliveryId(); - HttpEntity entity = new HttpEntity(deserializer.toJson(droneDelivery)); - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.exchange(uri, HttpMethod.PUT, entity, String.class)); - String result = ((HttpEntity)cfuture.get()).getBody(); - assertTrue(result.startsWith("AssignedDroneId")); - } - - @Test - public void can_schedule_delivery() throws IOException, InterruptedException, ExecutionException { - String uri = this.baseUri + "/api/Deliveries/" + deliverySchedule.getId(); - HttpEntity entity = new HttpEntity(deserializer.toJson(deliverySchedule)); - CompletableFuture> cfuture = toCompletableFuture( - this.restTemplate.exchange(uri, HttpMethod.PUT, entity, DeliverySchedule.class)); - HttpEntity resultEntity = cfuture.get(); - DeliverySchedule result = (DeliverySchedule)resultEntity.getBody(); - assertEquals(result.getDroneId(), DroneId); - } -} - diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java deleted file mode 100644 index 9c2df27d..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/ModelsAndUtilsTest.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; -import org.springframework.util.Assert; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; - -public class ModelsAndUtilsTest { - - private Gson serializer; - private ObjectMapper objMapper; - - private static final String envHostNameString = "HOST_POD_NAME"; - private static final String envHttpProxyString = "http_proxy"; - - @Rule - public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - - @Before - public void setUp() throws Exception { - serializer = new Gson(); - objMapper = new ObjectMapper(); - } - - @Test - public void can_serialize_delivery_with_gson() { - Delivery delivery = ModelsUtils.createDeliveryRequest(); - String jsonDelivery = serializer.toJson(delivery); - Delivery rehydratedDelivery = serializer.fromJson(jsonDelivery, Delivery.class); - Assert.isInstanceOf(Delivery.class, rehydratedDelivery, "Delivery serialized correctly"); - } - - @Test - public void can_serialize_delivery_with_jackson() throws IOException { - Delivery delivery = ModelsUtils.createDeliveryRequest(); - String jsonDelivery = objMapper.writeValueAsString(delivery); - Delivery rehydratedDelivery = objMapper.readValue(jsonDelivery, Delivery.class); - Assert.isInstanceOf(Delivery.class, rehydratedDelivery, "Delivery serialized correctly"); - } - - @Test - public void can_serialize_specific_workload_correctly() { - String jsonPayload = "{\"OwnerId\":\"f0a8680e-574a-4d30-89a6-9c8d8ca2302d\",\"DeliveryId\":\"37bbe418-2254-4ebf-a848-bd9929f10b75\",\"Packages\":[{\"PackageId\":\"5114e11d-0e23-47d4-a36c-2a297c83037d\",\"Size\":1}],\"deadline\":\"LineOfDeadlyZombiatedPeople\",\"confirmationRequired\":3,\"expedited\":false,\"DropoffLocation\":\"Seattle\",\"PickupTime\":\"Jul 28, 2017 04:06:24 PM\",\"PickupLocation\":\"Florida\"}"; - Delivery delivery = serializer.fromJson(jsonPayload, Delivery.class); - Assert.isInstanceOf(Delivery.class, delivery); - } - - @Test - public void can_set_env_vars_correctly() { - environmentVariables.set(envHostNameString, "mypod-1"); - assertEquals("mypod-1", System.getenv(envHostNameString)); - } - - @Test - public void can_parse_http_proxy_correctly(){ - environmentVariables.set(envHttpProxyString, "k8s-agent-f1cfb2cf-0:4140"); - String[] address = System.getenv(envHttpProxyString).split("\\s*:\\s*"); - assertEquals(address[0], "k8s-agent-f1cfb2cf-0"); - assertEquals(Integer.parseInt(address[1]), 4140); - } - - @Test - public void can_parse_stateful_set_config_correctly() { - environmentVariables.set(envHostNameString, "mypod-1"); - String hostName = System.getenv(envHostNameString); - int partitionId = -1; - if (hostName != null) { - String trimmedHostName = hostName.trim(); - if (trimmedHostName != "") { - partitionId = Integer.parseInt(trimmedHostName.substring(trimmedHostName.indexOf('-') + 1)); - } - } - - assertEquals(1, partitionId); - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java deleted file mode 100644 index 331e8a57..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/PostDeliveryRequests.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests; - -import static org.junit.Assert.assertEquals; - -import java.util.Collections; -import java.util.concurrent.ExecutionException; - -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.util.concurrent.ListenableFuture; -import org.springframework.web.client.AsyncRestTemplate; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers.ModelsUtils; - -public class PostDeliveryRequests { - - private AsyncRestTemplate asyncRest; - private HttpHeaders requestHeaders; - - // Set below to match with your deployment - private static String postUri = "http:ingestion/api/deliveryrequests"; - - @Before - public void setUp() throws Exception { - asyncRest = new AsyncRestTemplate(); - requestHeaders = new HttpHeaders(); - requestHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); - requestHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); - } - - /* - * Ignoring for Maven build since Uri needs to be set to run this test - */ - @Ignore - @Test - public void can_post_delivery_request() throws InterruptedException, ExecutionException { - Delivery delivery = ModelsUtils.createDeliveryRequest(); - System.out.println(delivery.toString()); - - HttpEntity entity = new HttpEntity(delivery, requestHeaders); - - ListenableFuture> response = asyncRest.postForEntity(postUri, entity, - Delivery.class, delivery); - - assertEquals(HttpStatus.ACCEPTED, response.get().getStatusCode()); - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java deleted file mode 100644 index 39aab754..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/services/tests/helpers/ModelsUtils.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.services.tests.helpers; - -import java.util.Date; -import java.util.Random; -import java.util.UUID; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DeliverySchedule; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.DroneDelivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.Location; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageGen; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.UserAccount; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.Delivery; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils.ModelsConverter; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -public class ModelsUtils { - private final static Random random = new Random(); - - private final static JsonParser jsonParser = new JsonParser(); - - private final static ContainerSize[] containers = ContainerSize.values(); - - private final static String Locations[] = { "Austin", "Seattle", "Berkley", "Oregon", "Florida", "Blaine", "Renton" }; - - private final static ConfirmationRequired[] confirmations = ConfirmationRequired.values(); - - private final static double leftLimit = 10D; - - private final static double rightLimit = 100D; - - - public static Location getRandomLocation(double... arguments) { - Location location = null; - if (arguments != null && arguments.length >= 1) { - double x = arguments[0]; - location = new Location(x, x, x); - } else { - location = new Location(); - location.setAltitude(random.nextDouble()); - location.setLatitude(random.nextDouble()); - location.setLongitude(random.nextDouble()); - } - - return location; - } - - public static PackageGen getPackageGen(String jsonStr) { - PackageGen pack = new PackageGen(); - JsonElement jsonElem = jsonParser.parse(jsonStr); - JsonObject jObject = jsonElem.getAsJsonObject(); - pack.setId(jObject.get("id").getAsString()); - pack.setSize(ContainerSize.valueOf(jObject.get("size").getAsString())); - pack.setTag(jObject.get("tag").getAsString()); - - JsonElement weight = jObject.get("weight"); - pack.setWeight(weight.isJsonNull() ? 0.0 : weight.getAsDouble()); - - return pack; - } - - public static PackageInfo getPackageInfo(String randomTag){ - PackageInfo packInfo = new PackageInfo(); - - packInfo.setPackageId(UUID.randomUUID().toString()); - packInfo.setSize(containers[random.nextInt(containers.length)]); - packInfo.setTag(randomTag); - packInfo.setWeight(1.0); - - return packInfo; - } - - public static PackageGen createPackageGen(String tag) - { - PackageGen pack = new PackageGen(); - pack.setId(UUID.randomUUID().toString()); - pack.setSize(containers[random.nextInt(containers.length)]); - pack.setTag(tag); - pack.setWeight(leftLimit + new Random().nextDouble() * (rightLimit - leftLimit)); - return pack; - } - - public static Delivery createDeliveryRequest() { - PackageInfo pack = new PackageInfo(); - - pack.setPackageId(UUID.randomUUID().toString()); - pack.setSize(containers[random.nextInt(containers.length)]); - pack.setWeight(leftLimit + new Random().nextDouble() * (rightLimit - leftLimit)); - - Delivery delivery = new Delivery(); - - delivery.setDeliveryId(UUID.randomUUID().toString()); - delivery.setOwnerId(UUID.randomUUID().toString()); - delivery.setPickupTime(new Date()); - delivery.setDropOffLocation(Locations[random.nextInt(Locations.length)]); - delivery.setPickupLocation(Locations[random.nextInt(Locations.length)]); - delivery.setConfirmationRequired(confirmations[random.nextInt(confirmations.length)]); - delivery.setDeadline("ByTheEndOfDay"); - delivery.setExpedited(true); - - delivery.setPackageInfo(pack); - - return delivery; - } - - public static DroneDelivery createDroneDelivery(Delivery deliveryRequest) { - DroneDelivery delivery = new DroneDelivery(); - delivery.setDeliveryId(deliveryRequest.getDeliveryId()); - - delivery.setDropoff(ModelsUtils.getRandomLocation()); - delivery.setPickup(ModelsUtils.getRandomLocation()); - - delivery.setExpedited(delivery.getExpedited()); - delivery.setPackageDetail(ModelsConverter.getPackageDetail(deliveryRequest.getPackageInfo())); - - return delivery; - } - - public static DeliverySchedule createDeliverySchedule(Delivery deliveryRequest, String droneId) { - UserAccount account = new UserAccount(UUID.randomUUID().toString(), deliveryRequest.getOwnerId()); - - DeliverySchedule scheduleDelivery = new DeliverySchedule(); - scheduleDelivery.setId(deliveryRequest.getDeliveryId()); - scheduleDelivery.setOwner(account); - scheduleDelivery.setPickup(ModelsUtils.getRandomLocation()); - scheduleDelivery.setDropoff(ModelsUtils.getRandomLocation()); - scheduleDelivery.setDeadline(deliveryRequest.getDeadline()); - scheduleDelivery.setExpedited(deliveryRequest.isExpedited()); - scheduleDelivery - .setConfirmationRequired(ModelsConverter.getConfirmationType(deliveryRequest.getConfirmationRequired())); - scheduleDelivery.setDroneId(droneId); - - return scheduleDelivery; - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java deleted file mode 100644 index db66c7ae..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ConfigReader.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ConfigReader { - - private static final Logger Log = LogManager.getLogger(ConfigReader.class); - - private static Map getConfig(Properties properties) { - Map config = new HashMap(10); - for (Object id : properties.keySet()) { - String key = (String) id; - config.put(key, properties.getProperty(key)); - } - - return config; - } - - public static void printProperties(Map map) { - for (Map.Entry entry : map.entrySet()) { - Log.error(entry.getKey() + ": " + entry.getValue()); - } - } - - public static Map readAllConfigurationValues(String filename) { - Map appConfig = null; - Properties properties = new Properties(); - try { - properties.load(ConfigReader.class.getClassLoader().getResourceAsStream(filename)); - appConfig = getConfig(properties); - } catch (Exception e) { - Log.error(ExceptionUtils.getStackTrace(e).toString()); - } - - return appConfig; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java deleted file mode 100644 index 5421e30c..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/CustomDateTimeDeserializer.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; - -public class CustomDateTimeDeserializer extends JsonDeserializer { - private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - - @Override - public Date deserialize(JsonParser paramJsonParser, DeserializationContext paramDeserializationContext) - throws IOException, JsonProcessingException { - String str = paramJsonParser.getText().trim(); - try { - return dateFormat.parse(str); - } catch (ParseException e) { - - } - - return paramDeserializationContext.parseDate(str); - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java deleted file mode 100644 index af2b2d32..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/IdleConnectionMonitorThread.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import java.util.concurrent.TimeUnit; - -import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; - -public class IdleConnectionMonitorThread extends Thread { - - private final PoolingNHttpClientConnectionManager connMgr; - private volatile boolean shutdown; - - public IdleConnectionMonitorThread(PoolingNHttpClientConnectionManager poolingConnManager) { - super(); - this.connMgr = poolingConnManager; - } - - @Override - public void run() { - try { - while (!shutdown) { - synchronized (this) { - wait(5000); - // Close expired connections - connMgr.closeExpiredConnections(); - // Optionally, close connections - // that have been idle longer than 30 sec - connMgr.closeIdleConnections(30, TimeUnit.SECONDS); - } - } - } catch (InterruptedException ex) { - // terminate - shutdown(); - } - } - - public void shutdown() { - shutdown = true; - synchronized (this) { - notifyAll(); - } - } - -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java deleted file mode 100644 index 35ee877f..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/KeepAliveStrategy.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import org.apache.http.HeaderElement; -import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpResponse; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.apache.http.protocol.HttpContext; - -public class KeepAliveStrategy { - - public static ConnectionKeepAliveStrategy getCustomKeepAliveStrategy() { - return new ConnectionKeepAliveStrategy() { - @Override - public long getKeepAliveDuration(HttpResponse response, HttpContext context) { - // Honor 'keep-alive' header - HeaderElementIterator it = new BasicHeaderElementIterator( - response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - HeaderElement he = it.nextElement(); - String param = he.getName(); - String value = he.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - try { - return Long.parseLong(value) * 1000; - } catch (NumberFormatException ignore) { - } - } - } - - return Long.MAX_VALUE; - } - }; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java deleted file mode 100644 index 6dbb8d07..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/LocationRandomizer.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import java.util.Random; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.Location; - -public class LocationRandomizer { - - private static Random random = new Random(); - - public static Location getRandomLocation() { - Location location = new Location(); - location.setAltitude(random.nextDouble()); - location.setLatitude(random.nextDouble()); - location.setLongitude(random.nextDouble()); - - return location; - } -} diff --git a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java b/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java deleted file mode 100644 index 62f8bfcf..00000000 --- a/src/shipping/scheduler/src/com/fabrikam/dronedelivery/deliveryscheduler/scheduler/utils/ModelsConverter.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.fabrikam.dronedelivery.deliveryscheduler.scheduler.utils; - -import java.util.ArrayList; -import java.util.List; - -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.ConfirmationType; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageDetail; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.invoker.PackageSize; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ConfirmationRequired; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.ContainerSize; -import com.fabrikam.dronedelivery.deliveryscheduler.scheduler.models.receiver.PackageInfo; - -public class ModelsConverter { - public static PackageDetail getPackageDetail(PackageInfo packageInfo) { - PackageDetail packageDetail = new PackageDetail(); - packageDetail.setId(packageInfo.getPackageId()); - packageDetail.setSize(getPackageSize(packageInfo.getSize())); - return packageDetail; - } - - public static List getListOfPackageDetail(List packages) { - List listOfPackages = new ArrayList(); - for (PackageInfo packageInfo : packages) { - PackageDetail packageDetail = new PackageDetail(); - packageDetail.setId(packageInfo.getPackageId()); - packageDetail.setSize(getPackageSize(packageInfo.getSize())); - listOfPackages.add(packageDetail); - } - - return listOfPackages; - } - - public static PackageSize getPackageSize(ContainerSize containerSize) { - return containerSize != null ? PackageSize.values()[containerSize.ordinal()] : PackageSize.Small; - } - - public static ConfirmationType getConfirmationType(ConfirmationRequired confirm) { - return confirm != null ? ConfirmationType.values()[confirm.ordinal()] : ConfirmationType.None; - } -} From 18f66c63488fa0456925cedc1428eca22a50d42b Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 7 Jan 2019 15:42:51 +0000 Subject: [PATCH 040/246] Merged PR 617: Migrate the delivery service to netcore 2.2 Applied the changes described in https://docs.microsoft.com/en-us/aspnet/core/migration/20_21?view=aspnetcore-2.2 and https://docs.microsoft.com/en-us/aspnet/core/migration/21-to-22?view=aspnetcore-2.2&tabs=visual-studio Removed custom extension methods for the Keyvault configuration provider to integrate with MSI, as it's available out of the box. Related work items: #8174 --- src/shipping/delivery/Dockerfile | 4 +-- .../ConfigurationBuilderExtensions.cs | 23 ------------- .../Fabrikam.DroneDelivery.Common.csproj | 4 +-- ...DroneDelivery.DeliveryService.Tests.csproj | 14 +++++--- ...rikam.DroneDelivery.DeliveryService.csproj | 12 +++---- .../Program.cs | 7 ++-- .../Startup.cs | 3 +- .../delivery/Fabrikam.DroneDelivery.sln | 32 +------------------ .../delivery/docker-compose.ci.build.yml | 2 +- ...brikam.DroneDelivery.DroneScheduler.csproj | 9 ++---- .../Program.cs | 7 ++-- .../Startup.cs | 3 +- 12 files changed, 34 insertions(+), 86 deletions(-) delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs diff --git a/src/shipping/delivery/Dockerfile b/src/shipping/delivery/Dockerfile index 4b7bf3e3..354e45cc 100644 --- a/src/shipping/delivery/Dockerfile +++ b/src/shipping/delivery/Dockerfile @@ -1,8 +1,8 @@ -FROM microsoft/dotnet:2-aspnetcore-runtime as base +FROM microsoft/dotnet:2.2-aspnetcore-runtime as base WORKDIR /app EXPOSE 80 -FROM microsoft/dotnet:2-sdk AS build +FROM microsoft/dotnet:2.2-sdk AS build MAINTAINER Fernando Antivero (https://github.com/ferantivero) diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs deleted file mode 100644 index 19c88803..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/ConfigurationBuilderExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using Microsoft.Azure.KeyVault; -using Microsoft.Azure.Services.AppAuthentication; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.AzureKeyVault; - -namespace Fabrikam.DroneDelivery.Common -{ - public static class ConfigurationBuilderExtensions - { - public static IConfigurationBuilder AddAzureKeyVault(this IConfigurationBuilder configurationBuilder, string vault) - { - var tokenProvider = new AzureServiceTokenProvider(); - var keyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(tokenProvider.KeyVaultTokenCallback)); - - return configurationBuilder.AddAzureKeyVault(vault, keyVaultClient, new DefaultKeyVaultSecretManager()); - } - } -} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj index 2a0abb6f..e7b6e6a5 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj @@ -1,12 +1,12 @@ - netcoreapp2.0 + netcoreapp2.2 - + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj index b13e25b5..863eb49d 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj @@ -1,14 +1,18 @@  - netcoreapp2.0 + netcoreapp2.2 - - - - + + + + + + + + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index 2c9982fd..1ad7dae9 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -1,7 +1,7 @@  - netcoreapp2.0 + netcoreapp2.2 ..\docker-compose.dcproj @@ -10,18 +10,18 @@ - + + + - + + - - - diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs index b9909ee2..c783747d 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs @@ -12,13 +12,12 @@ public class Program { public static void Main(string[] args) { - BuildWebHost(args).Run(); + CreateWebHostBuilder(args).Build().Run(); } - public static IWebHost BuildWebHost(string[] args) => + public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) .UseStartup() - .UseUrls("http://0.0.0.0:8080") - .Build(); + .UseUrls("http://0.0.0.0:8080"); } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 42c140e1..5360a9af 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -50,7 +51,7 @@ public void ConfigureServices(IServiceCollection services) loggingBuilder.AddSerilog(dispose: true)); // Add framework services. - services.AddMvc(); + services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.sln b/src/shipping/delivery/Fabrikam.DroneDelivery.sln index daae1a2f..95d5273f 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.sln +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.sln @@ -7,21 +7,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.Deli EndProject Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{2670610E-2495-42AA-BB4C-915D0B74E5CE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MockDroneScheduler", "MockDroneScheduler\MockDroneScheduler.csproj", "{D3DE4852-7375-44CE-B3A3-C358473776FD}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MockServices", "MockServices", "{76102849-AD4D-4E0C-BE88-D212E0DDE108}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MockDeliveryScheduler", "MockDeliveryScheduler\MockDeliveryScheduler.csproj", "{6B8AA758-C692-4204-9F10-F4EB72C3868C}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.Common", "Fabrikam.DroneDelivery.Common\Fabrikam.DroneDelivery.Common.csproj", "{530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MockAccountService", "MockAccountService\MockAccountService.csproj", "{9C184612-E6AE-4A62-B66B-F7348B78E289}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MockThirdPartyService", "MockThirdPartyService\MockThirdPartyService.csproj", "{200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{4D81DB69-724B-44C2-A779-66778C21722E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.DroneDelivery.DeliveryService.Tests", "Fabrikam.DroneDelivery.DeliveryService.Tests\Fabrikam.DroneDelivery.DeliveryService.Tests.csproj", "{5B8E1AE5-2C26-41D0-9035-74295F83FCB8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.DeliveryService.Tests", "Fabrikam.DroneDelivery.DeliveryService.Tests\Fabrikam.DroneDelivery.DeliveryService.Tests.csproj", "{5B8E1AE5-2C26-41D0-9035-74295F83FCB8}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -37,26 +27,10 @@ Global {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Debug|Any CPU.Build.0 = Debug|Any CPU {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Release|Any CPU.ActiveCfg = Release|Any CPU {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Release|Any CPU.Build.0 = Release|Any CPU - {D3DE4852-7375-44CE-B3A3-C358473776FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D3DE4852-7375-44CE-B3A3-C358473776FD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D3DE4852-7375-44CE-B3A3-C358473776FD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D3DE4852-7375-44CE-B3A3-C358473776FD}.Release|Any CPU.Build.0 = Release|Any CPU - {6B8AA758-C692-4204-9F10-F4EB72C3868C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6B8AA758-C692-4204-9F10-F4EB72C3868C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6B8AA758-C692-4204-9F10-F4EB72C3868C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6B8AA758-C692-4204-9F10-F4EB72C3868C}.Release|Any CPU.Build.0 = Release|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Debug|Any CPU.Build.0 = Debug|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Release|Any CPU.ActiveCfg = Release|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Release|Any CPU.Build.0 = Release|Any CPU - {9C184612-E6AE-4A62-B66B-F7348B78E289}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9C184612-E6AE-4A62-B66B-F7348B78E289}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9C184612-E6AE-4A62-B66B-F7348B78E289}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9C184612-E6AE-4A62-B66B-F7348B78E289}.Release|Any CPU.Build.0 = Release|Any CPU - {200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2}.Release|Any CPU.Build.0 = Release|Any CPU {5B8E1AE5-2C26-41D0-9035-74295F83FCB8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5B8E1AE5-2C26-41D0-9035-74295F83FCB8}.Debug|Any CPU.Build.0 = Debug|Any CPU {5B8E1AE5-2C26-41D0-9035-74295F83FCB8}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -66,10 +40,6 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {D3DE4852-7375-44CE-B3A3-C358473776FD} = {76102849-AD4D-4E0C-BE88-D212E0DDE108} - {6B8AA758-C692-4204-9F10-F4EB72C3868C} = {76102849-AD4D-4E0C-BE88-D212E0DDE108} - {9C184612-E6AE-4A62-B66B-F7348B78E289} = {76102849-AD4D-4E0C-BE88-D212E0DDE108} - {200F7B50-7D90-4BB8-BF28-87A3A3DA8CC2} = {76102849-AD4D-4E0C-BE88-D212E0DDE108} {5B8E1AE5-2C26-41D0-9035-74295F83FCB8} = {4D81DB69-724B-44C2-A779-66778C21722E} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/src/shipping/delivery/docker-compose.ci.build.yml b/src/shipping/delivery/docker-compose.ci.build.yml index b6034a2d..ab96d707 100644 --- a/src/shipping/delivery/docker-compose.ci.build.yml +++ b/src/shipping/delivery/docker-compose.ci.build.yml @@ -2,7 +2,7 @@ version: '3.4' services: ci-build: - image: microsoft/aspnetcore-build:1.0-2.0 + image: microsoft/dotnet:2.2-sdk volumes: - .:/src working_dir: /src diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj index b8c05b71..8240a2c9 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj @@ -1,7 +1,7 @@  - netcoreapp2.0 + netcoreapp2.2 ..\docker-compose.dcproj @@ -10,16 +10,13 @@ - + - + - - - diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs index 42cacc20..a906aa6c 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs @@ -17,12 +17,11 @@ public class Program { public static void Main(string[] args) { - BuildWebHost(args).Run(); + CreateWebHostBuilder(args).Build().Run(); } - public static IWebHost BuildWebHost(string[] args) => + public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) - .UseStartup() - .Build(); + .UseStartup(); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs index 64e5b211..376b437e 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -39,7 +40,7 @@ public void ConfigureServices(IServiceCollection services) loggingBuilder.AddSerilog(dispose: true)); // Add framework services. - services.AddMvc(); + services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => From 3a14320f5916e9b43d77b444125d267259f056e4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 7 Jan 2019 17:40:31 +0000 Subject: [PATCH 041/246] Merged PR 618: change acr change acr that happens to be in a different namespace than the cluster's one Additional considerations: - promote yaml variables to pipeline variables resolved: #7808 --- src/shipping/delivery/azure-pipelines-validation.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/shipping/delivery/azure-pipelines-validation.yml b/src/shipping/delivery/azure-pipelines-validation.yml index 41929866..c44ba31f 100644 --- a/src/shipping/delivery/azure-pipelines-validation.yml +++ b/src/shipping/delivery/azure-pipelines-validation.yml @@ -1,11 +1,6 @@ variables: - ResourceGroup: 'pnpdh-rg' - AzureKubernetesService: 'pnpdh-cluster' BuildConfiguration: "Release" - AzureSubscription: arm_aks-ra - AzureContainerRegistry: pnpdhacr.azurecr.io - AzureContainerRegistryHelmRepo: pnpdhacr - + name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy From f762b742b656a049f6b7a027e195832cbbb50916 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 7 Jan 2019 17:48:53 +0000 Subject: [PATCH 042/246] Merged PR 619: change package ACR change acr that happens to be in a different namespace than the cluster's one Additional considerations: - promote yaml variables to pipeline variables resolved: #7808 --- src/shipping/package/azure-pipelines-validation.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/shipping/package/azure-pipelines-validation.yml b/src/shipping/package/azure-pipelines-validation.yml index 6b183002..bf7d8fa0 100644 --- a/src/shipping/package/azure-pipelines-validation.yml +++ b/src/shipping/package/azure-pipelines-validation.yml @@ -1,10 +1,5 @@ variables: - ResourceGroup: 'pnpdh-rg' - AzureKubernetesService: 'pnpdh-cluster' BuildConfiguration: "Release" - AzureSubscription: arm_aks-ra - AzureContainerRegistry: pnpdhacr.azurecr.io - AzureContainerRegistryHelmRepo: pnpdhacr name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) From 37dfc4bef3b19fc3ddec2a3bac8f099aeac8b174 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 8 Jan 2019 16:57:52 +0000 Subject: [PATCH 043/246] Merged PR 623: add skeleton workflow service add skeleton workflow service Related work items: #8289 --- k8s/workflow-identity.yaml | 24 +++++++++ k8s/workflow.yaml | 54 +++++++++++++++++++ src/shipping/workflow/Dockerfile | 18 +++++++ .../Fabrikam.Workflow.Service.csproj | 21 ++++++++ .../Fabrikam.Workflow.Service/Program.cs | 46 ++++++++++++++++ .../WorkflowService.cs | 42 +++++++++++++++ src/shipping/workflow/Fabrikam.Workflow.sln | 25 +++++++++ 7 files changed, 230 insertions(+) create mode 100644 k8s/workflow-identity.yaml create mode 100644 k8s/workflow.yaml create mode 100644 src/shipping/workflow/Dockerfile create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.sln diff --git a/k8s/workflow-identity.yaml b/k8s/workflow-identity.yaml new file mode 100644 index 00000000..752ee8f5 --- /dev/null +++ b/k8s/workflow-identity.yaml @@ -0,0 +1,24 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service identityß +################################################################################################### +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentity +metadata: + name: workflow-identity +spec: + type: 0 + ResourceID: "identityResourceId" + ClientID: "identityClientid" +--- +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentityBinding +metadata: + name: workflow-identity-binding +spec: + AzureIdentity: workflow-identity + Selector: workflow \ No newline at end of file diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml new file mode 100644 index 00000000..8978780f --- /dev/null +++ b/k8s/workflow.yaml @@ -0,0 +1,54 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Workflow service +################################################################################################### +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: workflow + labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. + app: workflow + version: 0.1.0 + bc: shipping + co: fabrikam + #aadpodidbinding: workflow +spec: + replicas: 1 + selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. + matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. + app: workflow + template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template + metadata: + labels: + app: workflow + version: 0.1.0 + bc: shipping + co: fabrikam + aadpodidbinding: workflow + annotations: + team: workflowservice + spec: + securityContext: + fsGroup: 1 + containers: + - name: workflow + image: + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: http_proxy + value: $(NODE_NAME):4140 + - name: no_proxy + value: 169.254.169.254 + - name: CORRELATION_HEADER + value: l5d-ctx-trace \ No newline at end of file diff --git a/src/shipping/workflow/Dockerfile b/src/shipping/workflow/Dockerfile new file mode 100644 index 00000000..6bbd9bcf --- /dev/null +++ b/src/shipping/workflow/Dockerfile @@ -0,0 +1,18 @@ +FROM microsoft/dotnet:2.2-runtime AS base +WORKDIR /app + +FROM microsoft/dotnet:2.2-sdk AS build +WORKDIR /src +COPY Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj Fabrikam.Workflow.Service/ +RUN dotnet restore Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +COPY . . +WORKDIR /src/Fabrikam.Workflow.Service +RUN dotnet build Fabrikam.Workflow.Service.csproj -c Release -o /app + +FROM build AS publish +RUN dotnet publish Fabrikam.Workflow.Service.csproj -c Release -o /app + +FROM base AS final +WORKDIR /app +COPY --from=publish /app . +ENTRYPOINT ["dotnet", "Fabrikam.Workflow.Service.dll"] diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj new file mode 100644 index 00000000..a71cedd8 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp2.2 + 7.1 + + + + + + + + + + + + + + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs new file mode 100644 index 00000000..7e13a323 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -0,0 +1,46 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Serilog; +using Serilog.Formatting.Compact; +using System.Threading.Tasks; + +namespace Fabrikam.Workflow.Service +{ + class Program + { + static async Task Main(string[] args) + { + await CreateHostBuilder(args).Build().RunAsync(); + } + + private static IHostBuilder CreateHostBuilder(string[] args) + { + return new HostBuilder() + .ConfigureAppConfiguration((environment, builder) => + { + builder.AddEnvironmentVariables(); + + var buildConfig = builder.Build(); + if (buildConfig["CONFIGURATION_FOLDER"] is var configurationFolder && !string.IsNullOrEmpty(configurationFolder)) + { + builder.AddKeyPerFile(configurationFolder, false); + } + }) + .ConfigureLogging(builder => + { + var serilogBuilder = new LoggerConfiguration() + //.ReadFrom.Configuration(Configuration) + //.Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) + .WriteTo.Console(new CompactJsonFormatter()); + + builder.AddSerilog(serilogBuilder.CreateLogger(), true); + }) + .ConfigureServices(services => + { + services.AddHostedService(); + }) + .UseConsoleLifetime(); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs new file mode 100644 index 00000000..231a47e0 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Fabrikam.Workflow.Service +{ + internal class WorkflowService : IHostedService + { + private readonly ILogger _logger; + private Timer _timer; + + public WorkflowService(ILogger logger) + { + _logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _timer = new Timer(c => + { + _logger.LogInformation("Processing..."); + }, null, + TimeSpan.Zero, + TimeSpan.FromSeconds(30)); + + + _logger.LogInformation("Started"); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _timer.Dispose(); + + _logger.LogInformation("Stopped"); + return Task.CompletedTask; + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.sln b/src/shipping/workflow/Fabrikam.Workflow.sln new file mode 100644 index 00000000..c2e5574c --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28307.136 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Workflow.Service", "Fabrikam.Workflow.Service\Fabrikam.Workflow.Service.csproj", "{2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {2EB1F9E1-9576-495A-AE70-258803ADBE9E} + EndGlobalSection +EndGlobal From 2a41aecae77e63358bd0e245d0d927f323a3860f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 8 Jan 2019 19:26:34 +0000 Subject: [PATCH 044/246] Merged PR 626: remove linkerd integration for delivery remove linkerd integration for delivery resolved: #8372 Related work items: #8372 --- k8s/delivery.yaml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index b72a7217..39554139 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -27,7 +27,7 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: delivery - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. + labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: delivery version: 0.1.0 bc: shipping @@ -35,9 +35,9 @@ metadata: aadpodidbinding: delivery spec: replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. + selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: delivery + app: delivery template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template metadata: labels: @@ -93,21 +93,8 @@ spec: key: EH_ConnectionString - name: EH_ENTITYPATH value: "EH_EntityPath" - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - name: no_proxy value: 169.254.169.254 - - name: CORRELATION_HEADER - value: l5d-ctx-trace ports: - name: service containerPort: 8080 @@ -130,4 +117,4 @@ spec: keyvaultobjectversions: "" resourcegroup: "keyVaultResourceGroup" subscriptionid: "keyVaultSubscriptionId" - tenantid: "keyVaultTenantId" \ No newline at end of file + tenantid: "keyVaultTenantId" From 0d270bdd814a00e78e78a296a3634ae9d120508c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 9 Jan 2019 14:01:26 +0000 Subject: [PATCH 045/246] Merged PR 627: bug fix: dronesaheduler docker dotnet version and build context path 73ba94d4108c2ce952ca8f227c8807ff4d538328: upgrade dotnet ver to 2.2 in dronescheduler Dockerfile 4ddaeb2e97131936b57eb7b323822dd6e92636c8: fix build context for dronescheduler resolved: #8386 Related work items: #8386 --- deployment.md | 2 +- src/shipping/dronescheduler/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment.md b/deployment.md index 212e3a80..9ec0fc41 100644 --- a/deployment.md +++ b/deployment.md @@ -358,7 +358,7 @@ Build and publish the container image ```bash # Build the Docker images -docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ../$DRONE_PATH +docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ # Push the images to ACR az acr login --name $ACR_NAME diff --git a/src/shipping/dronescheduler/Dockerfile b/src/shipping/dronescheduler/Dockerfile index 31662a41..e11441d6 100644 --- a/src/shipping/dronescheduler/Dockerfile +++ b/src/shipping/dronescheduler/Dockerfile @@ -1,8 +1,8 @@ -FROM microsoft/dotnet:2-aspnetcore-runtime as base +FROM microsoft/dotnet:2.2-aspnetcore-runtime as base WORKDIR /app EXPOSE 80 -FROM microsoft/dotnet:2-sdk AS build +FROM microsoft/dotnet:2.2-sdk AS build MAINTAINER Fernando Antivero (https://github.com/ferantivero) From b680c314f237e54272c55f10f4fb18a9f08a6a28 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 9 Jan 2019 14:46:53 +0000 Subject: [PATCH 046/246] Merged PR 624: add ci for dronescheduler resolved: #8135 + #7808 (validation CI) Related work items: #7808, #8135 --- .../dronescheduler/azure-pipelines-ci.yml | 123 ++++++++++++++++++ .../azure-pipelines-validation.yml | 60 +++++++++ .../dronescheduler/azure-pipelines.yml | 44 +++++++ 3 files changed, 227 insertions(+) create mode 100644 src/shipping/dronescheduler/azure-pipelines-ci.yml create mode 100644 src/shipping/dronescheduler/azure-pipelines-validation.yml create mode 100644 src/shipping/dronescheduler/azure-pipelines.yml diff --git a/src/shipping/dronescheduler/azure-pipelines-ci.yml b/src/shipping/dronescheduler/azure-pipelines-ci.yml new file mode 100644 index 00000000..7d838514 --- /dev/null +++ b/src/shipping/dronescheduler/azure-pipelines-ci.yml @@ -0,0 +1,123 @@ +steps: +- script: echo '##vso[task.setvariable variable=repositoryName]dronescheduler' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- script: echo '##vso[task.setvariable variable=chartPath]charts/dronescheduler' + name: setvarChartPath +- script: echo $(chartPath) + name: echovarChartPath + +- script: echo '##vso[task.setvariable variable=buildContext]src/shipping' + name: setvarBuildContext +- script: echo $(buildContext) + name: echovarBuildContext + +- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/dronescheduler/Dockerfile' + name: setvarDockerFileName +- script: echo $(dockerFileName) + name: echovarDockerFileName + +- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' + name: setvarReleaseVersion +- script: echo $(releaseVersion) + name: echovarReleaseVersion + +- script: echo '##vso[task.setvariable variable=aksNamespace]backend' + name: setvarAKSNamespace +- script: echo $(aksNamespace) + name: echovarAKSNamespace + +- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' + name: setvarImageName +- script: echo $(imageName) + name: echovarImageName + +- task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + + buildContext: $(buildContext) + +- task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + +- task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.9.1' + inputs: + kubectlVersion: 1.10.3 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + +- task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(AzureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + force='--force' + packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + + export force + export packageExists + + if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then + force='' + fi + + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; diff --git a/src/shipping/dronescheduler/azure-pipelines-validation.yml b/src/shipping/dronescheduler/azure-pipelines-validation.yml new file mode 100644 index 00000000..0929ff0b --- /dev/null +++ b/src/shipping/dronescheduler/azure-pipelines-validation.yml @@ -0,0 +1,60 @@ +variables: + BuildConfiguration: "Release" + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/dronescheduler/ + exclude: + - / + - /src/ + + branches: + include: + + - master + + - release/dronescheduler/v* # for bug fixes + +trigger: + batch: true + paths: + include: + - /src/shipping/dronescheduler/ + exclude: + - / + - /src/ + + branches: + include: + + - master + + - feature/* + + - topic/* + + exclude: + + - feature/experimental/* + + - topic/experimental/* + +resources: +- repo: self + +jobs: + +# CI +- job: droneschedulercivalidationjob + displayName: "dronescheduler CI Validation (build & test)" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: False + buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] + steps: + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/dronescheduler/azure-pipelines.yml b/src/shipping/dronescheduler/azure-pipelines.yml new file mode 100644 index 00000000..d51d9417 --- /dev/null +++ b/src/shipping/dronescheduler/azure-pipelines.yml @@ -0,0 +1,44 @@ +variables: + ResourceGroup: 'pnpdh-rg' + AzureKubernetesService: 'pnpdh-cluster' + BuildConfiguration: "Release" + AzureSubscription: arm_aks-ra + AzureContainerRegistry: pnpdhacr.azurecr.io + AzureContainerRegistryHelmRepo: pnpdhacr + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +trigger: + batch: true + branches: + include: + + # for new release to production: release flow strategy + - release/dronescheduler/v* + + - refs/relelase/dronescheduler/v* + + paths: + include: + - /src/shipping/dronescheduler/ + exclude: + - / + - /src/ + +resources: +- repo: self + +jobs: + +# CI +- job: droneschedulerjobci + displayName: "dronescheduler CI" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/dronescheduler/v') ] + steps: + - script: echo $(fullCI) + name: echovar + - template: ./azure-pipelines-ci.yml From 6d7e343900f0e2029472490152a0b8ec4b5735a9 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 13:41:07 +0000 Subject: [PATCH 047/246] Merged PR 629: remove integration with linkerd in dronescheduler remove integration with linkerd in dronescheduler remove not in use yaml file resolved: #8376 Related work items: #8376 --- k8s/dronescheduler.yaml | 22 ++-------- k8s/mockdeliveryscheduler.yaml | 73 ---------------------------------- 2 files changed, 4 insertions(+), 91 deletions(-) delete mode 100644 k8s/mockdeliveryscheduler.yaml diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index f8d12e27..82957c38 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -27,16 +27,16 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: dronescheduler - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. + labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: dronescheduler version: 0.1.0 bc: dronemgmt co: fabrikam spec: replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. + selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: dronescheduler + app: dronescheduler template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template metadata: labels: @@ -49,21 +49,7 @@ spec: spec: containers: - name: dronescheduler - image: - env: - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: CORRELATION_HEADER - value: l5d-ctx-trace + image: ports: - name: service containerPort: 80 diff --git a/k8s/mockdeliveryscheduler.yaml b/k8s/mockdeliveryscheduler.yaml deleted file mode 100644 index ccabddef..00000000 --- a/k8s/mockdeliveryscheduler.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Mock Delivery Scheduler service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: deliveryscheduler - labels: - app: deliveryscheduler - bc: shipping -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app: deliveryscheduler - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: deliveryscheduler - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: deliveryscheduler - version: 0.1.0 - bc: shipping -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: deliveryscheduler - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: deliveryscheduler - version: 0.1.0 - bc: shipping - annotations: - team: deliveryschedulerservice - spec: - containers: - - name: deliveryscheduler - image: dronedelivery.azurecr.io/mockdeliveryscheduler:0.1.0 - env: - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: CORRELATION_HEADER - value: l5d-ctx-trace - ports: - - name: service - containerPort: 80 - resources: - requests: - cpu: 1 - memory: 1G - limits: - cpu: 1 - memory: 1G From e52b858b54ff34851e1974f94cd94697d3911c0e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 13:41:30 +0000 Subject: [PATCH 048/246] Merged PR 630: remove linkerd integration for delivery in CI remove linkerd integration for delivery in CI resolved: #8372 Related work items: #8372 --- charts/delivery/templates/delivery-deploy.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 14875480..edbaee6a 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -53,18 +53,6 @@ spec: secretKeyRef: name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf key: CosmosDB_CollectionId - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - - name: CORRELATION_HEADER - value: {{ .Values.correlationheader }} ports: - name: service containerPort: 80 From 00dfc04a1bc367191c024ed916b27aadf531078c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 13:42:22 +0000 Subject: [PATCH 049/246] Merged PR 631: remove linkerd integration in package remove linkerd integration in package resolved: #8373 Related work items: #8373 --- charts/package/templates/package-deploy.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index dbedadfa..8e925888 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -40,16 +40,6 @@ spec: key: {{ .Release.Name }}-mongodb-pwd - name: COLLECTION_NAME value: packages - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - name: CORRELATION_HEADER value: {{ .Values.correlationheader }} ports: From 539be5d2e1289cde8879b1fc270d0c90548aae9b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 13:42:34 +0000 Subject: [PATCH 050/246] Merged PR 632: remove linkerd integration from ingestion remove linkerd integration from ingestion resolved: #8374 Related work items: #8374 --- k8s/ingestion.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index 825a046e..21bd9bb7 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -20,8 +20,8 @@ spec: name: http selector: app: ingestion - type: - LoadBalancer + type: + LoadBalancer --- apiVersion: extensions/v1beta1 kind: Deployment @@ -82,14 +82,3 @@ spec: secretKeyRef: name: ingestion-secrets key: eventhub_keyvalue - # Integration with l5d Daemon Sets - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 From 58b8b80b3e338226d9751c0ff6c560f567ef7cf2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 13:42:57 +0000 Subject: [PATCH 051/246] Merged PR 633: remove linkerd integration from workflow remove linkerd integration from workflow resolved: #8375 Related work items: #8375 --- k8s/workflow.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 8978780f..7481d954 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -10,7 +10,7 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: workflow - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. + labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. app: workflow version: 0.1.0 bc: shipping @@ -18,9 +18,9 @@ metadata: #aadpodidbinding: workflow spec: replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. + selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: workflow + app: workflow template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template metadata: labels: @@ -38,17 +38,7 @@ spec: - name: workflow image: env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: http_proxy - value: $(NODE_NAME):4140 - name: no_proxy value: 169.254.169.254 - name: CORRELATION_HEADER - value: l5d-ctx-trace \ No newline at end of file + value: l5d-ctx-trace From 3c98c896361888383a34f6c9de551b0084cc2f20 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 11 Jan 2019 15:25:25 +0000 Subject: [PATCH 052/246] Merged PR 625: Add no-op queue client in workflow service Add a queue client in the workflow service authenticated using MSI update deployment instructions accordingly Related work items: #8368 --- deployment.md | 94 ++++++++++++++++++- k8s/workflow.yaml | 19 ++++ .../Fabrikam.Workflow.Service.Tests.csproj | 19 ++++ .../UnitTest1.cs | 14 +++ .../Fabrikam.Workflow.Service.csproj | 10 ++ .../Fabrikam.Workflow.Service/Program.cs | 18 ++-- .../WorkflowService.cs | 43 ++++++--- .../WorkflowServiceOptions.cs | 16 ++++ .../appsettings.json | 23 +++++ src/shipping/workflow/Fabrikam.Workflow.sln | 16 ++++ 10 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json diff --git a/deployment.md b/deployment.md index 9ec0fc41..a8bff0c0 100644 --- a/deployment.md +++ b/deployment.md @@ -188,7 +188,7 @@ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURC # Grant the identity access to the KeyVault az role assignment create --role Reader --assignee $DELIVERY_PRINCIPAL_ID --scope $DELIVERY_KEYVAULT_ID -az keyvault set-policy --name $DELIVERY_KEYVAULT_NAME --secret-permissions get, list --spn $DELIVERY_PRINCIPAL_CLIENT_ID +az keyvault set-policy --name $DELIVERY_KEYVAULT_NAME --secret-permissions get list --spn $DELIVERY_PRINCIPAL_CLIENT_ID # Allow the cluster to manage the identity to assign to pods az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DELIVERY_PRINCIPAL_RESOURCE_ID @@ -265,6 +265,98 @@ kubectl --namespace backend apply -f $K8S/package-0.yml kubectl get pods -n backend ``` +## Deploy the Workflow service + +Provision Azure resources + +```bash +export INGESTION_QUEUE_NAMESPACE="${UNIQUE_APP_NAME_PREFIX}-ingestion-ns" +export INGESTION_QUEUE_NAME="${UNIQUE_APP_NAME_PREFIX}-ingestion" + +az servicebus namespace create --location $LOCATION \ + --name $INGESTION_QUEUE_NAMESPACE \ + --resource-group $RESOURCE_GROUP \ + --sku Standard + +az servicebus queue create --name $INGESTION_QUEUE_NAME \ + --namespace-name $INGESTION_QUEUE_NAMESPACE \ + --resource-group $RESOURCE_GROUP + +export INGESTION_QUEUE_ID=$(az servicebus queue show --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_QUEUE_NAME --query "id" --output tsv) +export INGESTION_QUEUE_NAMESPACE_ID=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "id" --output tsv) +export INGESTION_QUEUE_NAMESPACE_ENDPOINT=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "serviceBusEndpoint" --output tsv) +``` + +Build the workflow service + +```bash +export WORKFLOW_PATH=./microservices-reference-implementation/src/shipping/workflow + +# Build the Docker image +docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/workflow:0.1.0 +``` + + +Create KeyVault and secrets + +```bash +export WORKFLOW_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-workflow-kv" +az keyvault create --name $WORKFLOW_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueName --value ${INGESTION_QUEUE_NAME} +az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueEndpoint --value ${INGESTION_QUEUE_NAMESPACE_ENDPOINT} + +export WORKFLOW_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "id" --output tsv) +export WORKFLOW_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) +``` + +Create and set up pod identity + +```bash +# Create the identity and extract properties +export WORKFLOW_PRINCIPAL_NAME="workflow" +az identity create --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "id" --output tsv) +export WORKFLOW_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "principalId" --output tsv) +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "clientId" --output tsv) + +# Grant the identity access to the KeyVault +az role assignment create --role Reader --assignee $WORKFLOW_PRINCIPAL_ID --scope $WORKFLOW_KEYVAULT_ID +az keyvault set-policy --name $WORKFLOW_KEYVAULT_NAME --secret-permissions get list --spn $WORKFLOW_PRINCIPAL_CLIENT_ID + +# Grant the identity access to the ingestion queue +az role assignment create --role Contributor --assignee $WORKFLOW_PRINCIPAL_ID --scope $INGESTION_QUEUE_NAMESPACE_ID + +# Allow the cluster to manage the identity to assign to pods +az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $WORKFLOW_PRINCIPAL_RESOURCE_ID + +# Deploy the identity resources +cat $K8S/workflow-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $WORKFLOW_PRINCIPAL_CLIENT_ID#g" > $K8S/workflow-identity-0.yaml +kubectl apply -f $K8S/workflow-identity-0.yaml +``` + +Deploy the Workflow service: + +```bash +# Update the image tag and config values in the deployment YAML +sed "s#image:#image: $ACR_SERVER/workflow:0.1.0#g" $K8S/workflow.yaml | \ + sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ + sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ + sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ + sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $WORKFLOW_KEYVAULT_NAME#g" > $K8S/workflow-0.yaml + +# Deploy the service +kubectl --namespace backend apply -f $K8S/workflow-0.yaml + +# Verify the pod is created +kubectl get pods -n backend +``` + ## Deploy the Ingestion service Provision Azure resources diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 7481d954..19e49935 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -37,8 +37,27 @@ spec: containers: - name: workflow image: + volumeMounts: + - name: workflow + mountPath: /kvmnt + readOnly: true env: + - name: CONFIGURATION_FOLDER + value: /kvmnt - name: no_proxy value: 169.254.169.254 - name: CORRELATION_HEADER value: l5d-ctx-trace + volumes: + - name: workflow + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "keyVaultName" + keyvaultobjectnames: QueueName;QueueEndpoint + keyvaultobjecttypes: secret;secret + keyvaultobjectversions: "" + resourcegroup: "keyVaultResourceGroup" + subscriptionid: "keyVaultSubscriptionId" + tenantid: "keyVaultTenantId" \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj new file mode 100644 index 00000000..8c5a29cf --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj @@ -0,0 +1,19 @@ + + + + netcoreapp2.2 + + false + + + + + + + + + + + + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs new file mode 100644 index 00000000..d577beb3 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs @@ -0,0 +1,14 @@ +using System; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class UnitTest1 + { + [Fact] + public void Test1() + { + + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index a71cedd8..5cd88e11 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -7,15 +7,25 @@ + + + + + + + PreserveNewest + + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs index 7e13a323..045fb066 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Hosting; using Serilog; using Serilog.Formatting.Compact; +using System.IO; using System.Threading.Tasks; namespace Fabrikam.Workflow.Service @@ -17,27 +18,32 @@ static async Task Main(string[] args) private static IHostBuilder CreateHostBuilder(string[] args) { return new HostBuilder() - .ConfigureAppConfiguration((environment, builder) => + .ConfigureAppConfiguration((context, builder) => { - builder.AddEnvironmentVariables(); + builder + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile($"appsettings.{context.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables(); var buildConfig = builder.Build(); if (buildConfig["CONFIGURATION_FOLDER"] is var configurationFolder && !string.IsNullOrEmpty(configurationFolder)) { - builder.AddKeyPerFile(configurationFolder, false); + builder.AddKeyPerFile(Path.Combine(context.HostingEnvironment.ContentRootPath, configurationFolder), false); } }) - .ConfigureLogging(builder => + .ConfigureLogging((context, builder) => { var serilogBuilder = new LoggerConfiguration() - //.ReadFrom.Configuration(Configuration) + .ReadFrom.Configuration(context.Configuration) //.Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) .WriteTo.Console(new CompactJsonFormatter()); builder.AddSerilog(serilogBuilder.CreateLogger(), true); }) - .ConfigureServices(services => + .ConfigureServices((context, services) => { + services.AddOptions(); + services.Configure(context.Configuration); services.AddHostedService(); }) .UseConsoleLifetime(); diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs index 231a47e0..8ce08e0a 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs @@ -1,5 +1,8 @@ -using Microsoft.Extensions.Hosting; +using Microsoft.Azure.ServiceBus; +using Microsoft.Azure.ServiceBus.Primitives; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System; using System.Threading; using System.Threading.Tasks; @@ -9,33 +12,49 @@ namespace Fabrikam.Workflow.Service internal class WorkflowService : IHostedService { private readonly ILogger _logger; - private Timer _timer; + private readonly IOptions _options; + private QueueClient _receiveClient; - public WorkflowService(ILogger logger) + public WorkflowService(ILogger logger, IOptions options) { _logger = logger; + _options = options; } public Task StartAsync(CancellationToken cancellationToken) { - _timer = new Timer(c => + TokenProvider tokenProvider = TokenProvider.CreateManagedServiceIdentityTokenProvider(); + _receiveClient = new QueueClient(_options.Value.QueueEndpoint, _options.Value.QueueName, tokenProvider, receiveMode: ReceiveMode.PeekLock); + _receiveClient.RegisterMessageHandler( + ProcessMessageAsync, + new MessageHandlerOptions(ProcessMessageExceptionAsync) { - _logger.LogInformation("Processing..."); - }, null, - TimeSpan.Zero, - TimeSpan.FromSeconds(30)); - + AutoComplete = true, + MaxConcurrentCalls = _options.Value.MaxConcurrency + }); _logger.LogInformation("Started"); return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) { - _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); - _timer.Dispose(); + await _receiveClient?.CloseAsync(); _logger.LogInformation("Stopped"); + } + + private Task ProcessMessageAsync(Message message, CancellationToken ct) + { + _logger.LogInformation("Processing message {messageId}", message.MessageId); + + return Task.CompletedTask; + } + + private Task ProcessMessageExceptionAsync(ExceptionReceivedEventArgs exceptionEvent) + { + _logger.LogWarning("Exception processing message {exception}", exceptionEvent.Exception); + return Task.CompletedTask; } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs new file mode 100644 index 00000000..e4ccf24f --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs @@ -0,0 +1,16 @@ +namespace Fabrikam.Workflow.Service +{ + internal class WorkflowServiceOptions + { + public WorkflowServiceOptions() + { + MaxConcurrency = 10; + } + + public string QueueEndpoint { get; set; } + + public string QueueName { get; set; } + + public int MaxConcurrency { get; internal set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json new file mode 100644 index 00000000..8e23600c --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json @@ -0,0 +1,23 @@ +{ + "Logging": { + "IncludeScopes": false, + "LogLevel": { + "Default": "Information" + }, + "CorrelationHeaderKey": "l5d-ctx-trace" + }, + "Serilog": { + "MinimumLevel": "Verbose", + "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], + "WriteTo": [ + //{ + // "Name": "RollingFile", + // "Args": { + // "pathFormat": "Logs/deliveryservice-{Date}.log", + // "formatter": "Serilog.Formatting.Json.JsonFormatter, Serilog", + // "retainedFileCountLimit": 10 + // } + //} + ] + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.sln b/src/shipping/workflow/Fabrikam.Workflow.sln index c2e5574c..924c96e9 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.sln +++ b/src/shipping/workflow/Fabrikam.Workflow.sln @@ -5,6 +5,15 @@ VisualStudioVersion = 15.0.28307.136 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Workflow.Service", "Fabrikam.Workflow.Service\Fabrikam.Workflow.Service.csproj", "{2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2DB20E67-904A-4146-A38C-E8DB4F458A5C}" + ProjectSection(SolutionItems) = preProject + Dockerfile = Dockerfile + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{0B10A889-9683-4C1B-82BC-141994066C34}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Workflow.Service.Tests", "Fabrikam.Workflow.Service.Tests\Fabrikam.Workflow.Service.Tests.csproj", "{4D0C1CF0-61B4-451E-B3C4-D5AFF308B657}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,10 +24,17 @@ Global {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Debug|Any CPU.Build.0 = Debug|Any CPU {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {2A3140EA-2FB2-4982-AF14-E9DC66D1D1FE}.Release|Any CPU.Build.0 = Release|Any CPU + {4D0C1CF0-61B4-451E-B3C4-D5AFF308B657}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D0C1CF0-61B4-451E-B3C4-D5AFF308B657}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D0C1CF0-61B4-451E-B3C4-D5AFF308B657}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D0C1CF0-61B4-451E-B3C4-D5AFF308B657}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {4D0C1CF0-61B4-451E-B3C4-D5AFF308B657} = {0B10A889-9683-4C1B-82BC-141994066C34} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2EB1F9E1-9576-495A-AE70-258803ADBE9E} EndGlobalSection From d692bc82e37767bcd572bd17f112f05b34d7ac57 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 11 Jan 2019 16:02:18 +0000 Subject: [PATCH 053/246] Merged PR 634: remove linkerd instructions remove linkerd instructions resolved: #8377 Related work items: #8377 --- deployment.md | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/deployment.md b/deployment.md index a8bff0c0..72f063b9 100644 --- a/deployment.md +++ b/deployment.md @@ -470,38 +470,6 @@ kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml kubectl get pods -n backend ``` -## Deploy linkerd - - -```bash -kubectl create ns linkerd -wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ -sed -i "s#/default#/shipping#g" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.backend ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.backend ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -kubectl apply -f servicemesh.yml -``` - -For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) - -> Note: -> The service mesh configuration linked above uses the default namespace for service discovery. -> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified as shown. This change modifies the dtab rules. - -The linkerd accounts need to be granted permissions to query the cluster for resources, as RBAC is enabled by default on AKS. - -```bash -wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/linkerd-rbac.yml && \ -sed -i "s#namespace: default#namespace: linkerd#g" linkerd-rbac.yml && \ -kubectl apply -f linkerd-rbac.yml -``` - -For more information on using linkerd with an RBAC-enabled cluster see [https://blog.buoyant.io/2017/07/24/using-linkerd-kubernetes-rbac/](https://blog.buoyant.io/2017/07/24/using-linkerd-kubernetes-rbac/) - ## Validate the application is running You can send delivery requests to the ingestion service using the Swagger UI. @@ -547,5 +515,3 @@ sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PA sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml ``` - -Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy From da400b5803a1d501e1056f2dc0c8a832b7aed2c1 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 14 Jan 2019 13:26:16 +0000 Subject: [PATCH 054/246] Merged PR 621: Remove SxS secret management for the delivery service Removes k8s secrets for delivery, retrieves secrets only from keyvault Related work items: #8149 --- deployment.md | 10 +---- k8s/delivery.yaml | 37 ------------------- .../Startup.cs | 6 +-- 3 files changed, 4 insertions(+), 49 deletions(-) diff --git a/deployment.md b/deployment.md index 72f063b9..2486f551 100644 --- a/deployment.md +++ b/deployment.md @@ -154,10 +154,6 @@ export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-gro export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) kubectl --namespace backend create --save-config=true secret generic delivery-storageconf \ - --from-literal=CosmosDB_Key=${COSMOSDB_KEY} \ - --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT} \ - --from-literal=Redis_Endpoint=${REDIS_ENDPOINT} \ - --from-literal=Redis_AccessKey=${REDIS_KEY} \ --from-literal=EH_ConnectionString= ``` @@ -208,11 +204,7 @@ sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ sed "s/value: \"EH_EntityPath\"/value:/g" | \ - sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" | \ - sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ - sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ - sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ - sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $DELIVERY_KEYVAULT_NAME#g" > $K8S/delivery-0.yaml + sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" > $K8S/delivery-0.yaml # Deploy the service kubectl --namespace backend apply -f $K8S/delivery-0.yaml diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index 39554139..e8628b4c 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -54,38 +54,14 @@ spec: containers: - name: delivery image: - volumeMounts: - - name: delivery - mountPath: /kvmnt - readOnly: true env: # Export K8s secrets - - name: DOCDB_KEY - valueFrom: - secretKeyRef: - name: delivery-storageconf - key: CosmosDB_Key - - name: DOCDB_ENDPOINT - valueFrom: - secretKeyRef: - name: delivery-storageconf - key: CosmosDB_Endpoint - name: DOCDB_DATABASEID value: "CosmosDB_DatabaseId" - name: DOCDB_COLLECTIONID value: "CosmosDB_CollectionId" - name: KEY_VAULT_URI value: "KeyVault_Name" - - name: REDIS_ENDPOINT - valueFrom: - secretKeyRef: - name: delivery-storageconf - key: Redis_Endpoint - - name: REDIS_KEY - valueFrom: - secretKeyRef: - name: delivery-storageconf - key: Redis_AccessKey - name: EH_CONNSTR valueFrom: secretKeyRef: @@ -105,16 +81,3 @@ spec: limits: cpu: 1 memory: 3G - volumes: - - name: delivery - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "keyVaultName" - keyvaultobjectnames: CosmosDB-Key;CosmosDB-Endpoint;Redis-Endpoint;Redis-AccessKey - keyvaultobjecttypes: secret;secret;secret;secret - keyvaultobjectversions: "" - resourcegroup: "keyVaultResourceGroup" - subscriptionid: "keyVaultSubscriptionId" - tenantid: "keyVaultTenantId" diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 5360a9af..16df0fec 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -95,9 +95,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF }); //TODO look into creating a factory of DocDBRepos/RedisCache/EventHubMessenger - DocumentDBRepository.Configure(Configuration["DOCDB_ENDPOINT"], Configuration["DOCDB_KEY"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["REDIS_ENDPOINT"], Configuration["REDIS_KEY"], loggerFactory); - RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["REDIS_ENDPOINT"], Configuration["REDIS_KEY"], loggerFactory); + DocumentDBRepository.Configure(Configuration["CosmosDB-Endpoint"], Configuration["CosmosDB-Key"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); + RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["Redis-Endpoint"], Configuration["Redis-AccessKey"], loggerFactory); + RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["Redis-Endpoint"], Configuration["Redis-AccessKey"], loggerFactory); EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); } } From 92e4686de5bf09148b3a4bb2744dbd6cecc45ad8 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 14 Jan 2019 16:48:32 +0000 Subject: [PATCH 055/246] Merged PR 635: Process incoming deliveries by sending to package service Implements service to send requests to the package service. Uses HttpClientFactory to connect the service to a properly configured httpclient (which will eventually integrate resiliency). Adds test to connect the service to a test server using an httpclient for end to end testing of the processing. Related work items: #8369 --- .../Fabrikam.Workflow.Service.Tests.csproj | 12 +- .../HttpContextTestExtensions.cs | 45 +++++ .../RequestProcessorIntegrationTests.cs | 174 ++++++++++++++++++ .../RequestProcessorTests.cs | 89 +++++++++ .../TestServerMessageHandlerBuilder.cs | 29 +++ .../UnitTest1.cs | 14 -- .../Fabrikam.Workflow.Service.csproj | 3 + .../Models/ConfirmationRequired.cs | 15 ++ .../Models/ContainerSize.cs | 12 ++ .../Models/Delivery.cs | 22 +++ .../Models/PackageGen.cs | 15 ++ .../Models/PackageInfo.cs | 15 ++ .../Fabrikam.Workflow.Service/Program.cs | 20 +- .../RequestProcessing/IRequestProcessor.cs | 16 ++ .../RequestProcessing/RequestProcessor.cs | 54 ++++++ .../ServiceStartup.cs | 31 ++++ .../BackendServiceCallFailedException.cs | 24 +++ .../Services/IPackageServiceCaller.cs | 15 ++ .../Services/PackageServiceCaller.cs | 49 +++++ .../WorkflowService.cs | 105 +++++++++-- .../WorkflowServiceOptions.cs | 10 +- 21 files changed, 728 insertions(+), 41 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs delete mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationRequired.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/ContainerSize.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/Delivery.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/IRequestProcessor.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/BackendServiceCallFailedException.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj index 8c5a29cf..bceb3d78 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj @@ -7,9 +7,17 @@ + + + + - - + + + + all + runtime; build; native; contentfiles; analyzers + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs new file mode 100644 index 00000000..77385e69 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Abstractions; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; + +namespace Fabrikam.Workflow.Service.Tests +{ + public static class HttpContextTestExtensions + { + private static readonly RouteData EmptyRouteData = new RouteData(); + + private static readonly ActionDescriptor EmptyActionDescriptor = new ActionDescriptor(); + + public static Task WriteResultAsync(this HttpContext context, TResult result) + where TResult : IActionResult + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + var executor = context.RequestServices.GetService>(); + + if (executor == null) + { + throw new InvalidOperationException($"No result executor for '{typeof(TResult).FullName}' has been registered."); + } + + var routeData = context.GetRouteData() ?? EmptyRouteData; + + var actionContext = new ActionContext(context, routeData, EmptyActionDescriptor); + + return executor.ExecuteAsync(actionContext, result); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs new file mode 100644 index 00000000..ac89fb40 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -0,0 +1,174 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.RequestProcessing; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class RequestProcessorFixture : IDisposable + { + private const string PackageHost = "package"; + private static readonly string PackageUri = $"http://{PackageHost}/api/packages/"; + + private readonly IRequestProcessor _requestProcessor; + private readonly TestServer _testServer; + private RequestDelegate _handleHttpRequest = ctx => Task.CompletedTask; + + public RequestProcessorFixture() + { + var context = new HostBuilderContext(new Dictionary()); + context.Configuration = + new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + ["SERVICE_URI_PACKAGE"] = PackageUri + }).Build(); + context.HostingEnvironment = + Mock.Of(e => e.EnvironmentName == "Test"); + + var serviceCollection = new ServiceCollection(); + ServiceStartup.ConfigureServices(context, serviceCollection); + serviceCollection.AddLogging(builder => builder.AddDebug()); + + _testServer = + new TestServer( + new WebHostBuilder() + .Configure(builder => + { + builder.UseMvc(); + builder.Run(ctx => _handleHttpRequest(ctx)); + }) + .ConfigureServices(builder => + { + builder.AddMvc(); + })); + + serviceCollection.Replace( + ServiceDescriptor.Transient( + sp => new TestServerMessageHandlerBuilder(_testServer))); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + _requestProcessor = serviceProvider.GetService(); + } + + public void Dispose() + { + _testServer.Dispose(); + } + + [Fact] + public async Task ProcessingDelivery_InvokesPackageService() + { + JObject actualPackageInfo = null; + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + actualPackageInfo = await JObject.LoadAsync(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = (int)HttpStatusCode.Created + }); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + }; + + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.True(success); + + Assert.NotNull(actualPackageInfo); + Assert.Equal("package", actualPackageInfo["PackageId"].Value()); + Assert.Equal((int)ContainerSize.Medium, actualPackageInfo["Size"].Value()); + Assert.Equal("sometag", actualPackageInfo["Tag"].Value()); + Assert.Equal(100d, actualPackageInfo["Weight"].Value()); + } + + [Fact] + public async Task WhenPackageServiceSucceeds_ThenRequestSucceeds() + { + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = (int)HttpStatusCode.Created + }); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + }; + + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.True(success); + } + + [Fact] + public async Task WhenPackageServiceFails_ThenRequestFails() + { + _handleHttpRequest = async ctx => + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + }; + + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.False(success); + } + } +} + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs new file mode 100644 index 00000000..f6c5f01e --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs @@ -0,0 +1,89 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.RequestProcessing; +using Fabrikam.Workflow.Service.Services; +using Moq; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class RequestProcessorTests + { + private readonly Mock packageServiceCallerMock; + private readonly RequestProcessor processor; + + public RequestProcessorTests() + { + var servicesBuilder = new ServiceCollection(); + servicesBuilder.AddLogging(logging => logging.AddDebug()); + var services = servicesBuilder.BuildServiceProvider(); + + packageServiceCallerMock = new Mock(); + + processor = new RequestProcessor(services.GetService>(), packageServiceCallerMock.Object); + } + + [Fact] + public async Task WhenInvokingPackageServiceThrows_ProcessingFails() + { + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + + packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); + + var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + packageServiceCallerMock.Verify(); + Assert.False(success); + } + + [Fact] + public async Task WhenInvokingPackageServiceFails_ProcessingFails() + { + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + + packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(default(PackageGen)).Verifiable(); + + var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + packageServiceCallerMock.Verify(); + Assert.False(success); + } + + [Fact] + public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() + { + var delivery = + new Delivery + { + DeliveryId = "delivery", + PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + + packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + + var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + packageServiceCallerMock.Verify(); + Assert.True(success); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs new file mode 100644 index 00000000..1c04e7c7 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Http; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class TestServerMessageHandlerBuilder : HttpMessageHandlerBuilder + { + public TestServerMessageHandlerBuilder(TestServer testServer) + { + AdditionalHandlers = new List(); + PrimaryHandler = testServer.CreateHandler(); + } + + public override string Name { get; set; } + + public override HttpMessageHandler PrimaryHandler { get; set; } + + public override IList AdditionalHandlers { get; } + + public override HttpMessageHandler Build() => CreateHandlerPipeline(PrimaryHandler, AdditionalHandlers); + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs deleted file mode 100644 index d577beb3..00000000 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/UnitTest1.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using Xunit; - -namespace Fabrikam.Workflow.Service.Tests -{ - public class UnitTest1 - { - [Fact] - public void Test1() - { - - } - } -} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index 5cd88e11..fb10a795 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -7,6 +7,7 @@ + @@ -14,8 +15,10 @@ + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationRequired.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationRequired.cs new file mode 100644 index 00000000..04cacec5 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationRequired.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public enum ConfirmationRequired + { + FingerPrint, + Picture, + Voice, + None + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ContainerSize.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ContainerSize.cs new file mode 100644 index 00000000..27f09ebf --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ContainerSize.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public enum ContainerSize + { + Small, Medium, Large + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Delivery.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Delivery.cs new file mode 100644 index 00000000..af1324dd --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Delivery.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace Fabrikam.Workflow.Service.Models +{ + public class Delivery + { + public string DeliveryId { get; set; } + public string OwnerId { get; set; } + public string PickupLocation { get; set; } + public string DropoffLocation { get; set; } + public string Deadline { get; set; } + public bool Expedited { get; set; } + public ConfirmationRequired ConfirmationRequired { get; set; } + public DateTime PickupTime { get; set; } + public PackageInfo PackageInfo { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs new file mode 100644 index 00000000..192195f5 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class PackageGen + { + public string Id { get; set; } + public ContainerSize Size { get; set; } + public string Tag { get; set; } + public double Weight { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs new file mode 100644 index 00000000..4e4f9903 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class PackageInfo + { + public string PackageId { get; set; } + public ContainerSize Size { get; set; } + public double Weight { get; set; } + public string Tag { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs index 045fb066..cc77ca44 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -1,10 +1,14 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Serilog; using Serilog.Formatting.Compact; -using System.IO; -using System.Threading.Tasks; namespace Fabrikam.Workflow.Service { @@ -35,17 +39,11 @@ private static IHostBuilder CreateHostBuilder(string[] args) { var serilogBuilder = new LoggerConfiguration() .ReadFrom.Configuration(context.Configuration) - //.Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) .WriteTo.Console(new CompactJsonFormatter()); builder.AddSerilog(serilogBuilder.CreateLogger(), true); }) - .ConfigureServices((context, services) => - { - services.AddOptions(); - services.Configure(context.Configuration); - services.AddHostedService(); - }) + .ConfigureServices(ServiceStartup.ConfigureServices) .UseConsoleLifetime(); } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/IRequestProcessor.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/IRequestProcessor.cs new file mode 100644 index 00000000..9cc5f378 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/IRequestProcessor.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Collections.Generic; +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.RequestProcessing +{ + public interface IRequestProcessor + { + Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IReadOnlyDictionary properties); + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs new file mode 100644 index 00000000..7532321d --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Services; + +namespace Fabrikam.Workflow.Service.RequestProcessing +{ + public class RequestProcessor : IRequestProcessor + { + private readonly ILogger _logger; + private readonly IPackageServiceCaller _packageServiceCaller; + + public RequestProcessor(ILogger logger, IPackageServiceCaller packageServiceCaller) + { + _logger = logger; + _packageServiceCaller = packageServiceCaller; + } + + public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IReadOnlyDictionary properties) + { + _logger.LogInformation("Processing delivery request {deliveryId}", deliveryRequest.DeliveryId); + + try + { + var packageGen = await CreatePackageAsync(deliveryRequest.PackageInfo).ConfigureAwait(false); + if (packageGen != null) + { + _logger.LogInformation("Generated package {packageId} for delivery {deliveryId}", packageGen.Id, deliveryRequest.DeliveryId); + + return true; + } + } + catch (Exception e) + { + _logger.LogError(e, "Error processing delivery request {deliveryId}", deliveryRequest.DeliveryId); + } + + return false; + } + + private async Task CreatePackageAsync(PackageInfo packageInfo) + { + var packageGen = await _packageServiceCaller.CreatePackageAsync(packageInfo); + return packageGen; + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs new file mode 100644 index 00000000..dfdbdfa0 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Fabrikam.Workflow.Service.RequestProcessing; +using Fabrikam.Workflow.Service.Services; + +namespace Fabrikam.Workflow.Service +{ + public static class ServiceStartup + { + public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) + { + services.AddOptions(); + + services.Configure(context.Configuration); + services.AddHostedService(); + + services.AddTransient(); + + services.AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_PACKAGE"]); + }); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/BackendServiceCallFailedException.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/BackendServiceCallFailedException.cs new file mode 100644 index 00000000..48c71d8c --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/BackendServiceCallFailedException.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace Fabrikam.Workflow.Service.Services +{ + public class BackendServiceCallFailedException : Exception + { + public BackendServiceCallFailedException() + { + } + + public BackendServiceCallFailedException(string message) : base(message) + { + } + + public BackendServiceCallFailedException(string message, Exception innerException) : base(message, innerException) + { + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs new file mode 100644 index 00000000..0c394c93 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Services +{ + public interface IPackageServiceCaller + { + Task CreatePackageAsync(PackageInfo packageInfo); + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs new file mode 100644 index 00000000..8ff13a78 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Net; +using System.Net.Http; +using System.Net.Http.Formatting; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Services +{ + public class PackageServiceCaller : IPackageServiceCaller + { + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + + public PackageServiceCaller(ILogger logger, HttpClient httpClient) + { + _logger = logger; + _httpClient = httpClient; + } + + public async Task CreatePackageAsync(PackageInfo packageInfo) + { + try + { + var response = await _httpClient.PutAsync($"{packageInfo.PackageId}", packageInfo, new JsonMediaTypeFormatter()); + if (response.StatusCode == HttpStatusCode.Created) + { + return await response.Content.ReadAsAsync(); + } + + throw new BackendServiceCallFailedException(response.ReasonPhrase); + } + catch (BackendServiceCallFailedException) + { + throw; + } + catch (Exception e) + { + throw new BackendServiceCallFailedException(e.Message, e); + } + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs index 8ce08e0a..e1ebd215 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs @@ -1,35 +1,67 @@ -using Microsoft.Azure.ServiceBus; +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.ObjectModel; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.ServiceBus; using Microsoft.Azure.ServiceBus.Primitives; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using System; -using System.Threading; -using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.RequestProcessing; +using Newtonsoft.Json; namespace Fabrikam.Workflow.Service { internal class WorkflowService : IHostedService { + private readonly JsonSerializer _serializer; + private readonly ILogger _logger; + private readonly IRequestProcessor _requestProcessor; + private readonly Func, IQueueClient> _createQueueClient; private readonly IOptions _options; - private QueueClient _receiveClient; + private IQueueClient _receiveClient; - public WorkflowService(ILogger logger, IOptions options) + public WorkflowService(IOptions options, ILogger logger, IRequestProcessor requestProcessor) + : this(options, logger, requestProcessor, CreateQueueClient) + { } + + public WorkflowService(IOptions options, ILogger logger, IRequestProcessor requestProcessor, Func, IQueueClient> createQueueClient) { - _logger = logger; _options = options; + _logger = logger; + _requestProcessor = requestProcessor; + _createQueueClient = createQueueClient; + + _serializer = new JsonSerializer(); } - public Task StartAsync(CancellationToken cancellationToken) + private static IQueueClient CreateQueueClient(IOptions options) { TokenProvider tokenProvider = TokenProvider.CreateManagedServiceIdentityTokenProvider(); - _receiveClient = new QueueClient(_options.Value.QueueEndpoint, _options.Value.QueueName, tokenProvider, receiveMode: ReceiveMode.PeekLock); + return new QueueClient(options.Value.QueueEndpoint, options.Value.QueueName, tokenProvider, receiveMode: ReceiveMode.PeekLock, transportType: TransportType.Amqp) + { + PrefetchCount = options.Value.PrefetchCount + }; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _receiveClient = _createQueueClient(_options); + _receiveClient.RegisterMessageHandler( ProcessMessageAsync, new MessageHandlerOptions(ProcessMessageExceptionAsync) { - AutoComplete = true, + AutoComplete = false, MaxConcurrentCalls = _options.Value.MaxConcurrency }); @@ -44,18 +76,65 @@ public async Task StopAsync(CancellationToken cancellationToken) _logger.LogInformation("Stopped"); } - private Task ProcessMessageAsync(Message message, CancellationToken ct) + private async Task ProcessMessageAsync(Message message, CancellationToken ct) { _logger.LogInformation("Processing message {messageId}", message.MessageId); - return Task.CompletedTask; + if (TryGetDelivery(message, out var delivery)) + { + try + { + if (await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new ReadOnlyDictionary(message.UserProperties))) + { + await _receiveClient.CompleteAsync(message.SystemProperties.LockToken); + return; + } + } + catch (Exception e) + { + _logger.LogError(e, "Error processing message {messageId}", message.MessageId); + } + } + + try + { + await _receiveClient.DeadLetterAsync(message.SystemProperties.LockToken); + } + catch (Exception e) + { + _logger.LogError(e, "Error moving message {messageId} to dead letter queue", message.MessageId); + } + + return; } private Task ProcessMessageExceptionAsync(ExceptionReceivedEventArgs exceptionEvent) { - _logger.LogWarning("Exception processing message {exception}", exceptionEvent.Exception); + _logger.LogError(exceptionEvent.Exception, "Exception processing message"); return Task.CompletedTask; } + + private bool TryGetDelivery(Message message, out Delivery delivery) + { + try + { + using (var payloadStream = new MemoryStream(message.Body, false)) + using (var streamReader = new StreamReader(payloadStream, Encoding.UTF8)) + using (var jsonReader = new JsonTextReader(streamReader)) + { + delivery = _serializer.Deserialize(jsonReader); + } + + return true; + } + catch (Exception e) + { + _logger.LogError(e, "Cannot parse payload from message {messageId}", message.MessageId); + } + + delivery = null; + return false; + } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs index e4ccf24f..c9bebec8 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs @@ -1,10 +1,16 @@ -namespace Fabrikam.Workflow.Service +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service { internal class WorkflowServiceOptions { public WorkflowServiceOptions() { MaxConcurrency = 10; + PrefetchCount = 10; } public string QueueEndpoint { get; set; } @@ -12,5 +18,7 @@ public WorkflowServiceOptions() public string QueueName { get; set; } public int MaxConcurrency { get; internal set; } + + public int PrefetchCount { get; internal set; } } } From 5c183522c6ee3adc2aad0e8b1c3162611aa3d1be Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 15 Jan 2019 15:53:38 +0000 Subject: [PATCH 056/246] Merged PR 637: Implement CI for the workflow service requires setting up the pipeline in Azure DevOps Related work items: #8379 --- charts/workflow/Chart.yaml | 4 + charts/workflow/templates/NOTES.txt | 10 ++ charts/workflow/templates/_helpers.tpl | 32 ++++ .../workflow/templates/workflow-deploy.yaml | 36 ++++ charts/workflow/values.yaml | 8 + src/shipping/workflow/Dockerfile | 18 +- .../RequestProcessorIntegrationTests.cs | 3 +- src/shipping/workflow/azure-pipelines-ci.yml | 157 ++++++++++++++++++ .../workflow/azure-pipelines-validation.yml | 56 +++++++ src/shipping/workflow/azure-pipelines.yml | 37 +++++ 10 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 charts/workflow/Chart.yaml create mode 100644 charts/workflow/templates/NOTES.txt create mode 100644 charts/workflow/templates/_helpers.tpl create mode 100644 charts/workflow/templates/workflow-deploy.yaml create mode 100644 charts/workflow/values.yaml create mode 100644 src/shipping/workflow/azure-pipelines-ci.yml create mode 100644 src/shipping/workflow/azure-pipelines-validation.yml create mode 100644 src/shipping/workflow/azure-pipelines.yml diff --git a/charts/workflow/Chart.yaml b/charts/workflow/Chart.yaml new file mode 100644 index 00000000..60565da6 --- /dev/null +++ b/charts/workflow/Chart.yaml @@ -0,0 +1,4 @@ +name: workflow +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Workflow Service diff --git a/charts/workflow/templates/NOTES.txt b/charts/workflow/templates/NOTES.txt new file mode 100644 index 00000000..0dd034e8 --- /dev/null +++ b/charts/workflow/templates/NOTES.txt @@ -0,0 +1,10 @@ +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +All the objects were created in the namespace {{ .Values.namespace }} + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} + $ helm get {{ .Release.Name }} diff --git a/charts/workflow/templates/_helpers.tpl b/charts/workflow/templates/_helpers.tpl new file mode 100644 index 00000000..051ea827 --- /dev/null +++ b/charts/workflow/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "workflow.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "workflow.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "workflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml new file mode 100644 index 00000000..a47b1640 --- /dev/null +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "workflow.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ include "workflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "workflow.chart" . }} + annotations: + kubernetes.io/change-cause: {{ .Values.reason }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "workflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "workflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "workflow.chart" . }} + spec: + securityContext: + fsGroup: 1 + containers: + - name: workflowservice + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} \ No newline at end of file diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml new file mode 100644 index 00000000..6882f354 --- /dev/null +++ b/charts/workflow/values.yaml @@ -0,0 +1,8 @@ +# Default values for workflow. +replicaCount: 1 +image: + repository: + tag: + pullPolicy: IfNotPresent +dockerregistry: +reason: unknown diff --git a/src/shipping/workflow/Dockerfile b/src/shipping/workflow/Dockerfile index 6bbd9bcf..b3f6f0b1 100644 --- a/src/shipping/workflow/Dockerfile +++ b/src/shipping/workflow/Dockerfile @@ -2,13 +2,23 @@ FROM microsoft/dotnet:2.2-runtime AS base WORKDIR /app FROM microsoft/dotnet:2.2-sdk AS build -WORKDIR /src -COPY Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj Fabrikam.Workflow.Service/ -RUN dotnet restore Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj -COPY . . WORKDIR /src/Fabrikam.Workflow.Service + +COPY Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj . +RUN dotnet restore Fabrikam.Workflow.Service.csproj + +COPY Fabrikam.Workflow.Service/. . RUN dotnet build Fabrikam.Workflow.Service.csproj -c Release -o /app +FROM build AS testrunner +WORKDIR /src/tests + +COPY Fabrikam.Workflow.Service.Tests/*.csproj . +RUN dotnet restore Fabrikam.Workflow.Service.Tests.csproj + +COPY Fabrikam.Workflow.Service.Tests/. . +ENTRYPOINT ["dotnet", "test", "--logger:trx"] + FROM build AS publish RUN dotnet publish Fabrikam.Workflow.Service.csproj -c Release -o /app diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs index ac89fb40..7631840c 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -154,9 +154,10 @@ await ctx.WriteResultAsync( [Fact] public async Task WhenPackageServiceFails_ThenRequestFails() { - _handleHttpRequest = async ctx => + _handleHttpRequest = ctx => { ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + return Task.CompletedTask; }; var delivery = diff --git a/src/shipping/workflow/azure-pipelines-ci.yml b/src/shipping/workflow/azure-pipelines-ci.yml new file mode 100644 index 00000000..1c9e4e97 --- /dev/null +++ b/src/shipping/workflow/azure-pipelines-ci.yml @@ -0,0 +1,157 @@ +steps: +- script: echo '##vso[task.setvariable variable=repositoryName]workflow' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- script: echo '##vso[task.setvariable variable=chartPath]charts/workflow' + name: setvarChartPath +- script: echo $(chartPath) + name: echovarChartPath + +- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/workflow/Dockerfile' + name: setvarDockerFileName +- script: echo $(dockerFileName) + name: echovarDockerFileName + +- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' + name: setvarReleaseVersion +- script: echo $(releaseVersion) + name: echovarReleaseVersion + +- script: echo '##vso[task.setvariable variable=aksNamespace]backend' + name: setvarAKSNamespace +- script: echo $(aksNamespace) + name: echovarAKSNamespace + +- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' + name: setvarImageName +- script: echo $(imageName) + name: echovarImageName + +- task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + +- task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/src/tests/TestResults' + + imageName: '$(imageName)-test' + + runInBackground: false + +- task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/*.trx' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + +- task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + +- task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + +- task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.9.1' + inputs: + kubectlVersion: 1.10.3 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + +- task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(AzureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + force='--force' + packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + + export force + export packageExists + + if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then + force='' + fi + + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; diff --git a/src/shipping/workflow/azure-pipelines-validation.yml b/src/shipping/workflow/azure-pipelines-validation.yml new file mode 100644 index 00000000..42456246 --- /dev/null +++ b/src/shipping/workflow/azure-pipelines-validation.yml @@ -0,0 +1,56 @@ +variables: + BuildConfiguration: "Release" + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + branches: + include: + + - master + + - release/workflow/v* # for bug fixes + + paths: + include: + + - /src/shipping/workflow/ + +trigger: + batch: true + branches: + include: + + - master + + - feature/* + + - topic/* + + exclude: + + - feature/experimental/* + + - topic/experimental/* + + paths: + include: + + - /src/shipping/workflow/ + +resources: +- repo: self + +jobs: + +# CI +- job: workflowcivalidationjob + displayName: "Workflow CI Validation (build & test)" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: False + buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] + steps: + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/workflow/azure-pipelines.yml b/src/shipping/workflow/azure-pipelines.yml new file mode 100644 index 00000000..bf3e647a --- /dev/null +++ b/src/shipping/workflow/azure-pipelines.yml @@ -0,0 +1,37 @@ +variables: + BuildConfiguration: "Release" + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +trigger: + batch: true + branches: + include: + + # for new release to production: release flow strategy + - release/workflow/v* + + - refs/relelase/workflow/v* + + paths: + include: + + - /src/shipping/workflow/ + +resources: +- repo: self + +jobs: + +# CI +- job: workflowjobci + displayName: "Workflow CI" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/workflow/v') ] + steps: + - script: echo $(fullCI) + name: echovar + - template: ./azure-pipelines-ci.yml From f0a9e2d3eae29787a13f9dd0bc04a33d76b239cc Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 21 Jan 2019 19:07:39 +0000 Subject: [PATCH 057/246] Merged PR 638: Add call to drone scheduler from workflow Add call to drone scheduler from workflow Add tests for new caller Refactor integration tests to be closer to the caller. Related work items: #8370 --- ...eSchedulerServiceCallerIntegrationTests.cs | 176 ++++++++++++++++++ .../PackageServiceCallerIntegrationTests.cs | 170 +++++++++++++++++ .../RequestProcessorIntegrationTests.cs | 93 +++------ .../RequestProcessorTests.cs | 89 ++++++--- .../{ => Utils}/HttpContextTestExtensions.cs | 2 +- .../TestServerMessageHandlerBuilder.cs | 2 +- .../Models/DroneDelivery.cs | 16 ++ .../Models/Location.cs | 14 ++ .../Models/PackageDetail.cs | 13 ++ .../Models/PackageSize.cs | 12 ++ .../RequestProcessing/RequestProcessor.cs | 22 ++- .../ServiceStartup.cs | 5 + .../Services/DroneSchedulerServiceCaller.cs | 62 ++++++ .../Services/IDroneSchedulerServiceCaller.cs | 15 ++ .../Services/PackageServiceCaller.cs | 8 +- .../Utils/LocationRandomizer.cs | 28 +++ .../Utils/ModelsConverter.cs | 28 +++ 17 files changed, 650 insertions(+), 105 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs rename src/shipping/workflow/Fabrikam.Workflow.Service.Tests/{ => Utils}/HttpContextTestExtensions.cs (97%) rename src/shipping/workflow/Fabrikam.Workflow.Service.Tests/{ => Utils}/TestServerMessageHandlerBuilder.cs (95%) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/DroneDelivery.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/Location.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageDetail.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageSize.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/DroneSchedulerServiceCaller.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDroneSchedulerServiceCaller.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Utils/LocationRandomizer.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Utils/ModelsConverter.cs diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs new file mode 100644 index 00000000..5bf58f2e --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs @@ -0,0 +1,176 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Services; +using Fabrikam.Workflow.Service.Tests.Utils; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class DroneSchedulerServiceCallerIntegrationTests : IDisposable + { + private const string DroneSchedulerHost = "dronescheduler"; + private static readonly string DroneSchedulerUri = $"http://{DroneSchedulerHost}/api/DroneDeliveries/"; + + private readonly TestServer _testServer; + private RequestDelegate _handleHttpRequest = ctx => Task.CompletedTask; + + private readonly IDroneSchedulerServiceCaller _caller; + + public DroneSchedulerServiceCallerIntegrationTests() + { + var context = new HostBuilderContext(new Dictionary()); + context.Configuration = + new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + ["SERVICE_URI_DRONE"] = DroneSchedulerUri + }).Build(); + context.HostingEnvironment = + Mock.Of(e => e.EnvironmentName == "Test"); + + var serviceCollection = new ServiceCollection(); + ServiceStartup.ConfigureServices(context, serviceCollection); + serviceCollection.AddLogging(builder => builder.AddDebug()); + + _testServer = + new TestServer( + new WebHostBuilder() + .Configure(builder => + { + builder.UseMvc(); + builder.Run(ctx => _handleHttpRequest(ctx)); + }) + .ConfigureServices(builder => + { + builder.AddMvc(); + })); + + serviceCollection.Replace( + ServiceDescriptor.Transient( + sp => new TestServerMessageHandlerBuilder(_testServer))); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + _caller = serviceProvider.GetService(); + } + + public void Dispose() + { + _testServer.Dispose(); + } + + [Fact] + public async Task WhenGettingDroneId_ThenInvokesDroneSchedulerAPI() + { + string actualDeliveryId = null; + DroneDelivery actualDelivery = null; + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == DroneSchedulerHost) + { + actualDeliveryId = ctx.Request.Path; + actualDelivery = + new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + await _caller.GetDroneIdAsync(delivery); + + Assert.NotNull(actualDeliveryId); + Assert.Equal($"/api/DroneDeliveries/{delivery.DeliveryId}", actualDeliveryId); + + Assert.NotNull(actualDelivery); + Assert.Equal(delivery.DeliveryId, actualDelivery.DeliveryId); + Assert.Equal(delivery.PackageInfo.PackageId, actualDelivery.PackageDetail.Id); + Assert.Equal((int)delivery.PackageInfo.Size, (int)actualDelivery.PackageDetail.Size); + } + + [Fact] + public async Task WhenDroneSchedulerAPIReturnsOK_ThenReturnsDroneId() + { + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == DroneSchedulerHost) + { + await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = (int)HttpStatusCode.OK }); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var actualDroneId = await _caller.GetDroneIdAsync(delivery); + + Assert.Equal("someDroneId", actualDroneId); + } + + [Fact] + public async Task WhenDroneSchedulerAPIDoesNotReturnOK_ThenThrows() + { + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == DroneSchedulerHost) + { + ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + + await Assert.ThrowsAsync(() => _caller.GetDroneIdAsync(delivery)); + } + } +} + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs new file mode 100644 index 00000000..14cdc302 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs @@ -0,0 +1,170 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Services; +using Fabrikam.Workflow.Service.Tests.Utils; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class PackageServiceCallerIntegrationTests : IDisposable + { + private const string PackageHost = "packagehost"; + private static readonly string PackageUri = $"http://{PackageHost}/api/packages/"; + + private readonly TestServer _testServer; + private RequestDelegate _handleHttpRequest = ctx => Task.CompletedTask; + + private readonly IPackageServiceCaller _caller; + + public PackageServiceCallerIntegrationTests() + { + var context = new HostBuilderContext(new Dictionary()); + context.Configuration = + new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + ["SERVICE_URI_PACKAGE"] = PackageUri + }).Build(); + context.HostingEnvironment = + Mock.Of(e => e.EnvironmentName == "Test"); + + var serviceCollection = new ServiceCollection(); + ServiceStartup.ConfigureServices(context, serviceCollection); + serviceCollection.AddLogging(builder => builder.AddDebug()); + + _testServer = + new TestServer( + new WebHostBuilder() + .Configure(builder => + { + builder.UseMvc(); + builder.Run(ctx => _handleHttpRequest(ctx)); + }) + .ConfigureServices(builder => + { + builder.AddMvc(); + })); + + serviceCollection.Replace( + ServiceDescriptor.Transient( + sp => new TestServerMessageHandlerBuilder(_testServer))); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + _caller = serviceProvider.GetService(); + } + + public void Dispose() + { + _testServer.Dispose(); + } + + [Fact] + public async Task WhenCreatingPackage_ThenInvokesDroneSchedulerAPI() + { + string actualPackageId = null; + PackageGen actualPackage = null; + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + actualPackageId = ctx.Request.Path; + actualPackage = + new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + ctx.Response.StatusCode = (int)HttpStatusCode.Created; + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; + await _caller.CreatePackageAsync(packageInfo); + + Assert.NotNull(actualPackageId); + Assert.Equal($"/api/packages/{packageInfo.PackageId}", actualPackageId); + + Assert.NotNull(actualPackage); + Assert.Equal((int)packageInfo.Size, (int)actualPackage.Size); + Assert.Equal(packageInfo.Tag, actualPackage.Tag); + Assert.Equal(packageInfo.Weight, actualPackage.Weight); + } + + [Fact] + public async Task WhenPackageAPIReturnsOK_ThenReturnsGeneratedPackage() + { + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = (int)HttpStatusCode.Created + }); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + }; + + var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; + var actualPackage = await _caller.CreatePackageAsync(packageInfo); + + Assert.NotNull(actualPackage); + Assert.Equal((int)packageInfo.Size, (int)actualPackage.Size); + Assert.Equal(packageInfo.Tag, actualPackage.Tag); + Assert.Equal(packageInfo.Weight, actualPackage.Weight); + } + + [Fact] + public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() + { + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; + + await Assert.ThrowsAsync(() => _caller.CreatePackageAsync(packageInfo)); + } + } +} + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs index 7631840c..6a530f9a 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -22,29 +22,32 @@ using Microsoft.Extensions.Logging; using Fabrikam.Workflow.Service.Models; using Fabrikam.Workflow.Service.RequestProcessing; +using Fabrikam.Workflow.Service.Tests.Utils; using Moq; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Xunit; namespace Fabrikam.Workflow.Service.Tests { - public class RequestProcessorFixture : IDisposable + public class RequestProcessorIntegrationTests : IDisposable { - private const string PackageHost = "package"; + private const string DroneSchedulerHost = "dronescheduler"; + private static readonly string DroneSchedulerUri = $"http://{DroneSchedulerHost}/api/DroneDeliveries/"; + private const string PackageHost = "packagehost"; private static readonly string PackageUri = $"http://{PackageHost}/api/packages/"; private readonly IRequestProcessor _requestProcessor; private readonly TestServer _testServer; private RequestDelegate _handleHttpRequest = ctx => Task.CompletedTask; - public RequestProcessorFixture() + public RequestProcessorIntegrationTests() { var context = new HostBuilderContext(new Dictionary()); context.Configuration = new ConfigurationBuilder().AddInMemoryCollection( new Dictionary { + ["SERVICE_URI_DRONE"] = DroneSchedulerUri, ["SERVICE_URI_PACKAGE"] = PackageUri }).Build(); context.HostingEnvironment = @@ -81,58 +84,30 @@ public void Dispose() } [Fact] - public async Task ProcessingDelivery_InvokesPackageService() + public async Task ProcessingDelivery_InvokesPackageServiceAndDroneSchedulerService() { - JObject actualPackageInfo = null; + PackageGen actualPackage = null; + DroneDelivery actualDelivery = null; _handleHttpRequest = async ctx => { + var serializer = new JsonSerializer(); + if (ctx.Request.Host.Host == PackageHost) { - actualPackageInfo = await JObject.LoadAsync(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + actualPackage = serializer.Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); await ctx.WriteResultAsync( new ObjectResult( - new PackageGen { Id = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) { StatusCode = (int)HttpStatusCode.Created }); } - else - { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; - } - }; - - var delivery = - new Delivery + else if (ctx.Request.Host.Host == DroneSchedulerHost) { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } - }; - var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - - Assert.True(success); + actualDelivery = serializer.Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); - Assert.NotNull(actualPackageInfo); - Assert.Equal("package", actualPackageInfo["PackageId"].Value()); - Assert.Equal((int)ContainerSize.Medium, actualPackageInfo["Size"].Value()); - Assert.Equal("sometag", actualPackageInfo["Tag"].Value()); - Assert.Equal(100d, actualPackageInfo["Weight"].Value()); - } - - [Fact] - public async Task WhenPackageServiceSucceeds_ThenRequestSucceeds() - { - _handleHttpRequest = async ctx => - { - if (ctx.Request.Host.Host == PackageHost) - { - await ctx.WriteResultAsync( - new ObjectResult( - new PackageGen { Id = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) - { - StatusCode = (int)HttpStatusCode.Created - }); + await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = (int)HttpStatusCode.OK }); } else { @@ -143,32 +118,20 @@ await ctx.WriteResultAsync( var delivery = new Delivery { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } }; - var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - - Assert.True(success); - } + await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - [Fact] - public async Task WhenPackageServiceFails_ThenRequestFails() - { - _handleHttpRequest = ctx => - { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; - return Task.CompletedTask; - }; - - var delivery = - new Delivery - { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } - }; - var success = await _requestProcessor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + Assert.NotNull(actualPackage); + Assert.Equal((int)delivery.PackageInfo.Size, (int)actualPackage.Size); + Assert.Equal(delivery.PackageInfo.Tag, actualPackage.Tag); + Assert.Equal(delivery.PackageInfo.Weight, actualPackage.Weight); - Assert.False(success); + Assert.NotNull(actualDelivery); + Assert.Equal(delivery.DeliveryId, actualDelivery.DeliveryId); + Assert.Equal(delivery.PackageInfo.PackageId, actualDelivery.PackageDetail.Id); + Assert.Equal((int)delivery.PackageInfo.Size, (int)actualDelivery.PackageDetail.Size); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs index f6c5f01e..1ac886f1 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs @@ -18,8 +18,9 @@ namespace Fabrikam.Workflow.Service.Tests { public class RequestProcessorTests { - private readonly Mock packageServiceCallerMock; - private readonly RequestProcessor processor; + private readonly Mock _packageServiceCallerMock; + private readonly Mock _droneSchedulerServiceCallerMock; + private readonly RequestProcessor _processor; public RequestProcessorTests() { @@ -27,63 +28,107 @@ public RequestProcessorTests() servicesBuilder.AddLogging(logging => logging.AddDebug()); var services = servicesBuilder.BuildServiceProvider(); - packageServiceCallerMock = new Mock(); + _packageServiceCallerMock = new Mock(); + _droneSchedulerServiceCallerMock = new Mock(); - processor = new RequestProcessor(services.GetService>(), packageServiceCallerMock.Object); + _processor = + new RequestProcessor( + services.GetService>(), + _packageServiceCallerMock.Object, + _droneSchedulerServiceCallerMock.Object); } [Fact] public async Task WhenInvokingPackageServiceThrows_ProcessingFails() { + _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); + var delivery = new Delivery { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); - - var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - - packageServiceCallerMock.Verify(); Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.VerifyNoOtherCalls(); } [Fact] public async Task WhenInvokingPackageServiceFails_ProcessingFails() { + _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(default(PackageGen)).Verifiable(); + var delivery = new Delivery { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(default(PackageGen)).Verifiable(); + Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.VerifyNoOtherCalls(); + } - var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + [Fact] + public async Task WhenInvokingDroneSchedulerThrows_ProcessingFails() + { + _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - packageServiceCallerMock.Verify(); Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.Verify(); } [Fact] - public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() + public async Task WhenInvokingDroneSchedulerFails_ProcessingFails() { + _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ReturnsAsync(default(string)).Verifiable(); + var delivery = new Delivery { - DeliveryId = "delivery", - PackageInfo = new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.Verify(); + } - packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + [Fact] + public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() + { + _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ReturnsAsync("droneId").Verifiable(); - var success = await processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); - packageServiceCallerMock.Verify(); Assert.True(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.Verify(); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/HttpContextTestExtensions.cs similarity index 97% rename from src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs rename to src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/HttpContextTestExtensions.cs index 77385e69..fd6e5036 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/HttpContextTestExtensions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/HttpContextTestExtensions.cs @@ -12,7 +12,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; -namespace Fabrikam.Workflow.Service.Tests +namespace Fabrikam.Workflow.Service.Tests.Utils { public static class HttpContextTestExtensions { diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/TestServerMessageHandlerBuilder.cs similarity index 95% rename from src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs rename to src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/TestServerMessageHandlerBuilder.cs index 1c04e7c7..dbd19a31 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/TestServerMessageHandlerBuilder.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Utils/TestServerMessageHandlerBuilder.cs @@ -8,7 +8,7 @@ using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Http; -namespace Fabrikam.Workflow.Service.Tests +namespace Fabrikam.Workflow.Service.Tests.Utils { public class TestServerMessageHandlerBuilder : HttpMessageHandlerBuilder { diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DroneDelivery.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DroneDelivery.cs new file mode 100644 index 00000000..3ebc8a35 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DroneDelivery.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class DroneDelivery + { + public string DeliveryId { get; set; } + public Location Pickup { get; set; } + public Location Dropoff { get; set; } + public PackageDetail PackageDetail { get; set; } + public bool Expedited { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Location.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Location.cs new file mode 100644 index 00000000..a47d7b1f --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/Location.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class Location + { + public double Altitude { get; set; } + public double Latitude { get; set; } + public double Longitude { get; set; } + } +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageDetail.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageDetail.cs new file mode 100644 index 00000000..56579ddc --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageDetail.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class PackageDetail + { + public string Id { get; set; } + public PackageSize Size { get; set; } + } +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageSize.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageSize.cs new file mode 100644 index 00000000..e5799586 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageSize.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public enum PackageSize + { + Small, Medium, Large + } +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs index 7532321d..c6ecb110 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs @@ -16,11 +16,13 @@ public class RequestProcessor : IRequestProcessor { private readonly ILogger _logger; private readonly IPackageServiceCaller _packageServiceCaller; + private readonly IDroneSchedulerServiceCaller _droneSchedulerServiceCaller; - public RequestProcessor(ILogger logger, IPackageServiceCaller packageServiceCaller) + public RequestProcessor(ILogger logger, IPackageServiceCaller packageServiceCaller, IDroneSchedulerServiceCaller droneSchedulerServiceCaller) { _logger = logger; _packageServiceCaller = packageServiceCaller; + _droneSchedulerServiceCaller = droneSchedulerServiceCaller; } public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IReadOnlyDictionary properties) @@ -29,12 +31,18 @@ public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IR try { - var packageGen = await CreatePackageAsync(deliveryRequest.PackageInfo).ConfigureAwait(false); + var packageGen = await _packageServiceCaller.CreatePackageAsync(deliveryRequest.PackageInfo).ConfigureAwait(false); if (packageGen != null) { _logger.LogInformation("Generated package {packageId} for delivery {deliveryId}", packageGen.Id, deliveryRequest.DeliveryId); - return true; + var droneId = await _droneSchedulerServiceCaller.GetDroneIdAsync(deliveryRequest).ConfigureAwait(false); + if (droneId != null) + { + _logger.LogInformation("Assigned drone {droneId} for delivery {deliveryId}", droneId, deliveryRequest.DeliveryId); + + return true; + } } } catch (Exception e) @@ -44,11 +52,5 @@ public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IR return false; } - - private async Task CreatePackageAsync(PackageInfo packageInfo) - { - var packageGen = await _packageServiceCaller.CreatePackageAsync(packageInfo); - return packageGen; - } } -} +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index dfdbdfa0..4a878e18 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -26,6 +26,11 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect { c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_PACKAGE"]); }); + + services.AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DRONE"]); + }); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DroneSchedulerServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DroneSchedulerServiceCaller.cs new file mode 100644 index 00000000..cb6124ea --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DroneSchedulerServiceCaller.cs @@ -0,0 +1,62 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Utils; + +namespace Fabrikam.Workflow.Service.Services +{ + public class DroneSchedulerServiceCaller : IDroneSchedulerServiceCaller + { + private readonly HttpClient _httpClient; + + public DroneSchedulerServiceCaller(HttpClient httpClient) + { + _httpClient = httpClient; + } + + public async Task GetDroneIdAsync(Delivery deliveryRequest) + { + try + { + var delivery = CreateDroneDelivery(deliveryRequest); + + var response = await _httpClient.PutAsJsonAsync($"{delivery.DeliveryId}", delivery); + if (response.StatusCode == HttpStatusCode.OK) + { + return await response.Content.ReadAsStringAsync(); + } + + throw new BackendServiceCallFailedException(response.ReasonPhrase); + } + catch (BackendServiceCallFailedException) + { + throw; + } + catch (Exception e) + { + throw new BackendServiceCallFailedException(e.Message, e); + } + } + + private DroneDelivery CreateDroneDelivery(Delivery deliveryRequest) + { + DroneDelivery delivery = new DroneDelivery(); + delivery.DeliveryId = deliveryRequest.DeliveryId; + + delivery.Dropoff = LocationRandomizer.GetRandomLocation(); + delivery.Pickup = LocationRandomizer.GetRandomLocation(); + + delivery.Expedited = delivery.Expedited; + delivery.PackageDetail = ModelsConverter.GetPackageDetail(deliveryRequest.PackageInfo); + + return delivery; + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDroneSchedulerServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDroneSchedulerServiceCaller.cs new file mode 100644 index 00000000..a3eafe77 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDroneSchedulerServiceCaller.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Services +{ + public interface IDroneSchedulerServiceCaller + { + Task GetDroneIdAsync(Delivery deliveryRequest); + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs index 8ff13a78..8d8dcba0 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs @@ -6,21 +6,17 @@ using System; using System.Net; using System.Net.Http; -using System.Net.Http.Formatting; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; using Fabrikam.Workflow.Service.Models; namespace Fabrikam.Workflow.Service.Services { public class PackageServiceCaller : IPackageServiceCaller { - private readonly ILogger _logger; private readonly HttpClient _httpClient; - public PackageServiceCaller(ILogger logger, HttpClient httpClient) + public PackageServiceCaller(HttpClient httpClient) { - _logger = logger; _httpClient = httpClient; } @@ -28,7 +24,7 @@ public async Task CreatePackageAsync(PackageInfo packageInfo) { try { - var response = await _httpClient.PutAsync($"{packageInfo.PackageId}", packageInfo, new JsonMediaTypeFormatter()); + var response = await _httpClient.PutAsJsonAsync($"{packageInfo.PackageId}", packageInfo); if (response.StatusCode == HttpStatusCode.Created) { return await response.Content.ReadAsAsync(); diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/LocationRandomizer.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/LocationRandomizer.cs new file mode 100644 index 00000000..d1c18c49 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/LocationRandomizer.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Utils +{ + static class LocationRandomizer + { + private static Random Random = new Random(); + + public static Location GetRandomLocation() + { + Location location = new Location(); + lock (Random) + { + location.Altitude = Random.NextDouble(); + location.Latitude = Random.NextDouble(); + location.Longitude = Random.NextDouble(); + } + + return location; + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/ModelsConverter.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/ModelsConverter.cs new file mode 100644 index 00000000..98e9c764 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Utils/ModelsConverter.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Utils +{ + class ModelsConverter + { + internal static PackageDetail GetPackageDetail(PackageInfo packageInfo) + { + var packageDetail = new PackageDetail + { + Id = packageInfo.PackageId, + Size = GetPackageSize(packageInfo.Size) + }; + + return packageDetail; + } + + private static PackageSize GetPackageSize(ContainerSize containerSize) + { + return (PackageSize)(int)containerSize; + } + } +} From a085b37e224b5ac75dae101130646934979ad260 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 22 Jan 2019 16:39:41 +0000 Subject: [PATCH 058/246] Merged PR 639: add call to delivery service from workflow add call to delivery service from workflow Related work items: #8371 --- .../DeliveryServiceCallerIntegrationTests.cs | 184 ++++++++++++++++++ .../RequestProcessorIntegrationTests.cs | 18 ++ .../RequestProcessorTests.cs | 99 +++++++++- .../Models/ConfirmationType.cs | 15 ++ .../Models/DeliverySchedule.cs | 19 ++ .../Models/UserAccount.cs | 13 ++ .../RequestProcessing/RequestProcessor.cs | 19 +- .../ServiceStartup.cs | 5 + .../Services/DeliveryServiceCaller.cs | 65 +++++++ .../Services/IDeliveryServiceCaller.cs | 15 ++ 10 files changed, 441 insertions(+), 11 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationType.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/DeliverySchedule.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Models/UserAccount.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/DeliveryServiceCaller.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDeliveryServiceCaller.cs diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs new file mode 100644 index 00000000..abfb05da --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs @@ -0,0 +1,184 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Services; +using Fabrikam.Workflow.Service.Tests.Utils; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class DeliveryServiceCallerIntegrationTests : IDisposable + { + private const string DeliveryHost = "deliveryhost"; + private static readonly string DeliveryUri = $"http://{DeliveryHost}/api/Deliveries/"; + + private readonly TestServer _testServer; + private RequestDelegate _handleHttpRequest = ctx => Task.CompletedTask; + + private readonly IDeliveryServiceCaller _caller; + + public DeliveryServiceCallerIntegrationTests() + { + var context = new HostBuilderContext(new Dictionary()); + context.Configuration = + new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + ["SERVICE_URI_DELIVERY"] = DeliveryUri + }).Build(); + context.HostingEnvironment = + Mock.Of(e => e.EnvironmentName == "Test"); + + var serviceCollection = new ServiceCollection(); + ServiceStartup.ConfigureServices(context, serviceCollection); + serviceCollection.AddLogging(builder => builder.AddDebug()); + + _testServer = + new TestServer( + new WebHostBuilder() + .Configure(builder => + { + builder.UseMvc(); + builder.Run(ctx => _handleHttpRequest(ctx)); + }) + .ConfigureServices(builder => + { + builder.AddMvc(); + })); + + serviceCollection.Replace( + ServiceDescriptor.Transient( + sp => new TestServerMessageHandlerBuilder(_testServer))); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + _caller = serviceProvider.GetService(); + } + + public void Dispose() + { + _testServer.Dispose(); + } + + [Fact] + public async Task WhenSchedulingDelivery_ThenInvokesDeliveryAPI() + { + string actualDeliveryId = null; + DeliverySchedule actualDeliverySchedule = null; + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == DeliveryHost) + { + actualDeliveryId = ctx.Request.Path; + actualDeliverySchedule = + new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + ctx.Response.StatusCode = (int)HttpStatusCode.Created; + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + await _caller.ScheduleDeliveryAsync(delivery, "someDroneId"); + + Assert.NotNull(actualDeliveryId); + Assert.Equal($"/api/Deliveries/{delivery.DeliveryId}", actualDeliveryId); + + Assert.NotNull(actualDeliverySchedule); + Assert.Equal(delivery.DeliveryId, actualDeliverySchedule.Id); + Assert.Equal((int)delivery.ConfirmationRequired, (int)actualDeliverySchedule.ConfirmationRequired); + Assert.Equal(delivery.Expedited, actualDeliverySchedule.Expedited); + Assert.Equal(delivery.OwnerId, actualDeliverySchedule.Owner.UserId); + Assert.Equal("someDroneId", actualDeliverySchedule.DroneId); + } + + [Fact] + public async Task WhenDeliveryAPIReturnsCreated_ThenReturnsGeneratedSchedule() + { + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == DeliveryHost) + { + await ctx.WriteResultAsync( + new ObjectResult( + new DeliverySchedule { Id = "someDeliveryId" }) + { + StatusCode = (int)HttpStatusCode.Created + }); + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var actualDeliverySchedule = await _caller.ScheduleDeliveryAsync(delivery, "someDroneId"); + + Assert.NotNull(actualDeliverySchedule); + Assert.Equal("someDeliveryId", actualDeliverySchedule.Id); + } + + [Fact] + public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() + { + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == DeliveryHost) + { + ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + } + else + { + ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + } + + return Task.CompletedTask; + }; + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + + await Assert.ThrowsAsync(() => _caller.ScheduleDeliveryAsync(delivery, "someDroneId")); + } + } +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs index 6a530f9a..d1ed71ac 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -31,6 +31,8 @@ namespace Fabrikam.Workflow.Service.Tests { public class RequestProcessorIntegrationTests : IDisposable { + private const string DeliveryHost = "deliveryhost"; + private static readonly string DeliveryUri = $"http://{DeliveryHost}/api/Deliveries/"; private const string DroneSchedulerHost = "dronescheduler"; private static readonly string DroneSchedulerUri = $"http://{DroneSchedulerHost}/api/DroneDeliveries/"; private const string PackageHost = "packagehost"; @@ -47,6 +49,7 @@ public RequestProcessorIntegrationTests() new ConfigurationBuilder().AddInMemoryCollection( new Dictionary { + ["SERVICE_URI_DELIVERY"] = DeliveryUri, ["SERVICE_URI_DRONE"] = DroneSchedulerUri, ["SERVICE_URI_PACKAGE"] = PackageUri }).Build(); @@ -88,6 +91,7 @@ public async Task ProcessingDelivery_InvokesPackageServiceAndDroneSchedulerServi { PackageGen actualPackage = null; DroneDelivery actualDelivery = null; + DeliverySchedule actualDeliverySchedule = null; _handleHttpRequest = async ctx => { var serializer = new JsonSerializer(); @@ -109,6 +113,16 @@ await ctx.WriteResultAsync( await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = (int)HttpStatusCode.OK }); } + else if (ctx.Request.Host.Host == DeliveryHost) + { + actualDeliverySchedule = serializer.Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); + + await ctx.WriteResultAsync( + new ObjectResult(new DeliverySchedule { Id = "someDeliveryId" }) + { + StatusCode = (int)HttpStatusCode.Created + }); + } else { ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; @@ -132,6 +146,10 @@ await ctx.WriteResultAsync( Assert.Equal(delivery.DeliveryId, actualDelivery.DeliveryId); Assert.Equal(delivery.PackageInfo.PackageId, actualDelivery.PackageDetail.Id); Assert.Equal((int)delivery.PackageInfo.Size, (int)actualDelivery.PackageDetail.Size); + + Assert.NotNull(actualDeliverySchedule); + Assert.Equal(delivery.DeliveryId, actualDeliverySchedule.Id); + Assert.Equal("someDroneId", actualDeliverySchedule.DroneId); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs index 1ac886f1..99b6b2eb 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs @@ -20,6 +20,7 @@ public class RequestProcessorTests { private readonly Mock _packageServiceCallerMock; private readonly Mock _droneSchedulerServiceCallerMock; + private readonly Mock _deliveryServiceCallerMock; private readonly RequestProcessor _processor; public RequestProcessorTests() @@ -30,18 +31,22 @@ public RequestProcessorTests() _packageServiceCallerMock = new Mock(); _droneSchedulerServiceCallerMock = new Mock(); + _deliveryServiceCallerMock = new Mock(); _processor = new RequestProcessor( services.GetService>(), _packageServiceCallerMock.Object, - _droneSchedulerServiceCallerMock.Object); + _droneSchedulerServiceCallerMock.Object, + _deliveryServiceCallerMock.Object); } [Fact] public async Task WhenInvokingPackageServiceThrows_ProcessingFails() { - _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ThrowsAsync(new Exception()).Verifiable(); var delivery = new Delivery @@ -54,12 +59,15 @@ public async Task WhenInvokingPackageServiceThrows_ProcessingFails() Assert.False(success); _packageServiceCallerMock.Verify(); _droneSchedulerServiceCallerMock.VerifyNoOtherCalls(); + _deliveryServiceCallerMock.VerifyNoOtherCalls(); } [Fact] public async Task WhenInvokingPackageServiceFails_ProcessingFails() { - _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(default(PackageGen)).Verifiable(); + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(default(PackageGen)).Verifiable(); var delivery = new Delivery @@ -72,13 +80,18 @@ public async Task WhenInvokingPackageServiceFails_ProcessingFails() Assert.False(success); _packageServiceCallerMock.Verify(); _droneSchedulerServiceCallerMock.VerifyNoOtherCalls(); + _deliveryServiceCallerMock.VerifyNoOtherCalls(); } [Fact] public async Task WhenInvokingDroneSchedulerThrows_ProcessingFails() { - _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); - _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ThrowsAsync(new Exception()).Verifiable(); + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock + .Setup(c => c.GetDroneIdAsync(It.IsAny())) + .ThrowsAsync(new Exception()).Verifiable(); var delivery = new Delivery @@ -91,13 +104,18 @@ public async Task WhenInvokingDroneSchedulerThrows_ProcessingFails() Assert.False(success); _packageServiceCallerMock.Verify(); _droneSchedulerServiceCallerMock.Verify(); + _deliveryServiceCallerMock.VerifyNoOtherCalls(); } [Fact] public async Task WhenInvokingDroneSchedulerFails_ProcessingFails() { - _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); - _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ReturnsAsync(default(string)).Verifiable(); + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock + .Setup(c => c.GetDroneIdAsync(It.IsAny())) + .ReturnsAsync(default(string)).Verifiable(); var delivery = new Delivery @@ -110,13 +128,75 @@ public async Task WhenInvokingDroneSchedulerFails_ProcessingFails() Assert.False(success); _packageServiceCallerMock.Verify(); _droneSchedulerServiceCallerMock.Verify(); + _deliveryServiceCallerMock.VerifyNoOtherCalls(); + } + + [Fact] + public async Task WhenInvokingDeliverySchedulerThrows_ProcessingFails() + { + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock + .Setup(c => c.GetDroneIdAsync(It.IsAny())) + .ReturnsAsync("droneId").Verifiable(); + _deliveryServiceCallerMock + .Setup(c => c.ScheduleDeliveryAsync(It.IsAny(), "droneId")) + .ThrowsAsync(new Exception()).Verifiable(); + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.Verify(); + _deliveryServiceCallerMock.Verify(); + } + + [Fact] + public async Task WhenInvokingDeliverySchedulerFails_ProcessingFails() + { + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock + .Setup(c => c.GetDroneIdAsync(It.IsAny())) + .ReturnsAsync("droneId").Verifiable(); + _deliveryServiceCallerMock + .Setup(c => c.ScheduleDeliveryAsync(It.IsAny(), "droneId")) + .ReturnsAsync(default(DeliverySchedule)).Verifiable(); + + var delivery = + new Delivery + { + DeliveryId = "someDeliveryId", + PackageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d } + }; + var success = await _processor.ProcessDeliveryRequestAsync(delivery, new Dictionary()); + + Assert.False(success); + _packageServiceCallerMock.Verify(); + _droneSchedulerServiceCallerMock.Verify(); + _deliveryServiceCallerMock.Verify(); } [Fact] public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() { - _packageServiceCallerMock.Setup(c => c.CreatePackageAsync(It.IsAny())).ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); - _droneSchedulerServiceCallerMock.Setup(c => c.GetDroneIdAsync(It.IsAny())).ReturnsAsync("droneId").Verifiable(); + _packageServiceCallerMock + .Setup(c => c.CreatePackageAsync(It.IsAny())) + .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); + _droneSchedulerServiceCallerMock + .Setup(c => c.GetDroneIdAsync(It.IsAny())) + .ReturnsAsync("droneId").Verifiable(); + _deliveryServiceCallerMock + .Setup(c => c.ScheduleDeliveryAsync(It.IsAny(), "droneId")) + .ReturnsAsync(new DeliverySchedule { Id = "someDeliveryId" }).Verifiable(); var delivery = new Delivery @@ -129,6 +209,7 @@ public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() Assert.True(success); _packageServiceCallerMock.Verify(); _droneSchedulerServiceCallerMock.Verify(); + _deliveryServiceCallerMock.Verify(); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationType.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationType.cs new file mode 100644 index 00000000..3c1f2646 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/ConfirmationType.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public enum ConfirmationType + { + FingerPrint, + Picture, + Voice, + None + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DeliverySchedule.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DeliverySchedule.cs new file mode 100644 index 00000000..a22f248c --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/DeliverySchedule.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class DeliverySchedule + { + public string Id { get; set; } + public UserAccount Owner { get; set; } + public Location Pickup { get; set; } + public Location Dropoff { get; set; } + public string Deadline { get; set; } + public bool Expedited { get; set; } + public ConfirmationType ConfirmationRequired { get; set; } + public string DroneId { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/UserAccount.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/UserAccount.cs new file mode 100644 index 00000000..7ffa6be2 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/UserAccount.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +namespace Fabrikam.Workflow.Service.Models +{ + public class UserAccount + { + public string UserId { get; set; } + public string AccountId { get; set; } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs index c6ecb110..e72163d0 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs @@ -17,12 +17,18 @@ public class RequestProcessor : IRequestProcessor private readonly ILogger _logger; private readonly IPackageServiceCaller _packageServiceCaller; private readonly IDroneSchedulerServiceCaller _droneSchedulerServiceCaller; + private readonly IDeliveryServiceCaller _deliveryServiceCaller; - public RequestProcessor(ILogger logger, IPackageServiceCaller packageServiceCaller, IDroneSchedulerServiceCaller droneSchedulerServiceCaller) + public RequestProcessor( + ILogger logger, + IPackageServiceCaller packageServiceCaller, + IDroneSchedulerServiceCaller droneSchedulerServiceCaller, + IDeliveryServiceCaller deliveryServiceCaller) { _logger = logger; _packageServiceCaller = packageServiceCaller; _droneSchedulerServiceCaller = droneSchedulerServiceCaller; + _deliveryServiceCaller = deliveryServiceCaller; } public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IReadOnlyDictionary properties) @@ -41,7 +47,16 @@ public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IR { _logger.LogInformation("Assigned drone {droneId} for delivery {deliveryId}", droneId, deliveryRequest.DeliveryId); - return true; + var deliverySchedule = await _deliveryServiceCaller.ScheduleDeliveryAsync(deliveryRequest, droneId); + if (deliverySchedule != null) + { + _logger.LogInformation("Completed delivery {deliveryId}", deliveryRequest.DeliveryId); + return true; + } + else + { + _logger.LogError("Failed delivery for request {deliveryId}", deliveryRequest.DeliveryId); + } } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index 4a878e18..2045cb33 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -31,6 +31,11 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect { c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DRONE"]); }); + + services.AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DELIVERY"]); + }); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DeliveryServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DeliveryServiceCaller.cs new file mode 100644 index 00000000..7dd4c8be --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/DeliveryServiceCaller.cs @@ -0,0 +1,65 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; +using Fabrikam.Workflow.Service.Utils; + +namespace Fabrikam.Workflow.Service.Services +{ + public class DeliveryServiceCaller : IDeliveryServiceCaller + { + private readonly HttpClient _httpClient; + + public DeliveryServiceCaller(HttpClient httpClient) + { + _httpClient = httpClient; + } + + public async Task ScheduleDeliveryAsync(Delivery deliveryRequest, string droneId) + { + try + { + var schedule = CreateDeliverySchedule(deliveryRequest, droneId); + + var response = await _httpClient.PutAsJsonAsync(schedule.Id, schedule); + if (response.StatusCode == HttpStatusCode.Created) + { + return await response.Content.ReadAsAsync(); + } + + throw new BackendServiceCallFailedException(response.ReasonPhrase); + } + catch (BackendServiceCallFailedException) + { + throw; + } + catch (Exception e) + { + throw new BackendServiceCallFailedException(e.Message, e); + } + } + + private DeliverySchedule CreateDeliverySchedule(Delivery deliveryRequest, string droneId) + { + DeliverySchedule scheduleDelivery = new DeliverySchedule + { + Id = deliveryRequest.DeliveryId, + Owner = new UserAccount { AccountId = Guid.NewGuid().ToString(), UserId = deliveryRequest.OwnerId }, + Pickup = LocationRandomizer.GetRandomLocation(), + Dropoff = LocationRandomizer.GetRandomLocation(), + Deadline = deliveryRequest.Deadline, + Expedited = deliveryRequest.Expedited, + ConfirmationRequired = (ConfirmationType)deliveryRequest.ConfirmationRequired, + DroneId = droneId, + }; + + return scheduleDelivery; + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDeliveryServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDeliveryServiceCaller.cs new file mode 100644 index 00000000..754bdfc8 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IDeliveryServiceCaller.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Threading.Tasks; +using Fabrikam.Workflow.Service.Models; + +namespace Fabrikam.Workflow.Service.Services +{ + public interface IDeliveryServiceCaller + { + Task ScheduleDeliveryAsync(Delivery deliveryRequest, string droneId); + } +} From ab7838ec53b99ccc4b60e34cf10daca9e6c835eb Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 25 Jan 2019 21:07:42 +0000 Subject: [PATCH 059/246] Merged PR 644: fix build context enforce the default context is not being used. --- src/shipping/dronescheduler/azure-pipelines-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shipping/dronescheduler/azure-pipelines-ci.yml b/src/shipping/dronescheduler/azure-pipelines-ci.yml index 7d838514..f61068cd 100644 --- a/src/shipping/dronescheduler/azure-pipelines-ci.yml +++ b/src/shipping/dronescheduler/azure-pipelines-ci.yml @@ -51,6 +51,8 @@ steps: buildContext: $(buildContext) + useDefaultContext: false + - task: Docker@1 condition: eq(variables['fullCI'],True) displayName: 'Push runtime image' From 2a5d686b5f99077cb0a14f31e933948dc7b2de27 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 25 Jan 2019 21:31:44 +0000 Subject: [PATCH 060/246] Merged PR 643: ingest messages into queue instead of eh change ingestion service to send messages to queue instead of event hub Related work items: #8535 --- deployment.md | 48 +++++-------------- k8s/dronescheduler.yaml | 4 +- k8s/ingestion.yaml | 16 +++---- k8s/workflow.yaml | 8 +++- .../Program.cs | 4 +- src/shipping/ingestion/pom.xml | 14 +----- .../configuration/ApplicationProperties.java | 26 +++++----- .../controller/IngestionController.java | 3 +- .../ingestion/service/IngestionImpl.java | 28 +++++------ .../ingestion/util/ClientPool.java | 10 ++-- .../ingestion/util/ClientPoolImpl.java | 37 +++++++------- .../src/main/resources/application.properties | 10 ++-- .../ingestion/ApplicationPropertiesTest.java | 6 +-- .../ingestion/IngestionControllerTest.java | 3 +- src/shipping/package/app/initializer.ts | 2 +- src/shipping/package/app/models/repository.ts | 2 +- 16 files changed, 93 insertions(+), 128 deletions(-) diff --git a/deployment.md b/deployment.md index 2486f551..8b8a9e6b 100644 --- a/deployment.md +++ b/deployment.md @@ -242,7 +242,7 @@ Deploy the Package service sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml # Create secret -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv) +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') kubectl -n backend create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION # Create KeyVault secret @@ -353,36 +353,14 @@ kubectl get pods -n backend Provision Azure resources ```bash -export INGESTION_EH_NS=[INGESTION_EVENT_HUB_NAMESPACE_HERE] -export INGESTION_EH_NAME=[INGESTION_EVENT_HUB_NAME_HERE] -export INGESTION_EH_CONSUMERGROUP_NAME=[INGESTION_EVENT_HUB_CONSUMERGROUP_NAME_HERE] - -# Create an Event Hubs namespace -az eventhubs namespace create --name $INGESTION_EH_NS \ - --resource-group $RESOURCE_GROUP \ - --location $LOCATION - -# Create an event hub -az eventhubs eventhub create --name $INGESTION_EH_NAME \ - --resource-group $RESOURCE_GROUP \ - --namespace-name $INGESTION_EH_NS \ - --partition-count 4 - -# Create consumer group -az eventhubs eventhub consumer-group create --eventhub-name $INGESTION_EH_NAME \ - --name $INGESTION_EH_CONSUMERGROUP_NAME \ - --namespace-name $INGESTION_EH_NS \ - --resource-group $RESOURCE_GROUP - -# Create authorization rule -az eventhubs eventhub authorization-rule create --eventhub-name $INGESTION_EH_NAME \ +# Create authorization rule to the ingestion queue +az servicebus namespace authorization-rule create --namespace-name $INGESTION_QUEUE_NAMESPACE \ --name IngestionServiceAccessKey \ - --namespace-name $INGESTION_EH_NS \ --resource-group $RESOURCE_GROUP \ - --rights Listen Send + --rights Send # Get access key -export EH_ACCESS_KEY_VALUE=$(az eventhubs eventhub authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_EH_NS --name IngestionServiceAccessKey --eventhub-name $INGESTION_EH_NAME --query primaryKey -o tsv) +export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name IngestionServiceAccessKey --query primaryKey -o tsv) ``` Build the Ingestion service @@ -410,18 +388,18 @@ sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/i # Create secret kubectl -n backend create secret generic ingestion-secrets \ ---from-literal=eventhub_namespace=${INGESTION_EH_NS} \ ---from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_keyname=IngestionServiceAccessKey \ ---from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} +--from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ +--from-literal=queue_name=${INGESTION_QUEUE_NAME} \ +--from-literal=queue_keyname=IngestionServiceAccessKey \ +--from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} # Create KeyVault secrets export INGESTION_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-ingestion-kv" az keyvault create --name $INGESTION_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-namespace --value ${INGESTION_EH_NS} -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-name --value ${INGESTION_EH_NAME} +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-namespace --value ${INGESTION_QUEUE_NAMESPACE} +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-name --value ${INGESTION_QUEUE_NAME} az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyname --value IngestionServiceAccessKey -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyvalue --value ${EH_ACCESS_KEY_VALUE} +az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyvalue --value ${INGESTION_ACCESS_KEY_VALUE} # Deploy service kubectl --namespace backend apply -f $K8S/ingestion-0.yaml @@ -469,7 +447,7 @@ You can send delivery requests to the ingestion service using the Swagger UI. Get the public IP address of the Ingestion Service: ```bash -export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") ``` Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index 82957c38..a4f82264 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -18,7 +18,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: 8080 selector: app: dronescheduler clusterIP: None @@ -52,7 +52,7 @@ spec: image: ports: - name: service - containerPort: 80 + containerPort: 8080 resources: requests: cpu: 1 diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index 21bd9bb7..bbe00441 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -62,23 +62,23 @@ spec: initialDelaySeconds: 30 periodSeconds: 20 env: - - name: EH_NAMESPACE + - name: QUEUE_NAMESPACE valueFrom: secretKeyRef: name: ingestion-secrets - key: eventhub_namespace - - name: EH_NAME + key: queue_namespace + - name: QUEUE_NAME valueFrom: secretKeyRef: name: ingestion-secrets - key: eventhub_name - - name: EH_KEYNAME + key: queue_name + - name: QUEUE_KEYNAME valueFrom: secretKeyRef: name: ingestion-secrets - key: eventhub_keyname - - name: EH_KEYVALUE + key: queue_keyname + - name: QUEUE_KEYVALUE valueFrom: secretKeyRef: name: ingestion-secrets - key: eventhub_keyvalue + key: queue_keyvalue diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 19e49935..a328a03d 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -15,7 +15,7 @@ metadata: version: 0.1.0 bc: shipping co: fabrikam - #aadpodidbinding: workflow + aadpodidbinding: workflow spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -44,6 +44,12 @@ spec: env: - name: CONFIGURATION_FOLDER value: /kvmnt + - name: SERVICE_URI_DELIVERY + value: http://delivery/api/Deliveries/ + - name: SERVICE_URI_DRONE + value: http://dronescheduler/api/DroneDeliveries/ + - name: SERVICE_URI_PACKAGE + value: http://package/api/packages/ - name: no_proxy value: 169.254.169.254 - name: CORRELATION_HEADER diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs index a906aa6c..4107979b 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs @@ -22,6 +22,8 @@ public static void Main(string[] args) public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) - .UseStartup(); + .UseStartup() + .UseUrls("http://0.0.0.0:8080"); + } } diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index ab779f5d..b49387e9 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -123,18 +123,8 @@ com.microsoft.azure - azure - 1.0.0 - - - com.microsoft.azure - azure-core - 0.9.7 - - - com.microsoft.azure - azure-eventhubs - 0.13.1 + azure-servicebus + 1.2.8 org.springframework.boot diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java index f8b46a6b..b0cd2f74 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java @@ -25,14 +25,14 @@ public class ApplicationProperties { // the properties are overriden by values // in application.properties - // Eventhub properties - private String namespace = "eventhubNamespace"; - private String eventHubName = "eventHubName"; + // Queue properties + private String namespace = "queueNamespace"; + private String queueName = "queueName"; private String sasKeyName = "sasKeyName"; private String sasKey = "sasKey"; - private String envNameSpace = "ENV_HUB_NS"; - private String envHubName = "ENV_HUB_NAME"; + private String envNameSpace = "ENV_QUEUE_NS"; + private String envQueueName = "ENV_QUEUE_NAME"; private String envsasKeyName = "ENV_KEY_NAME"; private String envsasKey = "ENV_KEY_VALUE"; @@ -56,12 +56,12 @@ public void setNamespace(String nameSpace) { this.namespace = nameSpace; } - public String getEventHubName() { - return eventHubName; + public String getQueueName() { + return queueName; } - public void setEventHubName(String eventHubName) { - this.eventHubName = eventHubName; + public void setQueueName(String queueName) { + this.queueName = queueName; } public String getSasKeyName() { @@ -142,12 +142,12 @@ public void setEnvNameSpace(String envNameSpace) { this.envNameSpace = envNameSpace; } - public String getEnvHubName() { - return envHubName; + public String getEnvQueueName() { + return envQueueName; } - public void setEnvHubName(String envHubName) { - this.envHubName = envHubName; + public void setEnvQueueName(String envQueueName) { + this.envQueueName = envQueueName; } public String getEnvsasKeyName() { diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java index 5fe5b38a..4d4898bd 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java @@ -27,9 +27,10 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; -import com.microsoft.azure.servicebus.ServiceBusException; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; import java.io.IOException; import com.fasterxml.jackson.core.JsonProcessingException; + import org.apache.logging.log4j.Logger; @RestController diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java index e4aee6fb..68962332 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java @@ -1,11 +1,9 @@ package com.fabrikam.dronedelivery.ingestion.service; import com.fabrikam.dronedelivery.ingestion.models.*; -import java.io.IOException; import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ExecutionException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -15,8 +13,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.microsoft.azure.eventhubs.EventData; -import com.microsoft.azure.servicebus.ServiceBusException; +import com.microsoft.azure.servicebus.Message; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; @Service public class IngestionImpl implements Ingestion { @@ -36,11 +34,11 @@ public IngestionImpl(ClientPool clientPool, ApplicationProperties appProps) { @Async @Override public void scheduleDeliveryAsync(DeliveryBase delivery, Map httpHeaders) { - EventData sendEvent = getEventData(delivery, httpHeaders); + Message sendEvent = getMessage(delivery, httpHeaders); sendEvent.getProperties().put("operation", "delivery"); try { - clientPool.getConnection().send(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ExecutionException | ServiceBusException | IOException e) { + clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); + } catch (InterruptedException | ServiceBusException e) { throw new RuntimeException(e); } } @@ -48,12 +46,12 @@ public void scheduleDeliveryAsync(DeliveryBase delivery, Map http @Async @Override public void cancelDeliveryAsync(String deliveryId, Map httpHeaders) { - EventData sendEvent = getEventData(deliveryId, httpHeaders); + Message sendEvent = getMessage(deliveryId, httpHeaders); sendEvent.getProperties().put("operation", "cancel"); try { - clientPool.getConnection().send(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ExecutionException | ServiceBusException | IOException e) { + clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); + } catch (InterruptedException | ServiceBusException e) { throw new RuntimeException(e); } @@ -62,19 +60,19 @@ public void cancelDeliveryAsync(String deliveryId, Map httpHeader @Async @Override public void rescheduleDeliveryAsync(DeliveryBase rescheduledDelivery, Map httpHeaders) { - EventData sendEvent = getEventData(rescheduledDelivery, httpHeaders); + Message sendEvent = getMessage(rescheduledDelivery, httpHeaders); sendEvent.getProperties().put("operation", "reschedule"); try { - clientPool.getConnection().send(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ExecutionException | ServiceBusException | IOException e) { + clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); + } catch (InterruptedException | ServiceBusException e) { throw new RuntimeException(e); } } - private EventData getEventData(Object deliveryObj, Map httpHeaders){ + private Message getMessage(Object deliveryObj, Map httpHeaders){ Gson gson = new GsonBuilder().create(); byte[] payloadBytes = gson.toJson(deliveryObj).getBytes(Charset.defaultCharset()); - EventData sendEvent = new EventData(payloadBytes); + Message sendEvent = new Message(payloadBytes); Map eventProps = new HashMap(); for(String header:appProps.getServiceMeshHeaders()){ diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java index 9ee854fa..e533be38 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java @@ -1,17 +1,13 @@ package com.fabrikam.dronedelivery.ingestion.util; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - import org.springframework.scheduling.annotation.Async; -import com.microsoft.azure.eventhubs.EventHubClient; -import com.microsoft.azure.servicebus.ServiceBusException; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; public interface ClientPool { @Async - public EventHubClient getConnection() - throws InterruptedException, ExecutionException, ServiceBusException, IOException; + public QueueClient getConnection() throws InterruptedException, ServiceBusException; } diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java index 7a1cbf12..812e6990 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java @@ -1,12 +1,11 @@ package com.fabrikam.dronedelivery.ingestion.util; import com.fabrikam.dronedelivery.ingestion.configuration.*; -import com.microsoft.azure.eventhubs.EventHubClient; -import com.microsoft.azure.servicebus.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.ServiceBusException; -import java.io.IOException; -import java.util.concurrent.ExecutionException; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.ReceiveMode; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; @@ -15,8 +14,8 @@ @Service public class ClientPoolImpl implements ClientPool { - private final EventHubClient[] eventHubClients; - private final String[] eventHubNames; + private final QueueClient[] queueClients; + private final String[] queueNames; private final ApplicationProperties appProperties; private final String nameSpace; private final String sasKeyName; @@ -24,35 +23,31 @@ public class ClientPoolImpl implements ClientPool { @Autowired - public ClientPoolImpl(ApplicationProperties appProps) - throws IOException, ServiceBusException, InterruptedException, ExecutionException { + public ClientPoolImpl(ApplicationProperties appProps) { this.appProperties = appProps; - this.eventHubNames = System.getenv(appProperties.getEnvHubName()).split(","); + this.queueNames = System.getenv(appProperties.getEnvQueueName()).split(","); nameSpace = System.getenv(appProperties.getEnvNameSpace()); sasKeyName = System.getenv(appProperties.getEnvsasKeyName()); sasKey = System.getenv(appProperties.getEnvsasKey()); - this.eventHubClients = new EventHubClient[this.appProperties.getMessageAmqpClientPoolSize()]; + this.queueClients = new QueueClient[this.appProperties.getMessageAmqpClientPoolSize()]; } @Async @Override - public EventHubClient getConnection() - throws InterruptedException, ExecutionException, ServiceBusException, IOException { + public QueueClient getConnection() throws InterruptedException, ServiceBusException { - int poolId = (int) (Math.random() * eventHubClients.length); - int eventHubId = (int) (Math.random() * eventHubNames.length); + int poolId = (int) (Math.random() * queueClients.length); + int eventHubId = (int) (Math.random() * queueNames.length); - if (eventHubClients[poolId] == null) { + if (queueClients[poolId] == null) { ConnectionStringBuilder connectionString = new ConnectionStringBuilder(nameSpace, - eventHubNames[eventHubId], sasKeyName, sasKey); - eventHubClients[poolId] = EventHubClient.createFromConnectionString(connectionString.toString()).get(); + queueNames[eventHubId], sasKeyName, sasKey); + queueClients[poolId] = new QueueClient(connectionString, ReceiveMode.PEEKLOCK); } - - return eventHubClients[poolId]; + return queueClients[poolId]; } - } diff --git a/src/shipping/ingestion/src/main/resources/application.properties b/src/shipping/ingestion/src/main/resources/application.properties index dadc26ab..5523f018 100644 --- a/src/shipping/ingestion/src/main/resources/application.properties +++ b/src/shipping/ingestion/src/main/resources/application.properties @@ -1,6 +1,6 @@ ############################################### # add environment variables -# EH_NAMESPACE EH_NAME EH_KEYNAME EH_KEYVALUE +# QUEUE_NAMESPACE QUEUE_NAME QUEUE_KEYNAME QUEUE_KEYVALUE # To debug/run tests. Yaml file will have to provide # environment va ################################################ @@ -8,10 +8,10 @@ service.threadPoolExecutorQueueSize=10000 service.threadPoolExecutorPoolSize=100 service.threadPoolExecutorMaxPoolSize=200 service.messageAmqpClientPoolSize=100 -service.envNameSpace=EH_NAMESPACE -service.envHubName=EH_NAME -service.envsasKeyName=EH_KEYNAME -service.envsasKey=EH_KEYVALUE +service.envNameSpace=QUEUE_NAMESPACE +service.envQueueName=QUEUE_NAME +service.envsasKeyName=QUEUE_KEYNAME +service.envsasKey=QUEUE_KEYVALUE service.serviceMeshHeaders=l5d-ctx-deadline,l5d-ctx-trace service.serviceMeshCorrelationHeader=l5d-ctx-trace server.port:80 diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java index f762cfdf..5a2bb3da 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java @@ -38,11 +38,11 @@ public void setUp() throws Exception { @Test public void canGetPropertiesfromEnvVariables() { Environment systemMock = Mockito.mock(Environment.class); - Mockito.when(systemMock.getenv("ENV_HUB_NAME")).thenReturn("variableMock"); + Mockito.when(systemMock.getenv("ENV_QUEUE_NAME")).thenReturn("variableMock"); ApplicationProperties appProps = new ApplicationProperties(); - assertEquals(systemMock.getenv(appProps.getEnvHubName()),"variableMock"); + assertEquals(systemMock.getenv(appProps.getEnvQueueName()),"variableMock"); Mockito.verify(systemMock, times(1)) - .getenv(appProps.getEnvHubName()); + .getenv(appProps.getEnvQueueName()); } } diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index dc51a1cd..25fd7e18 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -4,8 +4,7 @@ import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import com.microsoft.azure.servicebus.ServiceBusException; - +import com.microsoft.azure.servicebus.primitives.ServiceBusException; import java.util.Date; import java.util.UUID; diff --git a/src/shipping/package/app/initializer.ts b/src/shipping/package/app/initializer.ts index 004127bb..50931071 100644 --- a/src/shipping/package/app/initializer.ts +++ b/src/shipping/package/app/initializer.ts @@ -11,7 +11,7 @@ export class PackageServiceInitializer { static async initialize(connection: string, collectionName: string) { try { - var db = await MongoClient.connect(connection); + var db = (await MongoClient.connect(connection)).db(); await db.command({ shardCollection: db.databaseName + '.' + collectionName, key: { tag: "hashed" } }); } catch (ex) { diff --git a/src/shipping/package/app/models/repository.ts b/src/shipping/package/app/models/repository.ts index 69d2436a..de623168 100644 --- a/src/shipping/package/app/models/repository.ts +++ b/src/shipping/package/app/models/repository.ts @@ -22,7 +22,7 @@ export class Repository static async initialize(connection: string) { - Repository.db = await MongoClient.connect(connection); + Repository.db = (await MongoClient.connect(connection)).db(); } private collection() { From a0a93ad6e2ef9251e652e3102d58c14c96d4315e Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 28 Jan 2019 13:27:53 +0000 Subject: [PATCH 061/246] Merged PR 645: Address e2e testing issues tweak the serialization and deserialization used by workflow to communicate with the package service to match the implicit casing requirements remove the "clusterIP: none" from the service specs. this is supposed to be used for headless services and these arent https://kubernetes.io/docs/concepts/services-networking/service/#headless-services. communication worked before because there were no port mappings, but with port mappings this makes connections through the service to fail. --- k8s/delivery.yaml | 1 - k8s/dronescheduler.yaml | 1 - k8s/package.yml | 3 +-- .../Fabrikam.Workflow.Service/Models/PackageGen.cs | 8 ++++++++ .../Fabrikam.Workflow.Service/Models/PackageInfo.cs | 8 ++++++++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index e8628b4c..da6a701e 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -21,7 +21,6 @@ spec: targetPort: 8080 selector: app: delivery - clusterIP: None --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index a4f82264..61c78a3d 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -21,7 +21,6 @@ spec: targetPort: 8080 selector: app: dronescheduler - clusterIP: None --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/package.yml b/k8s/package.yml index b7577cca..da234924 100644 --- a/k8s/package.yml +++ b/k8s/package.yml @@ -57,5 +57,4 @@ spec: app: package ports: - name: http - port: 80 - clusterIP: None + port: 80 \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs index 192195f5..f9ef2501 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageGen.cs @@ -3,13 +3,21 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + namespace Fabrikam.Workflow.Service.Models { public class PackageGen { + [JsonProperty("id")] public string Id { get; set; } + [JsonProperty("size")] + [JsonConverter(typeof(StringEnumConverter))] public ContainerSize Size { get; set; } + [JsonProperty("tag")] public string Tag { get; set; } + [JsonProperty("weight")] public double Weight { get; set; } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs index 4e4f9903..5e3a546d 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Models/PackageInfo.cs @@ -3,13 +3,21 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + namespace Fabrikam.Workflow.Service.Models { public class PackageInfo { + [JsonProperty("packageId")] public string PackageId { get; set; } + [JsonProperty("size")] + [JsonConverter(typeof(StringEnumConverter))] public ContainerSize Size { get; set; } + [JsonProperty("weight")] public double Weight { get; set; } + [JsonProperty("tag")] public string Tag { get; set; } } } From 8319deeb31154044eb7067aa40a739189b1cde29 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 29 Jan 2019 13:06:47 +0000 Subject: [PATCH 062/246] Merged PR 646: Remove delivery history and streamline deployment instructions Removes stale delivery history service, and streamlines the deployment instructions to remove key vaults for services that will not use it. Related work items: #8674 --- deployment.md | 20 +--- k8s/delivery.yaml | 7 -- .../DeliveriesControllerFixture.cs | 94 ------------------- .../Controllers/DeliveriesController.cs | 12 --- .../Services/DeliveryHistoryService.cs | 26 ----- .../Services/EventHubSender.cs | 55 ----------- .../Services/IDeliveryHistoryService.cs | 16 ---- .../Startup.cs | 2 - src/shipping/delivery/docker-compose.yml | 24 ----- 9 files changed, 1 insertion(+), 255 deletions(-) delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs diff --git a/deployment.md b/deployment.md index 8b8a9e6b..08fdfa81 100644 --- a/deployment.md +++ b/deployment.md @@ -144,7 +144,7 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/delivery:0.1.0 ``` -Create Kubernetes secrets +Create resources ```bash export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) @@ -152,9 +152,6 @@ export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOU export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) - -kubectl --namespace backend create --save-config=true secret generic delivery-storageconf \ - --from-literal=EH_ConnectionString= ``` Create KeyVault and secrets @@ -166,7 +163,6 @@ az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Key az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Endpoint --value ${COSMOSDB_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-Endpoint --value ${REDIS_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-AccessKey --value ${REDIS_KEY} -# az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name EH-ConnectionString # cannot create a secret without a value or with an empty value export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) @@ -203,7 +199,6 @@ Deploy the Delivery service: sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ - sed "s/value: \"EH_EntityPath\"/value:/g" | \ sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" > $K8S/delivery-0.yaml # Deploy the service @@ -245,11 +240,6 @@ sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/packag export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') kubectl -n backend create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION -# Create KeyVault secret -export PACKAGE_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-package-kv" -az keyvault create --name $PACKAGE_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION -az keyvault secret set --vault-name $PACKAGE_KEYVAULT_NAME --name mongodb-pwd --value $COSMOSDB_CONNECTION - # Deploy service kubectl --namespace backend apply -f $K8S/package-0.yml @@ -393,14 +383,6 @@ kubectl -n backend create secret generic ingestion-secrets \ --from-literal=queue_keyname=IngestionServiceAccessKey \ --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} -# Create KeyVault secrets -export INGESTION_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-ingestion-kv" -az keyvault create --name $INGESTION_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-namespace --value ${INGESTION_QUEUE_NAMESPACE} -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-name --value ${INGESTION_QUEUE_NAME} -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyname --value IngestionServiceAccessKey -az keyvault secret set --vault-name $INGESTION_KEYVAULT_NAME --name eventhub-keyvalue --value ${INGESTION_ACCESS_KEY_VALUE} - # Deploy service kubectl --namespace backend apply -f $K8S/ingestion-0.yaml diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml index da6a701e..3f2535a8 100644 --- a/k8s/delivery.yaml +++ b/k8s/delivery.yaml @@ -61,13 +61,6 @@ spec: value: "CosmosDB_CollectionId" - name: KEY_VAULT_URI value: "KeyVault_Name" - - name: EH_CONNSTR - valueFrom: - secretKeyRef: - name: delivery-storageconf - key: EH_ConnectionString - - name: EH_ENTITYPATH - value: "EH_EntityPath" - name: no_proxy value: 169.254.169.254 ports: diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs index eb88c5ef..f8ffc156 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs @@ -40,7 +40,6 @@ public async Task Get_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -62,7 +61,6 @@ public async Task GetOwner_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -88,7 +86,6 @@ public async Task GetOwner_ReturnsOwner() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -110,7 +107,6 @@ public async Task GetStatus_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -137,7 +133,6 @@ public async Task Put_Returns204_IfDeliveryIdExists() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -163,7 +158,6 @@ public async Task Put_AddsToCache() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); // Act @@ -192,7 +186,6 @@ public async Task Put_AddscreatedDeliveryEvent() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, deliveryStatusEventRepository.Object, loggerFactory.Object); // Act @@ -217,7 +210,6 @@ public async Task Patch_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -251,7 +243,6 @@ public async Task Patch_UpdatesCache() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -295,7 +286,6 @@ public async Task Patch_AddsRescheduledDeliveryEvent() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, deliveryStatusEventRepository.Object, loggerFactory.Object); @@ -319,7 +309,6 @@ public async Task Delete_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -336,19 +325,10 @@ public async Task Delete_SendsMessageWithCancelledTrackingEvent() { // Arrange DeliveryTrackingEvent cancelledDelivery = null; - DeliveryTrackingEvent[] allTrackingEvents = null; var deliveryRepository = new Mock(); deliveryRepository.Setup(r => r.GetAsync("deliveryid")).ReturnsAsync(delivery); - var deliveryHistoryService = new Mock(); - deliveryHistoryService.Setup(r => r.CancelAsync(It.IsAny(), It.IsAny())) - .Returns(Task.CompletedTask) - .Callback((d, es) => - { - allTrackingEvents = es; - }); - var deliveryStatusEventRepository = new Mock(); deliveryStatusEventRepository.Setup(r => r.AddAsync(It.IsAny())) .Returns(Task.CompletedTask) @@ -363,7 +343,6 @@ public async Task Delete_SendsMessageWithCancelledTrackingEvent() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - deliveryHistoryService.Object, deliveryStatusEventRepository.Object, loggerFactory.Object); // Act @@ -374,7 +353,6 @@ public async Task Delete_SendsMessageWithCancelledTrackingEvent() Assert.AreEqual("deliveryid", cancelledDelivery.DeliveryId); Assert.AreEqual(DeliveryStage.Cancelled, cancelledDelivery.Stage); deliveryRepository.VerifyAll(); - deliveryHistoryService.Verify(s => s.CancelAsync(delivery, allTrackingEvents), Times.Once); } [TestMethod] @@ -387,7 +365,6 @@ public async Task NotifyMe_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -419,7 +396,6 @@ public async Task NotifyMe_AddsNotifyMeRequest() var target = new DeliveriesController(deliveryRepository.Object, notifyMeRequestRepository.Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); @@ -445,7 +421,6 @@ public async Task Confirm_Returns404_IfDeliveryIdNotValid() var target = new DeliveriesController(new Mock().Object, new Mock().Object, new Mock().Object, - new Mock().Object, new Mock().Object, loggerFactory.Object); // Act @@ -486,16 +461,12 @@ public async Task Confirm_SendsNotifications() deliveryStatusEventRepository.Setup(r => r.GetByDeliveryIdAsync("deliveryid")) .ReturnsAsync(new ReadOnlyCollection(new List() { completedDelivery })); - var deliveryHistoryService = new Mock(); - deliveryHistoryService.Setup(r => r.CompleteAsync(delivery, It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); - var loggerFactory = new Mock(); loggerFactory.Setup(f => f.CreateLogger(It.IsAny())).Returns(new Mock().Object); var target = new DeliveriesController(deliveryRepository.Object, notifyMeRequestRepository.Object, notificationService.Object, - new Mock().Object, deliveryStatusEventRepository.Object, loggerFactory.Object); @@ -511,69 +482,6 @@ public async Task Confirm_SendsNotifications() Assert.AreEqual(2, notificationServiceCalled); } - [TestMethod] - public async Task Confirm_SendsMessageCompleteToDeliveryHistory() - { - // Arrange - InternalDelivery confirmedDelivery = null; - InternalConfirmation sentConfirmation = null; - DeliveryTrackingEvent completedDelivery = null; - DeliveryTrackingEvent[] allTrackingEvents = null; - - var deliveryRepository = new Mock(); - deliveryRepository.Setup(r => r.GetAsync("deliveryid")).ReturnsAsync(delivery); - - var notifyMeRequestRepository = new Mock(); - notifyMeRequestRepository.Setup(r => r.GetAllByDeliveryIdAsync("deliveryid")) - .ReturnsAsync(new List()); - - var deliveryStatusEventRepository = new Mock(); - deliveryStatusEventRepository.Setup(r => r.AddAsync(It.IsAny())) - .Returns(Task.CompletedTask) - .Callback(e => completedDelivery = e); - - deliveryStatusEventRepository.Setup(r => r.GetByDeliveryIdAsync("deliveryid")) - .ReturnsAsync(new ReadOnlyCollection(new List() { completedDelivery })); - - var deliveryHistoryService = new Mock(); - deliveryHistoryService.Setup(r => r.CompleteAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(Task.CompletedTask) - .Callback((d, c, es) => - { - confirmedDelivery = d; - sentConfirmation = c; - allTrackingEvents = es; - }); - - var loggerFactory = new Mock(); - loggerFactory.Setup(f => f.CreateLogger(It.IsAny())).Returns(new Mock().Object); - - var target = new DeliveriesController(deliveryRepository.Object, - notifyMeRequestRepository.Object, - new Mock().Object, - deliveryHistoryService.Object, - deliveryStatusEventRepository.Object, - loggerFactory.Object); - - var confirmation = new Confirmation(new DateTimeStamp("datetimevalue"), - new Location(1, 2, 3), - ConfirmationType.Picture, - "confirmationblob"); - - // Act - var result = await target.Confirm("deliveryid", confirmation) as OkResult; - - // Assert - Assert.IsNotNull(result); - Assert.AreEqual("datetimevalue", sentConfirmation.DateTime.DateTimeValue); - Assert.AreEqual(1, sentConfirmation.GeoCoordinates.Altitude); - Assert.AreEqual(2, sentConfirmation.GeoCoordinates.Latitude); - Assert.AreEqual(3, sentConfirmation.GeoCoordinates.Longitude); - Assert.AreEqual(ConfirmationType.Picture, sentConfirmation.ConfirmationType); - Assert.AreEqual("confirmationblob", sentConfirmation.ConfirmationBlob); - deliveryHistoryService.Verify(s => s.CompleteAsync(confirmedDelivery, It.IsAny(), allTrackingEvents), Times.Once); - } - [TestMethod] public async Task Confirm_DeletesDeliveryLogically() { @@ -601,7 +509,6 @@ public async Task Confirm_DeletesDeliveryLogically() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, deliveryStatusEventRepository.Object, loggerFactory.Object); var location = new Location(1, 2, 3); @@ -642,7 +549,6 @@ public async Task Confirm_AddsDeliveryCompletedEvent() var target = new DeliveriesController(deliveryRepository.Object, new Mock().Object, new Mock().Object, - new Mock().Object, deliveryStatusEventRepository.Object, loggerFactory.Object); diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs index 7ae462f6..43259035 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs @@ -22,21 +22,18 @@ public class DeliveriesController : Controller private readonly IDeliveryRepository deliveryRepository; private readonly INotifyMeRequestRepository notifyMeRequestRepository; private readonly INotificationService notificationService; - private readonly IDeliveryHistoryService deliveryHistoryService; private readonly IDeliveryTrackingEventRepository deliveryTrackingRepository; private readonly ILogger logger; public DeliveriesController(IDeliveryRepository deliveryRepository, INotifyMeRequestRepository notifyMeRequestRepository, INotificationService notificationService, - IDeliveryHistoryService deliveryHistoryRepository, IDeliveryTrackingEventRepository deliveryTrackingRepository, ILoggerFactory loggerFactory) { this.deliveryRepository = deliveryRepository; this.notifyMeRequestRepository = notifyMeRequestRepository; this.notificationService = notificationService; - this.deliveryHistoryService = deliveryHistoryRepository; this.deliveryTrackingRepository = deliveryTrackingRepository; this.logger = loggerFactory.CreateLogger(); } @@ -179,10 +176,6 @@ public async Task Delete(string id) var deliveryTrackingEvent = new DeliveryTrackingEvent { DeliveryId = id, Stage = DeliveryStage.Cancelled }; await deliveryTrackingRepository.AddAsync(deliveryTrackingEvent); - // forwards cancelled delivery to the Delivery History - var allTrackingEvents = await deliveryTrackingRepository.GetByDeliveryIdAsync(id); - await deliveryHistoryService.CancelAsync(delivery, allTrackingEvents.ToArray()); - // logical delivery deletion await deliveryRepository.DeleteAsync(id, delivery); @@ -261,11 +254,6 @@ await deliveryTrackingRepository.AddAsync(new DeliveryTrackingEvent DeliveryId = id, Stage = DeliveryStage.Completed }); - // get all the milestones from cache - var allTrackingEvents = await deliveryTrackingRepository.GetByDeliveryIdAsync(id); - - // archives Delivery by sending it to the Delivery History + Confirmantion details as well as forwarding milestones to the Delivery History - await deliveryHistoryService.CompleteAsync(confirmedDelivery, internalConfirmation, allTrackingEvents.ToArray()); // sends notifications var notifyMeRequests = await notifyMeRequestRepository.GetAllByDeliveryIdAsync(id); diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs deleted file mode 100644 index 34d17899..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryHistoryService.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using System.Threading.Tasks; -using Fabrikam.DroneDelivery.DeliveryService.Models; -using Fabrikam.DroneDelivery.Common; - -namespace Fabrikam.DroneDelivery.DeliveryService.Services -{ - public class DeliveryHistoryService : IDeliveryHistoryService - { - public async Task CompleteAsync(InternalDelivery delivery, InternalConfirmation confirmation, params DeliveryTrackingEvent[] deliveryTrackingEvents) - { - //TODO: shallowing confirmation (TBD) - await EventHubSender.SendMessageAsync(new DeliveryHistory(delivery.Id, delivery, deliveryTrackingEvents), nameof(DeliveryStage.Completed), delivery.Id.Substring(0, Constants.PartitionKeyLength)).ConfigureAwait(continueOnCapturedContext: false); - } - - public async Task CancelAsync(InternalDelivery delivery, params DeliveryTrackingEvent[] deliveryTrackingEvents) - { - await EventHubSender.SendMessageAsync(new DeliveryHistory(delivery.Id, delivery, deliveryTrackingEvents), nameof(DeliveryStage.Cancelled), delivery.Id.Substring(0, Constants.PartitionKeyLength)).ConfigureAwait(continueOnCapturedContext: false); - } - } -} \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs deleted file mode 100644 index f1c79f9f..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/EventHubSender.cs +++ /dev/null @@ -1,55 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Azure.EventHubs; -using Newtonsoft.Json; -using Fabrikam.DroneDelivery.DeliveryService.Models; - -namespace Fabrikam.DroneDelivery.DeliveryService.Services -{ - public static class EventHubSender where T : BaseMessage - { - private static string EhConnectionString; - private static string EhEntityPath; - - private static Lazy lazyConnection = new Lazy(() => - { - // it is does guarantee thread-safety - var connectionStringBuilder = new EventHubsConnectionStringBuilder(EhConnectionString) - { - EntityPath = EhEntityPath - }; - - return EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); - - }); - - private static EventHubClient connection - { - get - { - return lazyConnection.Value; - } - } - - public static void Configure(string connectionString, string entityPath) - { - EhConnectionString = connectionString; - EhEntityPath = entityPath; - } - - public static async Task SendMessageAsync(DeliveryHistory deliveryHistory, string messageType, string partitionKey) - { - deliveryHistory.PartitionKey = partitionKey; - deliveryHistory.MessageType = messageType; - string jsonDeliveryHistory = await Task.Factory.StartNew(() => JsonConvert.SerializeObject(deliveryHistory)); - // TODO: send a batch to EH improves the performance a lot. Therefore, instead of sending milestones, we could send them all in a batch (TBD) - await connection.SendAsync(new EventData(Encoding.UTF8.GetBytes(jsonDeliveryHistory)), partitionKey).ConfigureAwait(continueOnCapturedContext: false); - } - } -} \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs deleted file mode 100644 index 3924edb4..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryHistoryService.cs +++ /dev/null @@ -1,16 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System.Threading.Tasks; -using Fabrikam.DroneDelivery.DeliveryService.Models; - -namespace Fabrikam.DroneDelivery.DeliveryService.Services -{ - public interface IDeliveryHistoryService - { - Task CompleteAsync(InternalDelivery delivery, InternalConfirmation confirmation, params DeliveryTrackingEvent[] deliveryTrackingEvents); - Task CancelAsync(InternalDelivery delivery, params DeliveryTrackingEvent[] deliveryTrackingEvents); - } -} \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 16df0fec..d5dd3357 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -62,7 +62,6 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); } @@ -98,7 +97,6 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF DocumentDBRepository.Configure(Configuration["CosmosDB-Endpoint"], Configuration["CosmosDB-Key"], Configuration["DOCDB_DATABASEID"], Configuration["DOCDB_COLLECTIONID"], loggerFactory); RedisCache.Configure(Constants.RedisCacheDBId_Delivery, Configuration["Redis-Endpoint"], Configuration["Redis-AccessKey"], loggerFactory); RedisCache.Configure(Constants.RedisCacheDBId_DeliveryStatus, Configuration["Redis-Endpoint"], Configuration["Redis-AccessKey"], loggerFactory); - EventHubSender.Configure(Configuration["EH_CONNSTR"], Configuration["EH_ENTITYPATH"]); } } } diff --git a/src/shipping/delivery/docker-compose.yml b/src/shipping/delivery/docker-compose.yml index 46c89110..b179717e 100644 --- a/src/shipping/delivery/docker-compose.yml +++ b/src/shipping/delivery/docker-compose.yml @@ -6,27 +6,3 @@ services: build: context: ./Fabrikam.DroneDelivery.DeliveryService dockerfile: Dockerfile - - dronescheduler: - image: mockdronescheduler - build: - context: ./MockDroneScheduler - dockerfile: Dockerfile - - deliveryscheduler: - image: mockdeliveryscheduler - build: - context: ./MockDeliveryScheduler - dockerfile: Dockerfile - - account: - image: mockaccountservice - build: - context: ./MockAccountService - dockerfile: Dockerfile - - thirdparty: - image: mockthirdpartyservice - build: - context: ./MockThirdPartyService - dockerfile: Dockerfile From 0dbf0c253845dcce4ba54041df15d28fecb1ab3c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 31 Jan 2019 18:27:14 +0000 Subject: [PATCH 063/246] Merged PR 647: add Kuberentes AppInsights add Kubernetes AppInsights - code light up for distributed tracing - remove correlation enricher - align deployment instructions - add ikey keyvault secret resolved: #8589 Related work items: #8590 --- deployment.md | 30 +++++++++++++-- k8s/k8s-rbac-ai.yaml | 23 +++++++++++ src/shipping/delivery/Dockerfile | 8 +--- .../CorrelationLogEventEnricher.cs | 38 ------------------- ...rikam.DroneDelivery.DeliveryService.csproj | 2 + .../Startup.cs | 7 +++- .../appsettings.json | 3 +- 7 files changed, 60 insertions(+), 51 deletions(-) create mode 100644 k8s/k8s-rbac-ai.yaml delete mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs diff --git a/deployment.md b/deployment.md index 08fdfa81..7797a2ca 100644 --- a/deployment.md +++ b/deployment.md @@ -28,7 +28,8 @@ export LOCATION=[YOUR_LOCATION_HERE] export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ -export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" +export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ +export AI_NAME="${UNIQUE_APP_NAME_PREFIX}-appinsights" export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) @@ -83,11 +84,33 @@ Grant the cluster access to the registry. ```bash # Acquire the necessary IDs -export CLUSTER_CLIENT_ID=$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "servicePrincipalProfile.clientId" --output tsv) export ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP --query "id" --output tsv) # Grant the cluster read access to the registry -az role assignment create --assignee $CLUSTER_CLIENT_ID --role Reader --scope $ACR_ID +az role assignment create --assignee $CLUSTER_SERVICE_PRINCIPAL --role Reader --scope $ACR_ID +``` + +Create an Application Insights instance + +```bash +# Create AppInsights instance +az resource create \ + --resource-group $RESOURCE_GROUP \ + --resource-type "Microsoft.Insights/components" \ + --name $AI_NAME \ + --location $LOCATION \ + --properties '{"Application_Type":"other"}' + +# Acquire Instrumentation Key +export AI_IKEY=$(az resource show \ + -g $RESOURCE_GROUP \ + -n $AI_NAME \ + --resource-type "Microsoft.Insights/components" \ + --query properties.InstrumentationKey \ + -o tsv) + +# add RBAC for AppInsights +kubectl apply -f $K8S/k8s-rbac-ai.yaml ``` ## Setup AAD pod identity and key vault flexvol infrastructure @@ -163,6 +186,7 @@ az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Key az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Endpoint --value ${COSMOSDB_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-Endpoint --value ${REDIS_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-AccessKey --value ${REDIS_KEY} +az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name ApplicationInsights--InstrumentationKey --value ${AI_IKEY} export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) diff --git a/k8s/k8s-rbac-ai.yaml b/k8s/k8s-rbac-ai.yaml new file mode 100644 index 00000000..1ca695af --- /dev/null +++ b/k8s/k8s-rbac-ai.yaml @@ -0,0 +1,23 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: metrics-reader +rules: +- apiGroups: ["","extensions"] + resources: ["pods","nodes", "replicasets","deployments"] + verbs: ["get", "watch", "list"] +--- +# actual binding to the role +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: metrics-reader-binding +subjects: +- kind: ServiceAccount + name: default + namespace: backend +roleRef: + kind: ClusterRole + name: metrics-reader + apiGroup: rbac.authorization.k8s.io + diff --git a/src/shipping/delivery/Dockerfile b/src/shipping/delivery/Dockerfile index 354e45cc..0496458a 100644 --- a/src/shipping/delivery/Dockerfile +++ b/src/shipping/delivery/Dockerfile @@ -1,11 +1,9 @@ FROM microsoft/dotnet:2.2-aspnetcore-runtime as base WORKDIR /app -EXPOSE 80 +EXPOSE 8080 FROM microsoft/dotnet:2.2-sdk AS build -MAINTAINER Fernando Antivero (https://github.com/ferantivero) - WORKDIR /app COPY Fabrikam.DroneDelivery.Common/*.csproj ./Fabrikam.DroneDelivery.Common/ COPY Fabrikam.DroneDelivery.DeliveryService/*.csproj ./Fabrikam.DroneDelivery.DeliveryService/ @@ -19,8 +17,6 @@ COPY Fabrikam.DroneDelivery.DeliveryService/. ./Fabrikam.DroneDelivery.DeliveryS FROM build AS testrunner -MAINTAINER Fernando Antivero (https://github.com/ferantivero) - WORKDIR /app/tests COPY Fabrikam.DroneDelivery.DeliveryService.Tests/*.csproj . WORKDIR /app/tests @@ -59,4 +55,4 @@ RUN chown -R $user.$user /app # Set it for subsequent commands USER $user -ENTRYPOINT ["/bin/bash", "/app/run.sh"] +ENTRYPOINT ["/bin/bash", "/app/run.sh"] \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs deleted file mode 100644 index 19a0d130..00000000 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/CorrelationLogEventEnricher.cs +++ /dev/null @@ -1,38 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System.Linq; -using Microsoft.AspNetCore.Http; -using Serilog.Core; -using Serilog.Events; - -namespace Fabrikam.DroneDelivery.Common -{ - public class CorrelationLogEventEnricher : ILogEventEnricher - { - public const string CorrelationPropertyName = "CorrelationId"; - - public CorrelationLogEventEnricher(IHttpContextAccessor httpContextAccessor, string correlationIdHeaderKey) - { - HttpContextAccessor = httpContextAccessor; - CorrelationIdHeaderKey = correlationIdHeaderKey; - } - - public IHttpContextAccessor HttpContextAccessor { get; } - public string CorrelationIdHeaderKey { get; } - - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) - { - if (HttpContextAccessor.HttpContext == null) return; - - var requestHeaders = HttpContextAccessor.HttpContext.Request.Headers; - var correlationId = requestHeaders[CorrelationIdHeaderKey].FirstOrDefault(); - if (correlationId != null) - { - logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty(CorrelationPropertyName, correlationId)); - } - } - } -} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index 1ad7dae9..19e2ae82 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -10,6 +10,8 @@ + + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index d5dd3357..eaa42504 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -47,6 +47,10 @@ public void ConfigureServices(IServiceCollection services) { services.AddSingleton(); + // Configure AppInsights + services.AddApplicationInsightsKubernetesEnricher(); + services.AddApplicationInsightsTelemetry(Configuration); + services.AddLogging(loggingBuilder => loggingBuilder.AddSerilog(dispose: true)); @@ -58,7 +62,7 @@ public void ConfigureServices(IServiceCollection services) { c.SwaggerDoc("v1", new Info { Title = "Fabrikam DroneDelivery DeliveryService API", Version = "v1" }); }); - + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -71,7 +75,6 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF Log.Logger = new LoggerConfiguration() .WriteTo.Console(new CompactJsonFormatter()) .ReadFrom.Configuration(Configuration) - .Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) .CreateLogger(); // Important: it has to be first: enable global logger diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json index 8909c7bf..a2bb0628 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json @@ -3,8 +3,7 @@ "IncludeScopes": false, "LogLevel": { "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" + } }, "Serilog": { "MinimumLevel": "Verbose", From c84adfa030c166cb81ef1a304b2808dc59abce42 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 5 Feb 2019 15:30:19 +0000 Subject: [PATCH 064/246] Merged PR 648: add Kubernetes AppInsights for DroneScheduler add Kubernetes AppInsights - code light up for distributed tracing - remove correlation enricher - align deployment instructions - add ikey keyvault secret - add identity for drone scheduler - add keyvault for drone scheduler resolved: #8590 Related work items: #8590 --- deployment.md | 39 ++++++++++++++++++- k8s/dronescheduler-identity.yaml | 24 ++++++++++++ k8s/dronescheduler.yaml | 7 ++++ k8s/k8s-rbac-ai.yaml | 6 +-- ...brikam.DroneDelivery.DroneScheduler.csproj | 4 ++ .../Startup.cs | 14 ++++++- 6 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 k8s/dronescheduler-identity.yaml diff --git a/deployment.md b/deployment.md index 7797a2ca..6e12aa6a 100644 --- a/deployment.md +++ b/deployment.md @@ -422,6 +422,41 @@ Build the dronescheduler services export DRONE_PATH=microservices-reference-implementation/src/shipping/dronescheduler ``` +Create KeyVault and secrets + +```bash +export DRONESCHEDULER_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-drone-kv" +az keyvault create --name $DRONESCHEDULER_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +az keyvault secret set --vault-name $DRONESCHEDULER_KEYVAULT_NAME --name ApplicationInsights--InstrumentationKey --value ${AI_IKEY} + +export DRONESCHEDULER_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_KEYVAULT_NAME --query "id" --output tsv) +export DRONESCHEDULER_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) +``` + +Create and set up pod identity + +```bash +# Create the identity and extract properties +export DRONESCHEDULER_PRINCIPAL_NAME=dronescheduler +az identity create --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "id" --output tsv) +export DRONESCHEDULER_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "principalId" --output tsv) +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "clientId" --output tsv) + +# Grant the identity access to the KeyVault +az role assignment create --role Reader --assignee $DRONESCHEDULER_PRINCIPAL_ID --scope $DRONESCHEDULER_KEYVAULT_ID +az keyvault set-policy --name $DRONESCHEDULER_KEYVAULT_NAME --secret-permissions get list --spn $DRONESCHEDULER_PRINCIPAL_CLIENT_ID + +# Allow the cluster to manage the identity to assign to pods +az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID + +# Deploy the identity resources +cat $K8S/dronescheduler-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" > $K8S/dronescheduler-identity-0.yaml +kubectl apply -f $K8S/dronescheduler-identity-0.yaml +``` + Build and publish the container image ```bash @@ -437,7 +472,9 @@ Deploy the dronescheduler services: ```bash # Update the image tag in the deployment YAML -sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" $K8S/dronescheduler.yaml > $K8S/dronescheduler-0.yaml +cat $K8S/dronescheduler.yaml | \ + sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" | \ + sed "s#value: \"KeyVault_Name\"#value: $DRONESCHEDULER_KEYVAULT_URI#g" > $K8S/dronescheduler-0.yaml # Deploy the service kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml diff --git a/k8s/dronescheduler-identity.yaml b/k8s/dronescheduler-identity.yaml new file mode 100644 index 00000000..50b62529 --- /dev/null +++ b/k8s/dronescheduler-identity.yaml @@ -0,0 +1,24 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Drone Scheduler service identity +################################################################################################### +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentity +metadata: + name: dronescheduler-identity +spec: + type: 0 + ResourceID: "identityResourceId" + ClientID: "identityClientid" +--- +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentityBinding +metadata: + name: dronescheduler-identity-binding +spec: + AzureIdentity: dronescheduler-identity + Selector: dronescheduler \ No newline at end of file diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml index 61c78a3d..50318788 100644 --- a/k8s/dronescheduler.yaml +++ b/k8s/dronescheduler.yaml @@ -31,6 +31,7 @@ metadata: version: 0.1.0 bc: dronemgmt co: fabrikam + aadpodidbinding: dronescheduler spec: replicas: 1 selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. @@ -43,12 +44,18 @@ spec: version: 0.1.0 bc: dronemgmt co: fabrikam + aadpodidbinding: dronescheduler annotations: team: droneschedulerservice spec: containers: - name: dronescheduler image: + env: + - name: KEY_VAULT_URI + value: "KeyVault_Name" + - name: no_proxy + value: 169.254.169.254 ports: - name: service containerPort: 8080 diff --git a/k8s/k8s-rbac-ai.yaml b/k8s/k8s-rbac-ai.yaml index 1ca695af..2fdeaec9 100644 --- a/k8s/k8s-rbac-ai.yaml +++ b/k8s/k8s-rbac-ai.yaml @@ -1,7 +1,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: metrics-reader + name: appinsights-k8s-property-reader rules: - apiGroups: ["","extensions"] resources: ["pods","nodes", "replicasets","deployments"] @@ -11,13 +11,13 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: metrics-reader-binding + name: appinsights-k8s-property-reader-binding subjects: - kind: ServiceAccount name: default namespace: backend roleRef: kind: ClusterRole - name: metrics-reader + name: appinsights-k8s-property-reader apiGroup: rbac.authorization.k8s.io diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj index 8240a2c9..dbbe2589 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj @@ -10,6 +10,10 @@ + + + + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs index 376b437e..e92730b5 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs @@ -13,7 +13,6 @@ using Swashbuckle.AspNetCore.Swagger; using Serilog; using Serilog.Formatting.Compact; -using Fabrikam.DroneDelivery.Common; namespace MockDroneScheduler { @@ -26,6 +25,14 @@ public Startup(IHostingEnvironment env) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) .AddEnvironmentVariables(); + + var buildConfig = builder.Build(); + + if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) + { + builder.AddAzureKeyVault(keyVaultUri); + } + Configuration = builder.Build(); } @@ -36,6 +43,10 @@ public void ConfigureServices(IServiceCollection services) { services.AddSingleton(); + // Configure AppInsights + services.AddApplicationInsightsKubernetesEnricher(); + services.AddApplicationInsightsTelemetry(Configuration); + services.AddLogging(loggingBuilder => loggingBuilder.AddSerilog(dispose: true)); @@ -55,7 +66,6 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF Log.Logger = new LoggerConfiguration() .WriteTo.Console(new CompactJsonFormatter()) .ReadFrom.Configuration(Configuration) - .Enrich.With(new CorrelationLogEventEnricher(httpContextAccessor, Configuration["Logging:CorrelationHeaderKey"])) .CreateLogger(); app.UseMvc(); From a4fe1bbce910e250875dcae438833fcc788807b1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 6 Feb 2019 13:42:19 +0000 Subject: [PATCH 065/246] Merged PR 650: add AppInsights for Package add AppInsights for Package - code light up for distributed tracing - remove correlation enricher - align deployment instructions - add ikey kubernetes secret resolved: #8593 Related work items: #8593 --- charts/package/templates/package-deploy.yaml | 11 +++++-- deployment.md | 5 ++- k8s/package.yml | 17 ++++++---- src/shipping/package/app/initializer.ts | 23 +++++++++++-- src/shipping/package/app/main.ts | 4 +-- src/shipping/package/app/models/repository.ts | 2 +- src/shipping/package/app/server.ts | 8 ++--- src/shipping/package/app/util/logging.ts | 32 ++++++++++++------- src/shipping/package/app/util/settings.ts | 6 ++-- src/shipping/package/package.json | 5 +-- 10 files changed, 76 insertions(+), 37 deletions(-) diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index 8e925888..a19dda8a 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -30,7 +30,7 @@ spec: helm.sh/chart: {{ include "package.chart" . }} spec: containers: - - name: packageservice + - name: &package-container_name package image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} env: - name: CONNECTION_STRING @@ -38,10 +38,15 @@ spec: secretKeyRef: name: package-secrets key: {{ .Release.Name }}-mongodb-pwd + - name: APPINSIGHTS_INSTRUMENTATIONKEY + valueFrom: + secretKeyRef: + name: package-secrets + key: {{ .Release.Name }}-appinsights-ikey - name: COLLECTION_NAME value: packages - - name: CORRELATION_HEADER - value: {{ .Values.correlationheader }} + - name: CONTAINER_NAME + value: *package-container_name ports: - name: service containerPort: 80 diff --git a/deployment.md b/deployment.md index 6e12aa6a..ab9426b9 100644 --- a/deployment.md +++ b/deployment.md @@ -262,7 +262,10 @@ sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/packag # Create secret export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') -kubectl -n backend create secret generic package-secrets --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION +kubectl -n backend create \ + secret generic package-secrets \ + --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION \ + --from-literal=appinsights-ikey=$AI_IKEY # Deploy service kubectl --namespace backend apply -f $K8S/package-0.yml diff --git a/k8s/package.yml b/k8s/package.yml index da234924..0b534248 100644 --- a/k8s/package.yml +++ b/k8s/package.yml @@ -3,7 +3,7 @@ # Licensed under the MIT License (MIT). See License.txt in the repo root for license information. # ------------------------------------------------------------ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta1 kind: Deployment metadata: name: package @@ -26,8 +26,8 @@ spec: co: fabrikam spec: containers: - - name: package - image: + - name: &package-container_name package + image: env: - name: CONNECTION_STRING valueFrom: @@ -36,10 +36,15 @@ spec: key: mongodb-pwd - name: COLLECTION_NAME value: packages - - name: CORRELATION_HEADER - value: l5d-ctx-trace + - name: APPINSIGHTS_INSTRUMENTATIONKEY + valueFrom: + secretKeyRef: + name: package-secrets + key: appinsights-ikey - name: LOG_LEVEL value: error + - name: CONTAINER_NAME + value: *package-container_name ports: - containerPort: 80 imagePullPolicy: Always @@ -57,4 +62,4 @@ spec: app: package ports: - name: http - port: 80 \ No newline at end of file + port: 80 diff --git a/src/shipping/package/app/initializer.ts b/src/shipping/package/app/initializer.ts index 50931071..f1231687 100644 --- a/src/shipping/package/app/initializer.ts +++ b/src/shipping/package/app/initializer.ts @@ -5,13 +5,25 @@ import { MongoErrors } from './util/mongo-err' +let appInsights = require('applicationinsights'); var MongoClient = require('mongodb').MongoClient; export class PackageServiceInitializer { - static async initialize(connection: string, collectionName: string) { + static async initialize(connection: string, collectionName: string, containerName: string) { try { - var db = (await MongoClient.connect(connection)).db(); + PackageServiceInitializer.initAppInsights(containerName); + await PackageServiceInitializer.initMongoDb(connection, + collectionName); + } + catch(ex) { + console.log(ex); + } + } + + private static async initMongoDb(connection: string, collectionName: string) { + try { + var db = (await MongoClient.connect(connection)); await db.command({ shardCollection: db.databaseName + '.' + collectionName, key: { tag: "hashed" } }); } catch (ex) { @@ -20,5 +32,12 @@ export class PackageServiceInitializer } } } + + private static initAppInsights(cloudRole = "package") { + appInsights.setup(); + appInsights.defaultClient.context.tags[appInsights.defaultClient.context.keys.cloudRole] = cloudRole; + appInsights.start(); + console.log('Application Insights started'); + } } diff --git a/src/shipping/package/app/main.ts b/src/shipping/package/app/main.ts index f6bd7207..e0371bed 100644 --- a/src/shipping/package/app/main.ts +++ b/src/shipping/package/app/main.ts @@ -3,11 +3,11 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -import { PackageService } from './server'; import { PackageServiceInitializer } from './initializer' +import { PackageService } from './server'; import { Settings } from './util/settings'; -PackageServiceInitializer.initialize(Settings.connectionString(), Settings.collectionName()) +PackageServiceInitializer.initialize(Settings.connectionString(), Settings.collectionName(), Settings.containerName()) .then(_ => { PackageService.start(); }); diff --git a/src/shipping/package/app/models/repository.ts b/src/shipping/package/app/models/repository.ts index de623168..f190ac89 100644 --- a/src/shipping/package/app/models/repository.ts +++ b/src/shipping/package/app/models/repository.ts @@ -22,7 +22,7 @@ export class Repository static async initialize(connection: string) { - Repository.db = (await MongoClient.connect(connection)).db(); + Repository.db = (await MongoClient.connect(connection)); } private collection() { diff --git a/src/shipping/package/app/server.ts b/src/shipping/package/app/server.ts index bada2e9d..f5a8ad7d 100644 --- a/src/shipping/package/app/server.ts +++ b/src/shipping/package/app/server.ts @@ -30,7 +30,7 @@ export class PackageService { .catch((ex) => { console.error("failed to initialize repository - make sure a connectiong string has been configured"); console.error(ex.message); - process.exit(1); // Crash the container + process.exit(1); // Crash the container }); var packageControllers = new PackageControllers(new Repository()); @@ -38,9 +38,7 @@ export class PackageService { let app = new Koa(); // Configure logging - app.use(logger(Settings.logLevel(), function (ctx) { - return ctx.headers[Settings.correlationHeader()]; - })); + app.use(logger(Settings.logLevel())); // Configure global exception handling // Use: ctx.throw('Error Message', 500); @@ -54,7 +52,7 @@ export class PackageService { if (logger) { logger.error(ex.message); } - + ctx.status = ex.status || 500; // consider api specific codes and localized messages as opposed to internal codes ctx.body = { diff --git a/src/shipping/package/app/util/logging.ts b/src/shipping/package/app/util/logging.ts index 50463679..c01a7cd2 100644 --- a/src/shipping/package/app/util/logging.ts +++ b/src/shipping/package/app/util/logging.ts @@ -4,7 +4,7 @@ // ------------------------------------------------------------ import * as winston from 'winston'; -import {Context} from "koa"; + export interface ILogger { log(level: string, msg: string, meta?: any) debug(msg: string, meta?: any) @@ -12,27 +12,35 @@ export interface ILogger { warn(msg: string, meta?: any) error(msg: string, meta?: any) } -export type CorrelationIdFn = (ctx: Context) => string; -export function logger(level: string, getCorrelationId: CorrelationIdFn) { - - winston.configure({ +export function logger(level: string) { + + winston.configure({ level: level, - transports: [ new (winston.transports.Console)()] + transports: [ + new (winston.transports.Console)({ + level: level, + colorize: true, + timestamp: true + })] }); return async function(ctx: any, next: any) { - ctx.state.logger = new WinstonLogger(getCorrelationId(ctx)); + ctx.state.logger = new WinstonLogger(); await next(); } } class WinstonLogger { - constructor(private correlationId: string) {} + constructor() {} log(level: string, msg: string, payload?: any) { var meta : any = {}; - if (payload) { meta.payload = payload }; - if (this.correlationId) { meta.correlationId = this.correlationId } - winston.log(level, msg, meta) + if (payload) { + winston.log(level, msg, payload); + } + else + { + winston.log(level, msg); + } } - + info(msg: string, payload?: any) { this.log('info', msg, payload); } diff --git a/src/shipping/package/app/util/settings.ts b/src/shipping/package/app/util/settings.ts index 5695bf71..786d9e24 100644 --- a/src/shipping/package/app/util/settings.ts +++ b/src/shipping/package/app/util/settings.ts @@ -14,11 +14,11 @@ export class Settings { return process.env["CONNECTION_STRING"] } - static correlationHeader() : string { - return process.env["CORRELATION_HEADER"] + static containerName() : string { + return process.env["CONTAINER_NAME"] } static logLevel() : string { return process.env["LOG_LEVEL"] || 'debug' } -} \ No newline at end of file +} diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 1c04bd9b..643191a9 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -23,6 +23,7 @@ "author": "Microsoft Patterns and Practices", "license": "MIT", "dependencies": { + "applicationinsights": "^1.1.0", "fleek-context": "^1.0.6", "fleek-router": "^2.1.0", "fleek-validator": "^2.0.1", @@ -33,8 +34,8 @@ "koa-convert": "^1.2.0", "koa-cors": "0.0.16", "koa-route": "^3.2.0", - "mongodb": "^3.1.8", - "winston": "^3.1.0" + "mongodb": "2.2.28", + "winston": "^2.4.0" }, "devDependencies": { "@types/koa": "^2.0.46", From 6a6c9a6f0a2a2af87201df491b1447e6d721f59a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 6 Feb 2019 13:46:57 +0000 Subject: [PATCH 066/246] Merged PR 649: add Kubernetes AppInsights for Workflow add Kubernetes AppInsights for Workflow - code light up for distributed tracing - remove correlation enricher - align deployment instructions - add ikey keyvault secret resolved: #8592 Related work items: #8592 --- deployment.md | 4 ++++ k8s/workflow.yaml | 6 ++---- .../Fabrikam.Workflow.Service.csproj | 4 +++- .../workflow/Fabrikam.Workflow.Service/ServiceStartup.cs | 7 +++++++ .../workflow/Fabrikam.Workflow.Service/appsettings.json | 3 +-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/deployment.md b/deployment.md index ab9426b9..16da6862 100644 --- a/deployment.md +++ b/deployment.md @@ -317,6 +317,7 @@ export WORKFLOW_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-workflow-kv" az keyvault create --name $WORKFLOW_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueName --value ${INGESTION_QUEUE_NAME} az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueEndpoint --value ${INGESTION_QUEUE_NAMESPACE_ENDPOINT} +az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name ApplicationInsights-InstrumentationKey --value ${AI_IKEY} export WORKFLOW_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "id" --output tsv) export WORKFLOW_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) @@ -324,6 +325,9 @@ export WORKFLOW_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP Create and set up pod identity + +> Note: after creating the identity, please wait for some time before assigning Reader role to the Workflow Principal Id. Otherwise it's possible to experience the following error: ```No matches in graph database for 'your-principal-id'``` + ```bash # Create the identity and extract properties export WORKFLOW_PRINCIPAL_NAME="workflow" diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index a328a03d..9918186a 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -52,8 +52,6 @@ spec: value: http://package/api/packages/ - name: no_proxy value: 169.254.169.254 - - name: CORRELATION_HEADER - value: l5d-ctx-trace volumes: - name: workflow flexVolume: @@ -61,8 +59,8 @@ spec: options: usepodidentity: "true" keyvaultname: "keyVaultName" - keyvaultobjectnames: QueueName;QueueEndpoint - keyvaultobjecttypes: secret;secret + keyvaultobjectnames: QueueName;QueueEndpoint;ApplicationInsights-InstrumentationKey + keyvaultobjecttypes: secret;secret;secret keyvaultobjectversions: "" resourcegroup: "keyVaultResourceGroup" subscriptionid: "keyVaultSubscriptionId" diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index fb10a795..cc9b26c3 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -1,4 +1,4 @@ - + Exe @@ -7,6 +7,8 @@ + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index 2045cb33..3acb82ca 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -13,10 +13,17 @@ namespace Fabrikam.Workflow.Service { public static class ServiceStartup { + private const string AppInsightsInstrumentationKey = "ApplicationInsights-InstrumentationKey"; + public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { services.AddOptions(); + // Configure AppInsights + services.AddApplicationInsightsKubernetesEnricher(); + services.AddApplicationInsightsTelemetry( + context.Configuration[AppInsightsInstrumentationKey]); + services.Configure(context.Configuration); services.AddHostedService(); diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json index 8e23600c..484bbe90 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json @@ -3,8 +3,7 @@ "IncludeScopes": false, "LogLevel": { "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" + } }, "Serilog": { "MinimumLevel": "Verbose", From ae3895743301a4cc7206db6bcc0a07374813a4a5 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Thu, 7 Feb 2019 13:24:55 +0000 Subject: [PATCH 067/246] Merged PR 651: Implement polly policies for resiliency set up policies. defaults are set based on recommendations from p&p for retries and from polly for circuit breaker https://github.com/App-vNext/Polly/wiki/Advanced-Circuit-Breaker#configuration-recommendations. For bulkhead there is no concrete recommendation ("it depends" https://github.com/App-vNext/Polly/wiki/Bulkhead#configuration-recommendations) Also removes the now unused delivery history service. Added tests in one of the service callers. Please check the comments associated to the changes. Related work items: #8059 --- k8s/workflow.yaml | 14 ++ .../DeliveryServiceCallerIntegrationTests.cs | 24 ++- ...eSchedulerServiceCallerIntegrationTests.cs | 22 ++- .../PackageServiceCallerIntegrationTests.cs | 141 ++++++++++++++++-- .../RequestProcessorIntegrationTests.cs | 28 ++-- .../ResiliencyEnvironmentVariablesFixture.cs | 23 +++ .../ResiliencyExtensions.cs | 47 ++++++ .../ServiceStartup.cs | 38 +++-- 8 files changed, 270 insertions(+), 67 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ResiliencyEnvironmentVariablesFixture.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/ResiliencyExtensions.cs diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 9918186a..9bbe08b0 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -50,6 +50,20 @@ spec: value: http://dronescheduler/api/DroneDeliveries/ - name: SERVICE_URI_PACKAGE value: http://package/api/packages/ + - name: SERVICEREQUEST__MAXRETRIES + value: 3 + - name: SERVICEREQUEST__CIRCUITBREAKERTHRESHOLD + value: 0.5 + - name: SERVICEREQUEST__CIRCUITBREAKERSAMPLINGPERIODSECONDS + value: 5 + - name: SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT + value: 20 + - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATION + value: 30 + - name: SERVICEREQUEST__MAXBULKHEADSIZE + value: 100 + - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE + value: 25 - name: no_proxy value: 169.254.169.254 volumes: diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs index abfb05da..bfbe43f4 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; @@ -29,7 +28,7 @@ namespace Fabrikam.Workflow.Service.Tests { - public class DeliveryServiceCallerIntegrationTests : IDisposable + public class DeliveryServiceCallerIntegrationTests : IDisposable, IClassFixture { private const string DeliveryHost = "deliveryhost"; private static readonly string DeliveryUri = $"http://{DeliveryHost}/api/Deliveries/"; @@ -43,11 +42,10 @@ public DeliveryServiceCallerIntegrationTests() { var context = new HostBuilderContext(new Dictionary()); context.Configuration = - new ConfigurationBuilder().AddInMemoryCollection( - new Dictionary - { - ["SERVICE_URI_DELIVERY"] = DeliveryUri - }).Build(); + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["SERVICE_URI_DELIVERY"] = DeliveryUri }) + .AddEnvironmentVariables() + .Build(); context.HostingEnvironment = Mock.Of(e => e.EnvironmentName == "Test"); @@ -93,11 +91,11 @@ public async Task WhenSchedulingDelivery_ThenInvokesDeliveryAPI() actualDeliveryId = ctx.Request.Path; actualDeliverySchedule = new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); - ctx.Response.StatusCode = (int)HttpStatusCode.Created; + ctx.Response.StatusCode = StatusCodes.Status201Created; } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; @@ -133,12 +131,12 @@ await ctx.WriteResultAsync( new ObjectResult( new DeliverySchedule { Id = "someDeliveryId" }) { - StatusCode = (int)HttpStatusCode.Created + StatusCode = StatusCodes.Status201Created }); } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } }; @@ -161,11 +159,11 @@ public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() { if (ctx.Request.Host.Host == DeliveryHost) { - ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + ctx.Response.StatusCode = StatusCodes.Status400BadRequest; } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs index 5bf58f2e..4e3a54a1 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; @@ -29,7 +28,7 @@ namespace Fabrikam.Workflow.Service.Tests { - public class DroneSchedulerServiceCallerIntegrationTests : IDisposable + public class DroneSchedulerServiceCallerIntegrationTests : IDisposable, IClassFixture { private const string DroneSchedulerHost = "dronescheduler"; private static readonly string DroneSchedulerUri = $"http://{DroneSchedulerHost}/api/DroneDeliveries/"; @@ -43,11 +42,10 @@ public DroneSchedulerServiceCallerIntegrationTests() { var context = new HostBuilderContext(new Dictionary()); context.Configuration = - new ConfigurationBuilder().AddInMemoryCollection( - new Dictionary - { - ["SERVICE_URI_DRONE"] = DroneSchedulerUri - }).Build(); + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["SERVICE_URI_DRONE"] = DroneSchedulerUri }) + .AddEnvironmentVariables() + .Build(); context.HostingEnvironment = Mock.Of(e => e.EnvironmentName == "Test"); @@ -96,7 +94,7 @@ public async Task WhenGettingDroneId_ThenInvokesDroneSchedulerAPI() } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; @@ -126,11 +124,11 @@ public async Task WhenDroneSchedulerAPIReturnsOK_ThenReturnsDroneId() { if (ctx.Request.Host.Host == DroneSchedulerHost) { - await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = (int)HttpStatusCode.OK }); + await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = StatusCodes.Status200OK }); } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } }; @@ -152,11 +150,11 @@ public async Task WhenDroneSchedulerAPIDoesNotReturnOK_ThenThrows() { if (ctx.Request.Host.Host == DroneSchedulerHost) { - ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + ctx.Response.StatusCode = StatusCodes.Status400BadRequest; } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs index 14cdc302..e6deb305 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs @@ -6,8 +6,9 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; +using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -29,7 +30,7 @@ namespace Fabrikam.Workflow.Service.Tests { - public class PackageServiceCallerIntegrationTests : IDisposable + public class PackageServiceCallerIntegrationTests : IDisposable, IClassFixture { private const string PackageHost = "packagehost"; private static readonly string PackageUri = $"http://{PackageHost}/api/packages/"; @@ -43,11 +44,10 @@ public PackageServiceCallerIntegrationTests() { var context = new HostBuilderContext(new Dictionary()); context.Configuration = - new ConfigurationBuilder().AddInMemoryCollection( - new Dictionary - { - ["SERVICE_URI_PACKAGE"] = PackageUri - }).Build(); + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["SERVICE_URI_PACKAGE"] = PackageUri }) + .AddEnvironmentVariables() + .Build(); context.HostingEnvironment = Mock.Of(e => e.EnvironmentName == "Test"); @@ -93,11 +93,11 @@ public async Task WhenCreatingPackage_ThenInvokesDroneSchedulerAPI() actualPackageId = ctx.Request.Path; actualPackage = new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); - ctx.Response.StatusCode = (int)HttpStatusCode.Created; + ctx.Response.StatusCode = StatusCodes.Status201Created; } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; @@ -126,12 +126,12 @@ await ctx.WriteResultAsync( new ObjectResult( new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) { - StatusCode = (int)HttpStatusCode.Created + StatusCode = StatusCodes.Status201Created }); } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } }; @@ -151,11 +151,11 @@ public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() { if (ctx.Request.Host.Host == PackageHost) { - ctx.Response.StatusCode = (int)HttpStatusCode.BadRequest; + ctx.Response.StatusCode = StatusCodes.Status400BadRequest; } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } return Task.CompletedTask; @@ -165,6 +165,121 @@ public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() await Assert.ThrowsAsync(() => _caller.CreatePackageAsync(packageInfo)); } + + [Fact] + public async Task WhenRequestsFail_TheyAreRetried() + { + var receivedRequests = 0; + + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = Interlocked.Increment(ref receivedRequests) <= 2 ? StatusCodes.Status500InternalServerError : StatusCodes.Status201Created + }); + } + else + { + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + } + }; + + var result = + await _caller.CreatePackageAsync(new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + + Assert.Equal(3, receivedRequests); + } + + [Fact] + public async Task WhenMultipleRequestsAreIssued_ThenBulkheadRestrictsConcurrentAccess() + { + const int totalRequests = 20; + var receivedRequests = 0; + var successfulRequests = 0; + var failedRequests = 0; + + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost) + { + Interlocked.Increment(ref receivedRequests); + await Task.Delay(TimeSpan.FromSeconds(1)); + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = StatusCodes.Status201Created + }); + } + else + { + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + } + }; + + var requestTasks = + Enumerable.Range(1, totalRequests) + .Select(async i => + { + try + { + var result = + await _caller.CreatePackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + Interlocked.Increment(ref successfulRequests); + } + catch + { + Interlocked.Increment(ref failedRequests); + } + }); + await Task.WhenAll(requestTasks); + + Assert.Equal(8, receivedRequests); + Assert.Equal(8, successfulRequests); + Assert.Equal(12, failedRequests); + } + + [Fact] + public async Task WhenRequestsFailAboveTheThreshold_ThenCircuitBreakerBreaks() + { + const int totalRequests = 100; + var receivedRequests = 0; + var successfulRequests = 0; + var failedRequests = 0; + + _handleHttpRequest = ctx => + { + Interlocked.Increment(ref receivedRequests); + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + return Task.CompletedTask; + }; + + var requestTasks = + Enumerable.Range(1, totalRequests) + .Select(async i => + { + try + { + await Task.Delay(TimeSpan.FromSeconds(i / 10)); + var result = + await _caller.CreatePackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + Interlocked.Increment(ref successfulRequests); + } + catch + { + Interlocked.Increment(ref failedRequests); + } + }); + await Task.WhenAll(requestTasks); + + Assert.NotEqual(totalRequests * 4, receivedRequests); + Assert.Equal(0, successfulRequests); + Assert.Equal(totalRequests, failedRequests); + } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs index d1ed71ac..a07fc772 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; @@ -29,7 +28,7 @@ namespace Fabrikam.Workflow.Service.Tests { - public class RequestProcessorIntegrationTests : IDisposable + public class RequestProcessorIntegrationTests : IDisposable, IClassFixture { private const string DeliveryHost = "deliveryhost"; private static readonly string DeliveryUri = $"http://{DeliveryHost}/api/Deliveries/"; @@ -46,13 +45,16 @@ public RequestProcessorIntegrationTests() { var context = new HostBuilderContext(new Dictionary()); context.Configuration = - new ConfigurationBuilder().AddInMemoryCollection( - new Dictionary - { - ["SERVICE_URI_DELIVERY"] = DeliveryUri, - ["SERVICE_URI_DRONE"] = DroneSchedulerUri, - ["SERVICE_URI_PACKAGE"] = PackageUri - }).Build(); + new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["SERVICE_URI_DELIVERY"] = DeliveryUri, + ["SERVICE_URI_DRONE"] = DroneSchedulerUri, + ["SERVICE_URI_PACKAGE"] = PackageUri + }) + .AddEnvironmentVariables() + .Build(); context.HostingEnvironment = Mock.Of(e => e.EnvironmentName == "Test"); @@ -104,14 +106,14 @@ await ctx.WriteResultAsync( new ObjectResult( new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) { - StatusCode = (int)HttpStatusCode.Created + StatusCode = StatusCodes.Status201Created }); } else if (ctx.Request.Host.Host == DroneSchedulerHost) { actualDelivery = serializer.Deserialize(new JsonTextReader(new StreamReader(ctx.Request.Body, Encoding.UTF8))); - await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = (int)HttpStatusCode.OK }); + await ctx.WriteResultAsync(new ContentResult { Content = "someDroneId", StatusCode = StatusCodes.Status200OK }); } else if (ctx.Request.Host.Host == DeliveryHost) { @@ -120,12 +122,12 @@ await ctx.WriteResultAsync( await ctx.WriteResultAsync( new ObjectResult(new DeliverySchedule { Id = "someDeliveryId" }) { - StatusCode = (int)HttpStatusCode.Created + StatusCode = StatusCodes.Status201Created }); } else { - ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; } }; diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ResiliencyEnvironmentVariablesFixture.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ResiliencyEnvironmentVariablesFixture.cs new file mode 100644 index 00000000..2335cc94 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ResiliencyEnvironmentVariablesFixture.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class ResiliencyEnvironmentVariablesFixture + { + public ResiliencyEnvironmentVariablesFixture() + { + Environment.SetEnvironmentVariable("SERVICEREQUEST__MAXRETRIES", "3"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__CIRCUITBREAKERTHRESHOLD", "0.75"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__CIRCUITBREAKERSAMPLINGPERIODSECONDS", "5"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT", "2"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__CIRCUITBREAKERBREAKDURATIONSECONDS", "5"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__MAXBULKHEADSIZE", "5"); + Environment.SetEnvironmentVariable("SERVICEREQUEST__MAXBULKHEADQUEUESIZE", "3"); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ResiliencyExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ResiliencyExtensions.cs new file mode 100644 index 00000000..a80d5fd3 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ResiliencyExtensions.cs @@ -0,0 +1,47 @@ +using System; +using System.Net.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Polly; + +namespace Fabrikam.Workflow.Service +{ + internal static class ResiliencyExtensions + { + public static IHttpClientBuilder AddResiliencyPolicies(this IHttpClientBuilder builder, IConfiguration configuration) + { + var resiliencyConfiguration = configuration.GetSection("ServiceRequest").Get(); + + builder + .AddPolicyHandler( + Policy.BulkheadAsync(resiliencyConfiguration.MaxBulkheadSize, resiliencyConfiguration.MaxBulkheadQueueSize)) + .AddTransientHttpErrorPolicy(p => + p.AdvancedCircuitBreakerAsync( + resiliencyConfiguration.CircuitBreakerThreshold, + TimeSpan.FromSeconds(resiliencyConfiguration.CircuitBreakerSamplingPeriodSeconds), + resiliencyConfiguration.CircuitBreakerMinimumThroughput, + TimeSpan.FromSeconds(resiliencyConfiguration.CircuitBreakerBreakDurationSeconds))) + .AddTransientHttpErrorPolicy(p => + p.WaitAndRetryAsync(resiliencyConfiguration.MaxRetries, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) - 2))); + + return builder; + } + + private class ResiliencyConfiguration + { + public int MaxRetries { get; set; } + + public double CircuitBreakerThreshold { get; set; } + + public int CircuitBreakerSamplingPeriodSeconds { get; set; } + + public int CircuitBreakerMinimumThroughput { get; set; } + + public int CircuitBreakerBreakDurationSeconds { get; set; } + + public int MaxBulkheadSize { get; set; } + + public int MaxBulkheadQueueSize { get; set; } + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index 3acb82ca..cb36dbfc 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -4,10 +4,10 @@ // ------------------------------------------------------------ using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using Fabrikam.Workflow.Service.RequestProcessing; using Fabrikam.Workflow.Service.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; namespace Fabrikam.Workflow.Service { @@ -29,20 +29,26 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect services.AddTransient(); - services.AddHttpClient(c => - { - c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_PACKAGE"]); - }); - - services.AddHttpClient(c => - { - c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DRONE"]); - }); - - services.AddHttpClient(c => - { - c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DELIVERY"]); - }); + services + .AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_PACKAGE"]); + }) + .AddResiliencyPolicies(context.Configuration); + + services + .AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DRONE"]); + }) + .AddResiliencyPolicies(context.Configuration); + + services + .AddHttpClient(c => + { + c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DELIVERY"]); + }) + .AddResiliencyPolicies(context.Configuration); } } } From b2f5647fb1a0f4b4f8a2b8efd8a4ee9ab941cef4 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 11 Feb 2019 13:24:58 +0000 Subject: [PATCH 068/246] Merged PR 655: Update ARM template deployment Bring the ARM template up to date: - provision aks instead of acs - provision ingestion queue instead of event hub - create managed identities - create key vaults - create app insights - set up necessary role assignments Update deployment instructions accordingly Note: mapped the CLI deployment instructions to the ARM instructions, replacing resource creation with output extraction as much as possible. Related work items: #8708, #8709 --- azuredeploy.json | 1156 ++++++++++++++++++++++++++++++---------------- deployment.md | 32 +- deploymentARM.md | 379 ++++++++------- 3 files changed, 980 insertions(+), 587 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 34c7169b..6ae90ebf 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -1,423 +1,759 @@ { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "sshRSAPublicKey": { - "type": "string", - "metadata": { - "description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example 'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm'" - } - }, - "servicePrincipalClientId": { - "metadata": { - "description": "Client ID (used by cloudprovider)" - }, - "type": "string" - }, - "servicePrincipalClientSecret": { - "metadata": { - "description": "The Service Principal Client Secret." - }, - "type": "securestring" - }, - "agentCount": { - "type": "int", - "defaultValue": 3, - "metadata": { - "description": "The number of agents for the cluster. This value can be from 1 to 100 (note, for Kubernetes clusters you will also get 1 or 2 public agents in addition to these seleted masters)" - }, - "minValue":1, - "maxValue":100 - }, - "agentVMSize": { - "type": "string", - "defaultValue": "Standard_D2_v2" , - "allowedValues": [ - "Standard_D2_v2", - "Standard_F8s_v2" - ], - "metadata": { - "description": "The size of the Virtual Machine." - } - }, - "adminUsername": { - "type": "string", - "metadata": { - "description": "User name for the Linux Virtual Machines." - }, - "defaultValue" : "azureuser" - }, - "masterCount": { - "type": "int", - "defaultValue": 1, - "allowedValues": [ - 1 - ], - "metadata": { - "description": "The number of Kubernetes masters for the cluster." - } - }, - "acrStorageType": { - "type": "string", - "defaultValue": "Standard_LRS", - "allowedValues": [ - "Standard_LRS", - "Standard_ZRS", - "Standard_GRS" - ], - "metadata": { - "description": "Type of the storage account that will store container registry datas." - } - }, - "deliveryRedisStorageType": { - "type": "string", - "defaultValue": "Standard_LRS", - "allowedValues": [ - "Standard_LRS", - "Standard_ZRS", - "Standard_GRS" - ], - "metadata": { - "description": "Type of the storage account that will store Redis Cache." - } - }, - "deliveryRedisCacheSKU": { - "type": "string", - "allowedValues": [ - "Basic", - "Standard", - "Premium" - ], - "defaultValue": "Premium", - "metadata": { - "description": "The pricing tier of the new Azure Redis Cache." - } - }, - "deliveryRedisCacheFamily": { - "type": "string", - "defaultValue": "P", - "metadata": { - "description": "The family for the sku." - }, - "allowedValues": [ - "C", - "P" - ] - }, - "deliveryRedisCacheCapacity": { - "type": "int", - "allowedValues": [ - 1, - 2, - 3, - 4 - ], - "defaultValue": 4, - "metadata": { - "description": "The size of the new Azure Redis Cache instance. " - } - }, - "deliveryRedisDiagnosticsEnabled": { - "type": "bool", - "allowedValues": [ - false, - true - ], - "defaultValue": false, - "metadata": { - "description": "A value that indicates whether diagnostics should be saved to the specified storage account." - } - }, - "schedulerStorageType": { - "type": "string", - "defaultValue": "Standard_LRS", - "allowedValues": [ - "Standard_LRS", - "Standard_ZRS", - "Standard_GRS" - ], - "metadata": { - "description": "Type of the storage account that will store Scheduler checkpoints." - } - } - }, - "variables": { - "agentsEndpointDNSNamePrefix":"[concat('agents',uniqueString(resourceGroup().id))]", - "mastersEndpointDNSNamePrefix":"[concat('mgmt',uniqueString(resourceGroup().id))]", - "useServicePrincipalDictionary": { - "DCOS": 0, - "Swarm": 0, - "Kubernetes": 1 - }, - "useServicePrincipal": "[variables('useServicePrincipalDictionary')['Kubernetes']]", - "servicePrincipalFields": [ - null, - { - "ClientId": "[parameters('servicePrincipalClientId')]", - "Secret": "[parameters('servicePrincipalClientSecret')]" - } - ], - "acsK8sClusterName":"[concat('acsK8sCluster-',resourceGroup().name)]", - "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", - "acrStorageName": "[concat('acrsto', uniqueString(resourceGroup().id))]", - "acrStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('acrStorageName'))]", - "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", - "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", - "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-cosmosdb')]", - "deliveryRedisName": "[concat(uniqueString(resourceGroup().id),'-delivery-service-redis')]", - "packageMongoDbName": "[concat(uniqueString(resourceGroup().id),'-package-service-cosmosdb')]", - "ingestionEHNamespace": "[concat(uniqueString(resourceGroup().id),'ingehns')]", - "ingestionEHName": "[concat(uniqueString(resourceGroup().id),'ingeh')]", - "ingestionEHConsumerGroupName": "[concat(uniqueString(resourceGroup().id),'ingehcg')]", - "schedulerStorageName": "[concat('schsto', uniqueString(resourceGroup().id))]", - "schedulerStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('schedulerStorageName'))]" - }, - "resources": [ - { - "apiVersion": "2016-09-30", - "type": "Microsoft.ContainerService/containerServices", - "location": "[resourceGroup().location]", - "name":"[variables('acsK8sClusterName')]", - "properties": { - "orchestratorProfile": { - "orchestratorType": "Kubernetes" - }, - "masterProfile": { - "count": "[parameters('masterCount')]", - "dnsPrefix": "[variables('mastersEndpointDNSNamePrefix')]" - }, - "agentPoolProfiles": [ - { - "name": "agentpools", - "count": "[parameters('agentCount')]", - "vmSize": "[parameters('agentVMSize')]", - "dnsPrefix": "[variables('agentsEndpointDNSNamePrefix')]" - } - ], - "linuxProfile": { - "adminUsername": "[parameters('adminUsername')]", - "ssh": { - "publicKeys": [ - { - "keyData": "[parameters('sshRSAPublicKey')]" - } + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sshRSAPublicKey": { + "type": "string", + "metadata": { + "description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example 'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm'" + } + }, + "servicePrincipalClientId": { + "metadata": { + "description": "Client ID (used by cloudprovider)" + }, + "type": "string" + }, + "servicePrincipalClientSecret": { + "metadata": { + "description": "The Service Principal Client Secret." + }, + "type": "securestring" + }, + "servicePrincipalId": { + "metadata": { + "description": "Principal ID (used by cloudprovider)" + }, + "type": "string" + }, + "agentCount": { + "type": "int", + "defaultValue": 3, + "metadata": { + "description": "The number of agents for the cluster. This value can be from 1 to 100 (note, for Kubernetes clusters you will also get 1 or 2 public agents in addition to these seleted masters)" + }, + "minValue": 1, + "maxValue": 100 + }, + "agentVMSize": { + "type": "string", + "defaultValue": "Standard_D2_v2", + "allowedValues": [ + "Standard_D2_v2", + "Standard_F8s_v2" + ], + "metadata": { + "description": "The size of the Virtual Machine." + } + }, + "osType": { + "type": "string", + "defaultValue": "Linux", + "allowedValues": [ + "Linux" + ], + "metadata": { + "description": "The type of operating system." + } + }, + "osDiskSizeGB": { + "type": "int", + "defaultValue": 0, + "metadata": { + "description": "Disk size (in GB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize." + }, + "minValue": 0, + "maxValue": 1023 + }, + "adminUsername": { + "type": "string", + "metadata": { + "description": "User name for the Linux Virtual Machines." + }, + "defaultValue": "azureuser" + }, + "kubernetesVersion": { + "type": "string", + "defaultValue": "1.12.4", + "allowedValues": [ + "1.9.10", + "1.9.11", + "1.10.9", + "1.10.12", + "1.11.5", + "1.11.6", + "1.12.4" + ], + "metadata": { + "description": "The version of Kubernetes." + } + }, + "deliveryRedisStorageType": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Standard_LRS", + "Standard_ZRS", + "Standard_GRS" + ], + "metadata": { + "description": "Type of the storage account that will store Redis Cache." + } + }, + "deliveryRedisCacheSKU": { + "type": "string", + "allowedValues": [ + "Basic", + "Standard", + "Premium" + ], + "defaultValue": "Premium", + "metadata": { + "description": "The pricing tier of the new Azure Redis Cache." + } + }, + "deliveryRedisCacheFamily": { + "type": "string", + "defaultValue": "P", + "metadata": { + "description": "The family for the sku." + }, + "allowedValues": [ + "C", + "P" ] - } }, - "servicePrincipalProfile": "[variables('servicePrincipalFields')[variables('useServicePrincipal')]]" - } - }, - { - "name": "[variables('acrStorageName')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "comments": "This storage account is used by Container Registry for storing its datas.", - "dependsOn": [], - "tags": { - "displayName": "ACR Image's storage", - "container.registry": "[variables('acrName')]" - }, - "properties": { - "accountType": "[parameters('acrStorageType')]" - } - }, - { - "name": "[variables('acrName')]", - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2016-06-27-preview", - "location": "[resourceGroup().location]", - "comments": "Container registry for storing docker images", - "dependsOn": [ - "[variables('acrStorageId')]" - ], - "tags": { - "displayName": "Container Registry", - "container.registry": "[variables('acrName')]" - }, - "properties": { - "adminUserEnabled": "true", - "storageAccount": { - "accessKey": "[listKeys(variables('acrStorageId'),'2015-06-15').key1]", - "name": "[variables('acrStorageName')]" + "deliveryRedisCacheCapacity": { + "type": "int", + "allowedValues": [ + 1, + 2, + 3, + 4 + ], + "defaultValue": 4, + "metadata": { + "description": "The size of the new Azure Redis Cache instance. " + } + }, + "deliveryRedisDiagnosticsEnabled": { + "type": "bool", + "allowedValues": [ + false, + true + ], + "defaultValue": false, + "metadata": { + "description": "A value that indicates whether diagnostics should be saved to the specified storage account." + } } - } }, - { - "name": "[variables('deliveryRedisStorageName')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "comments": "This storage account is used by Delivery Redis", - "dependsOn": [], - "tags": { - "displayName": "Delivery Redis storage" - }, - "properties": { - "accountType": "[parameters('deliveryRedisStorageType')]" - } + "variables": { + "deliveryIdName": "delivery", + "workflowIdName": "workflow", + "droneSchedulerIdName": "dronescheduler", + "aksClusterName": "[concat('aksCluster-', uniqueString(resourceGroup().id))]", + "aksClusterDnsNamePrefix": "[concat('cluster-',uniqueString(resourceGroup().id))]", + "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", + "appInsightsName": "[concat('ai', uniqueString(resourceGroup().id))]", + "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id), '-delivery-service-cosmosdb')]", + "deliveryRedisName": "[concat(uniqueString(resourceGroup().id), '-delivery-service-redis')]", + "deliveryKeyVaultName": "[concat(resourceGroup().name, '-d-kv')]", + "packageMongoDbName": "[concat(uniqueString(resourceGroup().id),'-package-service-cosmosdb')]", + "ingestionSBNamespace": "[concat(uniqueString(resourceGroup().id), 'ingsbns')]", + "ingestionSBName": "[concat(uniqueString(resourceGroup().id), 'ingsb')]", + "ingestionServiceAccessKey": "IngestionServiceAccessKey", + "droneSchedulerKeyVaultName": "[concat(resourceGroup().name, '-ds-kv')]", + "workflowKeyVaultName": "[concat(resourceGroup().name, '-wf-kv')]", + "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", + "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", + "contributorRoleObjectId": "b24988ac-6180-42a0-ab88-20f7382dd24c", + "contributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('contributorRoleObjectId'))]", + "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", + "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]" }, - { - "apiVersion": "2015-08-01", - "name": "[variables('deliveryRedisName')]", - "type": "Microsoft.Cache/Redis", - "location": "[resourceGroup().location]", - "dependsOn": [ - "[variables('deliveryRedisStorageId')]" - ], - "properties": { - "redisEnableNonSslPort": "false", - "sku": { - "capacity": "[parameters('deliveryRedisCacheCapacity')]", - "family": "[parameters('deliveryRedisCacheFamily')]", - "name": "[parameters('deliveryRedisCacheSKU')]" - }, - "vm-size": "P4" - }, - "resources": [ + "resources": [ + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('workflowIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('deliveryIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('droneSchedulerIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + }, + { + "name": "[variables('aksClusterName')]", + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2018-03-31", + "location": "[resourceGroup().location]", + "properties": { + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "dnsPrefix": "[variables('aksClusterDnsNamePrefix')]", + "agentPoolProfiles": [ + { + "name": "agentpool", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "count": "[parameters('agentCount')]", + "vmSize": "[parameters('agentVMSize')]", + "osType": "[parameters('osType')]", + "storageProfile": "ManagedDisks" + } + ], + "linuxProfile": { + "adminUsername": "[parameters('adminUsername')]", + "ssh": { + "publicKeys": [ + { + "keyData": "[parameters('sshRSAPublicKey')]" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "[parameters('servicePrincipalClientId')]", + "secret": "[parameters('servicePrincipalClientSecret')]" + }, + "enableRBAC": true + } + }, + { + "name": "[variables('acrName')]", + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2017-10-01", + "sku": { + "name": "Basic", + "tier": "Basic" + }, + "location": "[resourceGroup().location]", + "tags": { + "displayName": "Container Registry", + "container.registry": "[variables('acrName')]" + }, + "properties": { + "adminUserEnabled": false + } + }, { - "apiVersion": "2017-05-01-preview", - "type": "Microsoft.Cache/redis/providers/diagnosticsettings", - "name": "[concat(variables('deliveryRedisName'), '/Microsoft.Insights/', variables('deliveryRedisName'))]", - "location": "[resourceGroup().location]", - "dependsOn": [ - "[concat('Microsoft.Cache/Redis/', variables('deliveryRedisName'))]" - ], - "properties": { - "storageAccountId": "[variables('deliveryRedisStorageId')]", - "logs": [], - "metrics": [ - { - "timeGrain": "AllMetrics", - "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]", - "retentionPolicy": { - "days": 90, - "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]" + "name": "[variables('appInsightsName')]", + "type": "Microsoft.Insights/components", + "apiVersion": "2015-05-01", + "kind": "other", + "location": "[resourceGroup().location]", + "properties": { + "Application_Type": "other" + } + }, + { + "name": "[variables('deliveryRedisStorageName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "comments": "This storage account is used by Delivery Redis", + "dependsOn": [], + "tags": { + "displayName": "Delivery Redis storage" + }, + "properties": { + "accountType": "[parameters('deliveryRedisStorageType')]" + } + }, + { + "apiVersion": "2016-04-01", + "name": "[variables('deliveryRedisName')]", + "type": "Microsoft.Cache/Redis", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('deliveryRedisStorageId')]" + ], + "properties": { + "redisEnableNonSslPort": "false", + "sku": { + "capacity": "[parameters('deliveryRedisCacheCapacity')]", + "family": "[parameters('deliveryRedisCacheFamily')]", + "name": "[parameters('deliveryRedisCacheSKU')]" + }, + "vm-size": "P4" + }, + "resources": [ + { + "apiVersion": "2017-05-01-preview", + "type": "Microsoft.Cache/redis/providers/diagnosticsettings", + "name": "[concat(variables('deliveryRedisName'), '/Microsoft.Insights/', variables('deliveryRedisName'))]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + ], + "properties": { + "storageAccountId": "[variables('deliveryRedisStorageId')]", + "logs": [], + "metrics": [ + { + "timeGrain": "AllMetrics", + "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]", + "retentionPolicy": { + "days": 90, + "enabled": "[parameters('deliveryRedisDiagnosticsEnabled')]" + } + } + ] + } } - } ] - } - } - ] - }, - { - "apiVersion": "2015-04-08", - "type": "Microsoft.DocumentDB/databaseAccounts", - "name": "[variables('deliveryCosmosDbName')]", - "location": "[resourceGroup().location]", - "properties": { - "name": "[variables('deliveryCosmosDbName')]", - "databaseAccountOfferType": "['Standard']", - "locations": [ - { - "locationName": "[resourceGroup().location]", - "failoverPriority": 0 - } - ] - } - }, - { - "apiVersion": "2015-04-08", - "type": "Microsoft.DocumentDB/databaseAccounts", - "kind": "MongoDB", - "name": "[variables('packageMongoDbName')]", - "location": "[resourceGroup().location]", - "properties": { - "databaseAccountOfferType": "Standard", - "name": "[variables('packageMongoDbName')]" - } - }, - { - "apiVersion":"2017-04-01", - "name":"[variables('ingestionEHNamespace')]", - "type":"Microsoft.EventHub/Namespaces", - "location":"[resourceGroup().location]", - "sku":{ - "name":"Standard" - }, - "properties": { - "isAutoInflateEnabled": "true", - "maximumThroughputUnits": "20" - }, - "resources":[ - { - "apiVersion":"2017-04-01", - "name":"[variables('ingestionEHName')]", - "type":"EventHubs", - "dependsOn":[ - "[concat('Microsoft.EventHub/namespaces/', variables('ingestionEHNamespace'))]" + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "[variables('deliveryCosmosDbName')]", + "apiVersion": "2016-03-31", + "location": "[resourceGroup().location]", + "properties": { + "name": "[variables('deliveryCosmosDbName')]", + "databaseAccountOfferType": "Standard", + "locations": [ + { + "locationName": "[resourceGroup().location]", + "failoverPriority": 0 + } + ] + } + }, + { + "apiVersion": "2015-04-08", + "type": "Microsoft.DocumentDB/databaseAccounts", + "kind": "MongoDB", + "name": "[variables('packageMongoDbName')]", + "location": "[resourceGroup().location]", + "properties": { + "databaseAccountOfferType": "Standard", + "name": "[variables('packageMongoDbName')]" + } + }, + { + "type": "Microsoft.ServiceBus/namespaces", + "name": "[variables('ingestionSBNamespace')]", + "apiVersion": "2017-04-01", + "location": "[resourceGroup().location]", + "sku": { + "name": "Standard", + "tier": "Standard" + }, + "resources": [ + { + "name": "[variables('ingestionSBName')]", + "type": "queues", + "apiVersion": "2017-04-01", + "dependsOn": [ + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + ] + }, + { + "name": "[variables('ingestionServiceAccessKey')]", + "type": "AuthorizationRules", + "apiVersion": "2017-04-01", + "properties": { + "rights": [ + "Send" + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + ] + } + ] + }, + { + "type": "Microsoft.KeyVault/vaults", + "name": "[variables('deliveryKeyVaultName')]", + "apiVersion": "2016-10-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "[variables('deliveryRedisStorageId')]" ], - "properties":{ - "messageRetentionInDays": "7", - "partitionCount": "4" + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [ + { + "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('deliveryIdName'))).tenantId]", + "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('deliveryIdName'))).principalId]", + "permissions": { + "secrets": [ + "get", + "list" + ] + } + }, + { + "tenantId": "[subscription().tenantId]", + "objectId": "[variables('readerRoleObjectId')]", + "permissions": { + "secrets": [ + "get", + "list" + ] + } + } + ] }, - "resources":[ - { - "apiVersion":"2017-04-01", - "name":"[variables('ingestionEHConsumerGroupName')]", - "type":"ConsumerGroups", - "dependsOn":[ - "[variables('ingestionEHName')]" - ], - "properties":{ - "userMetadata": "Consumer Group for ingestion Event Hub" - } - } + "resources": [ + { + "type": "secrets", + "name": "CosmosDB-Endpoint", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))).documentEndpoint]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + ] + }, + { + "type": "secrets", + "name": "CosmosDB-Key", + "apiVersion": "2015-06-01", + "properties": { + "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName')), '2016-03-31').primaryMasterKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + ] + }, + { + "type": "secrets", + "name": "Redis-Endpoint", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))).hostName]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + ] + }, + { + "type": "secrets", + "name": "Redis-AccessKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "[listKeys(resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName')), '2016-04-01').primaryKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + ] + }, + { + "type": "secrets", + "name": "ApplicationInsights--InstrumentationKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + ] + } ] - } - ] - }, - { - "name": "[variables('schedulerStorageName')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "comments": "This storage account is used by the Scheduler checkpoints.", - "dependsOn": [], - "tags": { - "displayName": "Scheduler Checkpoints storage" - }, - "properties": { - "accountType": "[parameters('schedulerStorageType')]" - } - } - ], - "outputs": { - "acrName": { - "value": "[variables('acrName')]", - "type": "string" - }, - "acrLoginServer": { - "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('acrName')),'2016-06-27-preview').loginServer]", - "type": "string" - }, - "acsK8sClusterName": { - "value": "[variables('acsK8sClusterName')]", - "type": "string" - }, - "deliveryRedisName": { - "value": "[variables('deliveryRedisName')]", - "type": "string" - }, - "deliveryCosmosDbName": { - "value": "[variables('deliveryCosmosDbName')]", - "type": "string" - }, - "packageMongoDbName": { - "value": "[variables('packageMongoDbName')]", - "type": "string" - }, - "ingestionEHNamespace": { - "value": "[variables('ingestionEHNamespace')]", - "type": "string" - }, - "ingestionEHName": { - "value": "[variables('ingestionEHName')]", - "type": "string" - }, - "schedulerStorageAccountName": { - "value": "[listKeys(variables('schedulerStorageName'),'2016-12-01')]", - "type": "object" + }, + { + "type": "Microsoft.KeyVault/vaults", + "name": "[variables('droneSchedulerKeyVaultName')]", + "apiVersion": "2016-10-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]" + ], + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [ + { + "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('droneSchedulerIdName'))).tenantId]", + "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('droneSchedulerIdName'))).principalId]", + "permissions": { + "secrets": [ + "get", + "list" + ] + } + } + ] + }, + "resources": [ + { + "type": "secrets", + "name": "ApplicationInsights--InstrumentationKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]", + "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + ] + } + ] + }, + { + "type": "Microsoft.KeyVault/vaults", + "name": "[variables('workflowKeyVaultName')]", + "apiVersion": "2016-10-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]" + ], + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [ + { + "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('workflowIdName'))).tenantId]", + "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('workflowIdName'))).principalId]", + "permissions": { + "secrets": [ + "get", + "list" + ] + } + } + ] + }, + "resources": [ + { + "type": "secrets", + "name": "QueueName", + "apiVersion": "2015-06-01", + "properties": { + "value": "[variables('ingestionSBName')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + ] + }, + { + "type": "secrets", + "name": "QueueEndpoint", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))).serviceBusEndpoint]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + ] + }, + { + "type": "secrets", + "name": "ApplicationInsights--InstrumentationKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", + "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + ] + } + ] + }, + { + "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", + "name": "[concat(variables('deliveryKeyVaultName'), '/Microsoft.Authorization/', guid('delivery-kv', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).principalId]", + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + }, + { + "type": "Microsoft.ServiceBus/namespaces/providers/roleAssignments", + "name": "[concat(variables('ingestionSBNamespace'), '/Microsoft.Authorization/', guid('workflow-sb', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "properties": { + "roleDefinitionId": "[variables('contributorRoleId')]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "scope": "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + }, + { + "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", + "name": "[concat(variables('workflowKeyVaultName'), '/Microsoft.Authorization/', guid('workflow-kv', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + }, + { + "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", + "name": "[concat(variables('acrName'), '/Microsoft.Authorization/', guid('aks', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "comments": "Grant the AKS cluster access to the ACR instance", + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[parameters('servicePrincipalId')]", + "scope": "[resourceId('Microsoft.ContainerRegistry/registries', variables('acrName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ContainerRegistry/registries', variables('acrName'))]" + ] + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", + "name": "[concat(variables('deliveryIdName'), '/Microsoft.Authorization/', guid('msi-delivery', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "comments": "Grant the AKS cluster access to the delivery managed id", + "properties": { + "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", + "principalId": "[parameters('servicePrincipalId')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", + "name": "[concat(variables('workflowIdName'), '/Microsoft.Authorization/', guid('msi-workflow', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "comments": "Grant the AKS cluster access to the workflow managed id", + "properties": { + "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", + "principalId": "[parameters('servicePrincipalId')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", + "name": "[concat(variables('droneSchedulerIdName'), '/Microsoft.Authorization/', guid('msi-dronescheduler', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "comments": "Grant the AKS cluster access to the drone scheduler managed id", + "properties": { + "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", + "principalId": "[parameters('servicePrincipalId')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + ] + } + ], + "outputs": { + "deliveryPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "type": "string" + }, + "deliveryPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).clientId]", + "type": "string" + }, + "droneSchedulerPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", + "type": "string" + }, + "droneSchedulerPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).clientId]", + "type": "string" + }, + "workflowPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "type": "string" + }, + "workflowPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).clientId]", + "type": "string" + }, + "acrName": { + "value": "[variables('acrName')]", + "type": "string" + }, + "acrLoginServer": { + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('acrName')),'2016-06-27-preview').loginServer]", + "type": "string" + }, + "aksClusterName": { + "value": "[variables('aksClusterName')]", + "type": "string" + }, + "appInsightsName": { + "value": "[variables('appInsightsName')]", + "type": "string" + }, + "deliveryKeyVaultUri": { + "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))).vaultUri]", + "type": "string" + }, + "deliveryCosmosDbName": { + "value": "[variables('deliveryCosmosDbName')]", + "type": "string" + }, + "droneSchedulerKeyVaultUri": { + "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))).vaultUri]", + "type": "string" + }, + "packageMongoDbName": { + "value": "[variables('packageMongoDbName')]", + "type": "string" + }, + "workflowKeyVaultName": { + "value": "[variables('workflowKeyVaultName')]", + "type": "string" + }, + "ingestionQueueNamespace": { + "value": "[variables('ingestionSBNamespace')]", + "type": "string" + }, + "ingestionQueueName": { + "value": "[variables('ingestionSBName')]", + "type": "string" + }, + "ingestionServiceAccessKeyName": { + "value": "[variables('ingestionServiceAccessKey')]", + "type": "string" + } } - } -} +} \ No newline at end of file diff --git a/deployment.md b/deployment.md index 16da6862..befb1a0e 100644 --- a/deployment.md +++ b/deployment.md @@ -167,29 +167,28 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/delivery:0.1.0 ``` -Create resources +Create KeyVault and secrets ```bash +# Create the vault +export DELIVERY_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-kv" +az keyvault create --name $DELIVERY_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION + +export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) +export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) + +# Retrieve resource details export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query primaryKey -o tsv) - export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) -``` -Create KeyVault and secrets - -```bash -export DELIVERY_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-kv" -az keyvault create --name $DELIVERY_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION +# Create secrets az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Key --value ${COSMOSDB_KEY} # (consider using encoding base64 to keep the actual values) az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Endpoint --value ${COSMOSDB_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-Endpoint --value ${REDIS_ENDPOINT} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-AccessKey --value ${REDIS_KEY} az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name ApplicationInsights--InstrumentationKey --value ${AI_IKEY} - -export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) -export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) ``` Create and set up pod identity @@ -371,6 +370,7 @@ kubectl get pods -n backend ``` ## Deploy the Ingestion service + Provision Azure resources ```bash @@ -409,10 +409,10 @@ sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/i # Create secret kubectl -n backend create secret generic ingestion-secrets \ ---from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ ---from-literal=queue_name=${INGESTION_QUEUE_NAME} \ ---from-literal=queue_keyname=IngestionServiceAccessKey \ ---from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} + --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ + --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ + --from-literal=queue_keyname=IngestionServiceAccessKey \ + --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} # Deploy service kubectl --namespace backend apply -f $K8S/ingestion-0.yaml @@ -467,7 +467,7 @@ kubectl apply -f $K8S/dronescheduler-identity-0.yaml Build and publish the container image ```bash -# Build the Docker images +# Build the Docker image docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ # Push the images to ACR diff --git a/deploymentARM.md b/deploymentARM.md index 44c64ed6..45955bd4 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -7,7 +7,10 @@ - Azure suscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) -- [Docker Compose](https://docs.docker.com/compose/install/) + +> Note: in linux systems, it is possible to run the docker command without prefacing +> with sudo. For more information, please refer to [the Post-installation steps +> for linux](https://docs.docker.com/install/linux/linux-postinstall/) Clone or download this repo locally. @@ -34,7 +37,10 @@ export LOCATION=[YOUR_LOCATION_HERE] export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] -export SP_CLIENT_SECRET=$(uuidgen) +export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export TENANT_ID=$(az account show --query tenantId --output tsv) + +export K8S=./microservices-reference-implementation/k8s ``` Infrastructure Prerequisites @@ -43,11 +49,12 @@ Infrastructure Prerequisites # Log in to Azure az login -# Create a resource group and service principal for ACS +# Create a resource group and service principal for AKS +export SP_CLIENT_SECRET=$(uuidgen) + az group create --name $RESOURCE_GROUP --location $LOCATION && \ -export SP_APP_ID=$(az ad sp create-for-rbac --role="Contributor" -p $SP_CLIENT_SECRET | grep -oP '(?<="appId": ")[^"]*') && \ -export BEARER_TOKEN=$(az account get-access-token --query accessToken | sed -e 's/^"//' -e 's/"$//') && \ -export SUBS_ID=$(az account show --query id | sed -e 's/^"//' -e 's/"$//') +export SP_APP_ID=$(az ad sp create-for-rbac --role="Contributor" -p $SP_CLIENT_SECRET -o tsv --query appId) && \ +export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) ``` Deployment @@ -58,8 +65,9 @@ Deployment ```bash az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ - --parameters servicePrincipalClientId=${SP_APP_ID} \ + --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + servicePrincipalId=${SP_OBJECT_ID} \ sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" ``` @@ -69,250 +77,333 @@ Deployment > Note: > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group > 2. paste the content of your ssh-rsa public key file in the Ssh RSA Plublic Key field. - > 3. paste the $SP_APP_ID and $SP_CLIENT_SECRET in the Client Id and Secret fields. + > 3. paste the $SP_APP_ID, $SP_CLIENT_SECRET, and $SP_OBJECT_ID in the corresponding fields. Get outputs from Azure Deploy ```bash # Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value | sed -e 's/^"//' -e 's/"$//') && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value | sed -e 's/^"//' -e 's/"$//') && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acsK8sClusterName.value | sed -e 's/^"//' -e 's/"$//') +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) ``` Download kubectl and create a k8s namespace ```bash # Install kubectl -sudo az acs kubernetes install-cli +sudo az aks install-cli # Get the Kubernetes cluster credentials -az acs kubernetes get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME \ - --ssh-key-file $SSH_PRIVATE_KEY_FILE - -# Create the BC namespaces -kubectl create namespace shipping && \ -kubectl create namespace accounts && \ -kubectl create namespace dronemgmt && \ -kubectl create namespace 3rdparty +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME + +# Create namespaces +kubectl create namespace backend && \ +kubectl create namespace frontend +``` + + +Integrate Application Insights instance + +```bash +# Acquire Instrumentation Key +export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) +export AI_IKEY=$(az resource show \ + -g $RESOURCE_GROUP \ + -n $AI_NAME \ + --resource-type "Microsoft.Insights/components" \ + --query properties.InstrumentationKey \ + -o tsv) + +# add RBAC for AppInsights +kubectl apply -f $K8S/k8s-rbac-ai.yaml +``` + +## Setup AAD pod identity and key vault flexvol infrastructure + +Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol + +```bash +# setup AAD pod identity +kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml + +kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` ## Deploy the Delivery service +Extract resource details from deployment + +```bash +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ +export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ +export COLLECTION_NAME="${DATABASE_NAME}-col" && \ +export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) +``` + Build the Delivery service ```bash export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery -docker-compose -f $DELIVERY_PATH/docker-compose.ci.build.yml up ``` Build and publish the container image ```bash # Build the Docker image -docker build --pull --compress -t $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 $DELIVERY_PATH/. +docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/. # Push the image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0 - +docker push $ACR_SERVER/delivery:0.1.0 ``` -Create Kubernetes secrets +Set up pod identity ```bash -export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) -export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query primaryKey -o tsv) - -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value | sed -e 's/^"//' -e 's/"$//') && \ -export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey) && \ -export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint) - -kubectl --namespace shipping create --save-config=true secret generic delivery-storageconf \ - --from-literal=CosmosDB_Key=${COSMOSDB_KEY[@]//\"/} \ - --from-literal=CosmosDB_Endpoint=${COSMOSDB_ENDPOINT[@]//\"/} \ - --from-literal=Redis_Endpoint=${REDIS_ENDPOINT} \ - --from-literal=Redis_AccessKey=${REDIS_KEY} \ - --from-literal=EH_ConnectionString= +# Extract outputs from deployment +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalClientId.value -o tsv) + +# Deploy the identity resources +cat $K8S/delivery-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $DELIVERY_PRINCIPAL_CLIENT_ID#g" > $K8S/delivery-identity-0.yaml +kubectl apply -f $K8S/delivery-identity-0.yaml ``` Deploy the Delivery service: ```bash -# Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/fabrikam.dronedelivery.deliveryservice:0.1.0#g" ./microservices-reference-implementation/k8s/delivery.yaml - -## Update config values in the deployment YAML -sed -i "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" "./microservices-reference-implementation/k8s/delivery.yaml" && \ -sed -i "s/value: \"EH_EntityPath\"/value:/g" "./microservices-reference-implementation/k8s/delivery.yaml" +# Update the image tag and config values in the deployment YAML +sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ + sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ + sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ + sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" > $K8S/delivery-0.yaml # Deploy the service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/delivery.yaml +kubectl --namespace backend apply -f $K8S/delivery-0.yaml + +# Verify the pod is created +kubectl get pods -n backend ``` ## Deploy the Package service +Extract resource details from deployment + +```bash +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) +``` + Build the Package service ```bash export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package -# Build the app -docker-compose -f $PACKAGE_PATH/build/docker-compose.ci.build.yml up - # Build the docker image -sudo docker build -f $PACKAGE_PATH/build/prod.dockerfile -t $ACR_SERVER/package-service:0.1.0 $PACKAGE_PATH +docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH # Push the docker image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/package-service:0.1.0 +docker push $ACR_SERVER/package:0.1.0 ``` Deploy the Package service ```bash # Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/package-service:0.1.0#g" ./microservices-reference-implementation/k8s/package.yml +sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml # Create secret -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value | sed -e 's/^"//' -e 's/"$//') && \ -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString") -kubectl -n shipping create secret generic package-secrets --from-literal=mongodb-pwd=${COSMOSDB_CONNECTION[@]//\"/} +# Note: Connection strings cannot be exported as outputs in ARM deployments +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') +kubectl -n backend create \ + secret generic package-secrets \ + --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION \ + --from-literal=appinsights-ikey=$AI_IKEY # Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/package.yml +kubectl --namespace backend apply -f $K8S/package-0.yml + +# Verify the pod is created +kubectl get pods -n backend ``` -## Deploy the Ingestion service +## Deploy the Workflow service -Build the Ingestion service +Extract resource details from deployment ```bash -export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion +export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) +``` -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH && \ -docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk +Build the workflow service -# Build the docker image -docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH +```bash +export WORKFLOW_PATH=./microservices-reference-implementation/src/shipping/workflow -# Push the docker image to ACR +# Build the Docker image +docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. + +# Push the image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/ingestion:0.1.0 +docker push $ACR_SERVER/workflow:0.1.0 ``` -Deploy the Ingestion service +Create and set up pod identity ```bash -# Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" ./microservices-reference-implementation/k8s/ingestion.yaml +# Extract outputs from deployment +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowPrincipalClientId.value -o tsv) + +# Deploy the identity resources +cat $K8S/workflow-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $WORKFLOW_PRINCIPAL_CLIENT_ID#g" > $K8S/workflow-identity-0.yaml +kubectl apply -f $K8S/workflow-identity-0.yaml +``` -# Get the EventHub shared access policy name and key from the Azure Portal -export INGESTION_EH_NS=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionEHNamespace.value | sed -e 's/^"//' -e 's/"$//') && \ -export INGESTION_EH_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionEHName.value | sed -e 's/^"//' -e 's/"$//') && \ -export EH_KEYS=$(curl -X POST "https://management.azure.com/subscriptions/${SUBS_ID}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.EventHub/namespaces/${INGESTION_EH_NS}/AuthorizationRules/RootManageSharedAccessKey/ListKeys?api-version=2017-04-01" -H "Content-Type: application/json" -H "Authorization: Bearer ${BEARER_TOKEN}" -H "Content-Length: 0" --stderr - --silent) && \ -export EH_ACCESS_KEY_NAME=$(echo $EH_KEYS | grep -oP '(?<="keyName":")[^"]*') && \ -export EH_ACCESS_KEY_VALUE=$(echo $EH_KEYS | grep -oP '(?<="primaryKey":")[^"]*') && \ -export EH_CONNECTION_STRING=$(echo $EH_KEYS | grep -oP '(?<="primaryConnectionString":")[^"]*') +Deploy the Workflow service: -# Create secret -kubectl -n shipping create secret generic ingestion-secrets --from-literal=eventhub_namespace=${INGESTION_EH_NS} \ ---from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_keyname=${EH_ACCESS_KEY_NAME} \ ---from-literal=eventhub_keyvalue=${EH_ACCESS_KEY_VALUE} +```bash +# Update the image tag and config values in the deployment YAML +sed "s#image:#image: $ACR_SERVER/workflow:0.1.0#g" $K8S/workflow.yaml | \ + sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ + sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ + sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ + sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $WORKFLOW_KEYVAULT_NAME#g" > $K8S/workflow-0.yaml -# Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/ingestion.yaml +# Deploy the service +kubectl --namespace backend apply -f $K8S/workflow-0.yaml + +# Verify the pod is created +kubectl get pods -n backend ``` -## Deploy the Scheduler service +## Deploy the Ingestion service -Build the Scheduler service +Extract resource details from deployment ```bash -export SCHEDULER_PATH=./microservices-reference-implementation/src/shipping/scheduler +export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ +export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) +export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) +``` + +Build the Ingestion service + +```bash +export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion # Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $SCHEDULER_PATH/Dockerfilemaven $SCHEDULER_PATH && \ -docker run -it --rm -v $( cd "${SCHEDULER_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk +docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH +docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk # Build the docker image -docker build -f $SCHEDULER_PATH/Dockerfile -t $ACR_SERVER/scheduler:0.1.0 $SCHEDULER_PATH +docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH # Push the docker image to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/scheduler:0.1.0 +docker push $ACR_SERVER/ingestion:0.1.0 ``` -Deploy the Scheduler service +Deploy the Ingestion service ```bash # Update deployment YAML with image tage -sed -i "s#image:#image: $ACR_SERVER/scheduler:0.1.0#g" ./microservices-reference-implementation/k8s/scheduler.yaml +sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml + +# Create secret +kubectl -n backend create secret generic ingestion-secrets \ + --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ + --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ + --from-literal=queue_keyname=${INGESTION_ACCESS_KEY_NAME} \ + --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} -# Get the following values from the Azure Portal -export SCHEDULER_STORAGE_ACCOUNT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.schedulerStorageAccountName.value | sed -e 's/^"//' -e 's/"$//') && \ -export EH_KEYS=$(curl -X POST "https://management.azure.com/subscriptions/${SUBS_ID}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.EventHub/namespaces/${INGESTION_EH_NS}/AuthorizationRules/RootManageSharedAccessKey/ListKeys?api-version=2017-04-01" -H "Content-Type: application/json" -H "Authorization: Bearer ${BEARER_TOKEN}" -H "Content-Length: 0" --stderr - --silent) && \ -export EH_CONNECTION_STRING=$(echo $EH_KEYS | grep -oP '(?<="primaryConnectionString":")[^"]*') +# Deploy service +kubectl --namespace backend apply -f $K8S/ingestion-0.yaml -export STORAGE_ACCOUNT_ACCESS_KEY=[YOUR_STORAGE_ACCOUNT_ACCESS_KEY_HERE] -export STORAGE_ACCOUNT_CONNECTION_STRING="[YOUR_STORAGE_ACCOUNT_CONNECTION_STRING_HERE]" +# Verify the pod is created +kubectl get pods -n backend +``` -# Create secrets -kubectl -n shipping create secret generic scheduler-secrets --from-literal=eventhub_name=${INGESTION_EH_NAME} \ ---from-literal=eventhub_sas_connection_string=${EH_CONNECTION_STRING} \ ---from-literal=storageaccount_name=${SCHEDULER_STORAGE_ACCOUNT_NAME} \ ---from-literal=storageaccount_key=${STORAGE_ACCOUNT_ACCESS_KEY} \ ---from-literal=queueconstring=${STORAGE_ACCOUNT_CONNECTION_STRING} +## Deploy DroneScheduler service -# Deploy service -kubectl --namespace shipping apply -f ./microservices-reference-implementation/k8s/scheduler.yaml +Extract resource details from deployment + +```bash +export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) ``` -## Deploy mock services +Build the dronescheduler services + +```bash +export DRONE_PATH=microservices-reference-implementation/src/shipping/dronescheduler +``` -Build the mock services +Create and set up pod identity ```bash -export MOCKS_PATH=microservices-reference-implementation/src/shipping/delivery -docker-compose -f $MOCKS_PATH/docker-compose.ci.build.yml up +# Extract outputs from deployment +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) + +# Deploy the identity resources +cat $K8S/dronescheduler-identity.yaml | \ + sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#ClientID: \"identityClientid\"#ClientID: $DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" > $K8S/dronescheduler-identity-0.yaml +kubectl apply -f $K8S/dronescheduler-identity-0.yaml ``` Build and publish the container image ```bash # Build the Docker image -docker build -t $ACR_SERVER/account:0.1.0 $MOCKS_PATH/MockAccountService/. && \ -docker build -t $ACR_SERVER/dronescheduler:0.1.0 $MOCKS_PATH/MockDroneScheduler/. && \ -docker build -t $ACR_SERVER/thirdparty:0.1.0 $MOCKS_PATH/MockThirdPartyService/. +docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ -# Push the image to ACR +# Push the images to ACR az acr login --name $ACR_NAME -docker push $ACR_SERVER/account:0.1.0 && \ -docker push $ACR_SERVER/dronescheduler:0.1.0 && \ -docker push $ACR_SERVER/thirdparty:0.1.0 +docker push $ACR_SERVER/dronescheduler:0.1.0 ``` -Deploy the mock services: +Deploy the dronescheduler services: ```bash # Update the image tag in the deployment YAML -sed -i "s#image:#image: $ACR_SERVER/account:0.1.0#g" ./microservices-reference-implementation/k8s/account.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -sed -i "s#image:#image: $ACR_SERVER/thirdparty:0.1.0#g" ./microservices-reference-implementation/k8s/thirdparty.yaml +cat $K8S/dronescheduler.yaml | \ + sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" | \ + sed "s#value: \"KeyVault_Name\"#value: $DRONESCHEDULER_KEYVAULT_URI#g" > $K8S/dronescheduler-0.yaml # Deploy the service -kubectl --namespace accounts apply -f ./microservices-reference-implementation/k8s/account.yaml && \ -kubectl --namespace dronemgmt apply -f ./microservices-reference-implementation/k8s/dronescheduler.yaml && \ -kubectl --namespace 3rdparty apply -f ./microservices-reference-implementation/k8s/thirdparty.yaml -``` +kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml ## Verify all services are running: +kubectl get pods -n backend +``` + +## Validate the application is running + +You can send delivery requests to the ingestion service using the Swagger UI. + +Get the public IP address of the Ingestion Service: ```bash -kubectl get all --all-namespaces -l co=fabrikam +export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") ``` +Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. + +```bash +open "http://$EXTERNAL_IP_ADDRESS/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +``` + +> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. + +## Optional steps + +Follow these steps to add logging and monitoring capabilities to the solution. + Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch ```bash @@ -321,7 +412,7 @@ kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kuber kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml ``` -Deploy Fluend. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd +Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd ```bash # The example elasticsearch yaml files deploy a service named "elasticsearch" @@ -336,37 +427,3 @@ sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PA sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml ``` - -#### Deploy linkerd - -For more information, see [https://linkerd.io/getting-started/k8s/](https://linkerd.io/getting-started/k8s/) - -> Note: -> the service mesh configuration linked above is defaulting the namespace to "default" for service discovery. -> Since Drone Delivery microservices are getting deployed into several custom namespaces, this config needs to be modified. This will consist of a small change in the dtab rules. - -Deploy linkerd defaulting the namespace to shipping instead: - -```bash -wget https://raw.githubusercontent.com/linkerd/linkerd-examples/master/k8s-daemonset/k8s/servicemesh.yml && \ -sed -i "s#/default#/shipping#g" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ -sed -i "149i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/account => /svc/account.accounts ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/dronescheduler => /svc/dronescheduler.dronemgmt ;" servicemesh.yml && \ -sed -i "176i \ \ \ \ \ \ \ \ /svc/thirdparty => /svc/thirdparty.3rdparty ;" servicemesh.yml && \ -kubectl apply -f servicemesh.yml -``` - -Deploy Prometheus and Grafana. For more information, see https://github.com/linkerd/linkerd-viz#kubernetes-deploy - -It is recommended to put an API Gateway in front of all APIs you want exposed to the public, -however for convenience, we exposed the Ingestion service with a public IP address. - -You can send delivery requests to the ingestion service using the swagger ui. - -```bash -export INGESTION_SERVICE_EXTERNAL_IP_ADDRESS=$(kubectl get --namespace shipping svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") -curl "http://${INGESTION_SERVICE_EXTERNAL_IP_ADDRESS}"/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST -``` From 3ca33fca06f81d5b466bdca858036f809f610702 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 12 Feb 2019 20:54:18 +0000 Subject: [PATCH 069/246] Merged PR 657: Enable Container Monitoring Enable Container monitoring using the CLI when deploying using ARM (cannot be done through ARM itself https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-onboard) Also Address warning when creating a SP for the principal and supplying a password. Instead let the CLI create the password and retrieve it. Related work items: #8708 --- deploymentARM.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/deploymentARM.md b/deploymentARM.md index 45955bd4..30996220 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -7,6 +7,7 @@ - Azure suscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) +- [JQ](https://stedolan.github.io/jq/download/) > Note: in linux systems, it is possible to run the docker command without prefacing > with sudo. For more information, please refer to [the Post-installation steps @@ -50,10 +51,10 @@ Infrastructure Prerequisites az login # Create a resource group and service principal for AKS -export SP_CLIENT_SECRET=$(uuidgen) - az group create --name $RESOURCE_GROUP --location $LOCATION && \ -export SP_APP_ID=$(az ad sp create-for-rbac --role="Contributor" -p $SP_CLIENT_SECRET -o tsv --query appId) && \ +export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ +export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ +export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) ``` @@ -87,6 +88,11 @@ export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy - export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) ``` +Enable Azure Monitoring for the AKS cluster +```bash +az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +``` + Download kubectl and create a k8s namespace ```bash # Install kubectl From 7ccab67703104fe2b3d586441eeb0a455bc67af7 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 12 Feb 2019 21:06:01 +0000 Subject: [PATCH 070/246] Merged PR 658: split arm template to create identities first split arm template to create identities first Related work items: #8708 --- azuredeploy-identities.json | 79 ++++++++++++++++++ azuredeploy.json | 155 ++++++++++++++++-------------------- deploymentARM.md | 46 +++++++---- 3 files changed, 175 insertions(+), 105 deletions(-) create mode 100644 azuredeploy-identities.json diff --git a/azuredeploy-identities.json b/azuredeploy-identities.json new file mode 100644 index 00000000..9e12ac7f --- /dev/null +++ b/azuredeploy-identities.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "variables": { + "deliveryIdName": "delivery", + "workflowIdName": "workflow", + "droneSchedulerIdName": "dronescheduler" + }, + "resources": [ + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('workflowIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('deliveryIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('droneSchedulerIdName')]", + "apiVersion": "2015-08-31-preview", + "location": "[resourceGroup().location]" + } + ], + "outputs": { + "deliveryIdName": { + "value": "[variables('deliveryIdName')]", + "type": "string" + }, + "deliveryPrincipalId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).principalId]", + "type": "string" + }, + "deliveryPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "type": "string" + }, + "deliveryPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).clientId]", + "type": "string" + }, + "droneSchedulerIdName": { + "value": "[variables('droneSchedulerIdName')]", + "type": "string" + }, + "droneSchedulerPrincipalId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).principalId]", + "type": "string" + }, + "droneSchedulerPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", + "type": "string" + }, + "droneSchedulerPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).clientId]", + "type": "string" + }, + "workflowIdName": { + "value": "[variables('workflowIdName')]", + "type": "string" + }, + "workflowPrincipalId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "type": "string" + }, + "workflowPrincipalResourceId": { + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "type": "string" + }, + "workflowPrincipalClientId": { + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).clientId]", + "type": "string" + } + } +} \ No newline at end of file diff --git a/azuredeploy.json b/azuredeploy.json index 6ae90ebf..4bda6453 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -2,6 +2,42 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { + "deliveryIdName": { + "metadata": { + "description": "Name of the delivery managed identity" + }, + "type": "string" + }, + "deliveryPrincipalId": { + "metadata": { + "description": "Principal id for the delivery managed identity" + }, + "type": "string" + }, + "droneSchedulerIdName": { + "metadata": { + "description": "Name of the drone scheduler managed identity" + }, + "type": "string" + }, + "droneSchedulerPrincipalId": { + "metadata": { + "description": "Principal id for the drone scheduler managed identity" + }, + "type": "string" + }, + "workflowIdName": { + "metadata": { + "description": "Name of the workflow managed identity" + }, + "type": "string" + }, + "workflowPrincipalId": { + "metadata": { + "description": "Principal id for the workflow managed identity" + }, + "type": "string" + }, "sshRSAPublicKey": { "type": "string", "metadata": { @@ -149,9 +185,6 @@ } }, "variables": { - "deliveryIdName": "delivery", - "workflowIdName": "workflow", - "droneSchedulerIdName": "dronescheduler", "aksClusterName": "[concat('aksCluster-', uniqueString(resourceGroup().id))]", "aksClusterDnsNamePrefix": "[concat('cluster-',uniqueString(resourceGroup().id))]", "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", @@ -175,24 +208,6 @@ "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]" }, "resources": [ - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('workflowIdName')]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('deliveryIdName')]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('droneSchedulerIdName')]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" - }, { "name": "[variables('aksClusterName')]", "type": "Microsoft.ContainerService/managedClusters", @@ -378,7 +393,6 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", "[variables('deliveryRedisStorageId')]" ], "properties": { @@ -389,8 +403,8 @@ "tenantId": "[subscription().tenantId]", "accessPolicies": [ { - "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('deliveryIdName'))).tenantId]", - "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('deliveryIdName'))).principalId]", + "tenantId": "[subscription().tenantId]", + "objectId": "[parameters('deliveryPrincipalId')]", "permissions": { "secrets": [ "get", @@ -478,9 +492,6 @@ "name": "[variables('droneSchedulerKeyVaultName')]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]" - ], "properties": { "sku": { "family": "A", @@ -489,8 +500,8 @@ "tenantId": "[subscription().tenantId]", "accessPolicies": [ { - "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('droneSchedulerIdName'))).tenantId]", - "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('droneSchedulerIdName'))).principalId]", + "tenantId": "[subscription().tenantId]", + "objectId": "[parameters('droneSchedulerPrincipalId')]", "permissions": { "secrets": [ "get", @@ -520,9 +531,6 @@ "name": "[variables('workflowKeyVaultName')]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]" - ], "properties": { "sku": { "family": "A", @@ -531,8 +539,8 @@ "tenantId": "[subscription().tenantId]", "accessPolicies": [ { - "tenantId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('workflowIdName'))).tenantId]", - "objectId": "[reference(concat('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('workflowIdName'))).principalId]", + "tenantId": "[subscription().tenantId]", + "objectId": "[parameters('workflowPrincipalId')]", "permissions": { "secrets": [ "get", @@ -586,13 +594,24 @@ "apiVersion": "2017-05-01", "properties": { "roleDefinitionId": "[variables('readerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).principalId]", + "principalId": "[parameters('deliveryPrincipalId')]", "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]" }, "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]" + ] + }, + { + "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", + "name": "[concat(variables('droneSchedulerKeyVaultName'), '/Microsoft.Authorization/', guid('dronescheduler-kv', resourceGroup().id))]", + "apiVersion": "2017-05-01", + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[parameters('droneSchedulerPrincipalId')]", + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]" ] }, { @@ -601,13 +620,11 @@ "apiVersion": "2017-05-01", "properties": { "roleDefinitionId": "[variables('contributorRoleId')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "principalId": "[parameters('workflowPrincipalId')]", "scope": "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" }, "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" ] }, { @@ -616,13 +633,11 @@ "apiVersion": "2017-05-01", "properties": { "roleDefinitionId": "[variables('readerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "principalId": "[parameters('workflowPrincipalId')]", "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" }, "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" ] }, { @@ -641,72 +656,36 @@ }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(variables('deliveryIdName'), '/Microsoft.Authorization/', guid('msi-delivery', resourceGroup().id))]", + "name": "[concat(parameters('deliveryIdName'), '/Microsoft.Authorization/', guid('msi-delivery', resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the delivery managed id", "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" - ] + } }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(variables('workflowIdName'), '/Microsoft.Authorization/', guid('msi-workflow', resourceGroup().id))]", + "name": "[concat(parameters('workflowIdName'), '/Microsoft.Authorization/', guid('msi-workflow', resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the workflow managed id", "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" - ] + } }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(variables('droneSchedulerIdName'), '/Microsoft.Authorization/', guid('msi-dronescheduler', resourceGroup().id))]", + "name": "[concat(parameters('droneSchedulerIdName'), '/Microsoft.Authorization/', guid('msi-dronescheduler', resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the drone scheduler managed id", "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))]" - ] + } } ], "outputs": { - "deliveryPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", - "type": "string" - }, - "deliveryPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).clientId]", - "type": "string" - }, - "droneSchedulerPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", - "type": "string" - }, - "droneSchedulerPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).clientId]", - "type": "string" - }, - "workflowPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", - "type": "string" - }, - "workflowPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).clientId]", - "type": "string" - }, "acrName": { "value": "[variables('acrName')]", "type": "string" diff --git a/deploymentARM.md b/deploymentARM.md index 30996220..7e80aef2 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -62,23 +62,35 @@ Deployment > Note: this deployment might take up to 20 minutes -* using Azure CLI 2.0 - - ```bash - az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ - --parameters servicePrincipalClientId=${SP_APP_ID} \ - servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ - servicePrincipalId=${SP_OBJECT_ID} \ - sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" - ``` - -* from Azure Portal - - [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fsummer-heart-0930.chufeiyun1688.workers.dev%3A443%2Fhttps%2Fraw.githubusercontent.com%2Fmspnp%2Fmicroservices-reference-implementation%2Fmaster%2Fazuredeploy.json) - > Note: - > 1. paste the $RESOURCE_GROUP value in the resource group field. Important: choose use existing resource group - > 2. paste the content of your ssh-rsa public key file in the Ssh RSA Plublic Key field. - > 3. paste the $SP_APP_ID, $SP_CLIENT_SECRET, and $SP_OBJECT_ID in the corresponding fields. +```bash +# Deploy the managed identities +# These are deployed first in a separate template to avoid propagation delays with AAD +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file azuredeploy-identities.json +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) + +# Wait for AAD propagation +until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done + +# Deploy all other resources +az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ +--parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + servicePrincipalId=${SP_OBJECT_ID} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ + deliveryIdName=${DELIVERY_ID_NAME} \ + deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ + droneSchedulerIdName=${DELIVERY_ID_NAME} \ + droneSchedulerPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ + workflowIdName=${DELIVERY_ID_NAME} \ + workflowPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} +``` Get outputs from Azure Deploy ```bash From dc2e8c7ce09661dc31b6977e914b8df3d3b7cffe Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Thu, 14 Feb 2019 17:27:31 +0000 Subject: [PATCH 071/246] Merged PR 660: add quotes for numeric env variable values in workflow yaml Unquoted numeric values are rejected in the yaml (recent change?) --- k8s/workflow.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 9bbe08b0..02d2e56d 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -51,19 +51,19 @@ spec: - name: SERVICE_URI_PACKAGE value: http://package/api/packages/ - name: SERVICEREQUEST__MAXRETRIES - value: 3 + value: "3" - name: SERVICEREQUEST__CIRCUITBREAKERTHRESHOLD - value: 0.5 + value: "0.5" - name: SERVICEREQUEST__CIRCUITBREAKERSAMPLINGPERIODSECONDS - value: 5 + value: "5" - name: SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT - value: 20 + value: "20" - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATION - value: 30 + value: "30" - name: SERVICEREQUEST__MAXBULKHEADSIZE - value: 100 + value: "100" - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE - value: 25 + value: "25" - name: no_proxy value: 169.254.169.254 volumes: From 4b663da246c5e02d7487761a6a714ff21cda4e31 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 15 Feb 2019 17:24:07 +0000 Subject: [PATCH 072/246] Merged PR 662: use the correct variables for parameters use the correct variables for parameters this caused random deployment failures, and it defeated the purpose of using separate identities. --- deploymentARM.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploymentARM.md b/deploymentARM.md index 7e80aef2..84b83299 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -86,10 +86,10 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ deliveryIdName=${DELIVERY_ID_NAME} \ deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ - droneSchedulerIdName=${DELIVERY_ID_NAME} \ - droneSchedulerPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ - workflowIdName=${DELIVERY_ID_NAME} \ - workflowPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} + droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ + droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ + workflowIdName=${WORKFLOW_ID_NAME} \ + workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} ``` Get outputs from Azure Deploy From 9c51fe0bb6903297f0b1742832c05f8e939c9697 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 15 Feb 2019 17:25:32 +0000 Subject: [PATCH 073/246] Merged PR 663: add ingress for ingestion add ingress for ingestion update instructions to install the ingress controller and retrieve the external IP Related work items: #8824 --- deployment.md | 6 +++++- deploymentARM.md | 6 +++++- k8s/ingestion.yaml | 11 ++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/deployment.md b/deployment.md index befb1a0e..559821c2 100644 --- a/deployment.md +++ b/deployment.md @@ -404,6 +404,10 @@ docker push $ACR_SERVER/ingestion:0.1.0 Deploy the Ingestion service ```bash +# Deploy the ngnix ingress controller +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml + # Update deployment YAML with image tage sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml @@ -497,7 +501,7 @@ You can send delivery requests to the ingestion service using the Swagger UI. Get the public IP address of the Ingestion Service: ```bash -export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend ingress/ingestion-ingress -o jsonpath="{.status.loadBalancer.ingress[0].ip}") ``` Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. diff --git a/deploymentARM.md b/deploymentARM.md index 84b83299..c011c8f2 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -329,6 +329,10 @@ docker push $ACR_SERVER/ingestion:0.1.0 Deploy the Ingestion service ```bash +# Deploy the ngnix ingress controller +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml + # Update deployment YAML with image tage sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml @@ -407,7 +411,7 @@ You can send delivery requests to the ingestion service using the Swagger UI. Get the public IP address of the Ingestion Service: ```bash -export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend svc ingestion -o jsonpath="{.status.loadBalancer.ingress[0].*}") +export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend ingress/ingestion-ingress -o jsonpath="{.status.loadBalancer.ingress[0].ip}") ``` Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index bbe00441..c434e42b 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -21,7 +21,16 @@ spec: selector: app: ingestion type: - LoadBalancer + ClusterIP +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingestion-ingress +spec: + backend: + serviceName: ingestion + servicePort: 80 --- apiVersion: extensions/v1beta1 kind: Deployment From 0e1b969ef1e18a81f6ae243d2a276399126b6c67 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 20 Feb 2019 13:00:12 +0000 Subject: [PATCH 074/246] Merged PR 652: some minor improvements for ingestion some minor improvements for ingestion - use maven official Docker images from Ingestion - remove hand-coded mvn docker image - align deployment instructions --- deployment.md | 4 --- src/shipping/ingestion/Dockerfile | 22 ++++++++++++--- src/shipping/ingestion/Dockerfilemaven | 25 ----------------- src/shipping/ingestion/pom.xml | 37 +++++++++----------------- 4 files changed, 31 insertions(+), 57 deletions(-) delete mode 100644 src/shipping/ingestion/Dockerfilemaven diff --git a/deployment.md b/deployment.md index 559821c2..e5f43756 100644 --- a/deployment.md +++ b/deployment.md @@ -389,10 +389,6 @@ Build the Ingestion service ```bash export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH -docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk - # Build the docker image docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH diff --git a/src/shipping/ingestion/Dockerfile b/src/shipping/ingestion/Dockerfile index 7e33186b..762bfcd2 100644 --- a/src/shipping/ingestion/Dockerfile +++ b/src/shipping/ingestion/Dockerfile @@ -1,6 +1,20 @@ -FROM openjdk:8-jre-alpine +FROM openjdk:8-jre-alpine as base +WORKDIR /usr/src/app +EXPOSE 80 -ENTRYPOINT ["/usr/bin/java", "-jar", "/usr/share/dronedelivery/Ingestion-1.0.0.jar"] +FROM maven:3.5-jdk-8-slim as build +WORKDIR /usr/src/app -# Add Maven dependencies (not shaded into the artifact; Docker-cached) -COPY target /usr/share/dronedelivery +COPY pom.xml ./ +COPY settings-docker.xml /usr/share/maven/ref/ +RUN /usr/local/bin/mvn-entrypoint.sh \ + mvn package -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true --fail-never +COPY . . +RUN mvn package -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true + +FROM base as final + +WORKDIR /app +COPY --from=build /usr/src/app/target ./ + +ENTRYPOINT ["java","-jar","ingestion-0.1.0.jar"] diff --git a/src/shipping/ingestion/Dockerfilemaven b/src/shipping/ingestion/Dockerfilemaven deleted file mode 100644 index 6fca0b81..00000000 --- a/src/shipping/ingestion/Dockerfilemaven +++ /dev/null @@ -1,25 +0,0 @@ -FROM openjdk:8-jdk - -ARG MAVEN_VERSION=3.5.4 -ARG USER_HOME_DIR="/root" -ARG SHA=ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries - -RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ - && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && echo "${SHA} /tmp/apache-maven.tar.gz" | sha256sum -c - \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn - -ENV MAVEN_HOME /usr/share/maven -ENV MAVEN_CONFIG "$USER_HOME_DIR/.m2" -ENV SOURCEPATH /sln - -COPY mvn-entrypoint.sh /usr/local/bin/mvn-entrypoint.sh -COPY settings-docker.xml /usr/share/maven/ref/ -RUN chmod +x /usr/local/bin/mvn-entrypoint.sh -VOLUME "$USER_HOME_DIR/.m2" - -ENTRYPOINT ["/usr/local/bin/mvn-entrypoint.sh"] -CMD ["mvn","package"] diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index b49387e9..82ee20ae 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -3,13 +3,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - DroneDelivery - Ingestion - 1.0.0 + com.fabrikam.dronedelivery + ingestion + 0.1.0 jar Ingestion - Ingestion Service + Fabrikam Drone Delivery Ingestion Service org.springframework.boot @@ -44,13 +44,13 @@ spring-boot-starter-test test - + com.google.common.html.types types 1.0.5 - + org.mockito @@ -58,7 +58,7 @@ 2.13.0 test - + org.powermock powermock-module-junit4 @@ -73,7 +73,7 @@ - + junit junit @@ -94,27 +94,18 @@ openejb-core 4.7.4 - + org.springframework.boot spring-boot-starter-actuator - + io.springfox springfox-swagger2 2.7.0 - - - - - org.springframework.boot - spring-boot-starter-test - 1.5.9.RELEASE - test - - + io.springfox springfox-swagger-ui @@ -153,7 +144,7 @@ org.springframework.boot spring-boot-maven-plugin - + org.apache.maven.plugins maven-failsafe-plugin @@ -172,7 +163,7 @@ - + org.apache.maven.plugins maven-dependency-plugin @@ -192,6 +183,4 @@ - - From 1a06e65168f731b8d155db7832e0e531cc6295df Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 20 Feb 2019 19:22:39 +0000 Subject: [PATCH 075/246] Merged PR 665: fix incorrect config name fix incorrect config name (missing seconds suffix) --- k8s/workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml index 02d2e56d..99be3705 100644 --- a/k8s/workflow.yaml +++ b/k8s/workflow.yaml @@ -58,7 +58,7 @@ spec: value: "5" - name: SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT value: "20" - - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATION + - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATIONSECONDS value: "30" - name: SERVICEREQUEST__MAXBULKHEADSIZE value: "100" From 8ff1af8ec84f3e5c61a024022a9501baefba996c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 25 Feb 2019 19:06:23 +0000 Subject: [PATCH 076/246] Merged PR 656: add Kubernetes AppInsights for Workflow add Kubernetes AppInsights for Workflow - code light up for distributed tracing in a NET Core app resolved: #8592 Related work items: #8592 --- .../Fabrikam.Workflow.Service/Program.cs | 24 +++- .../ServiceStartup.cs | 12 +- .../TracingExtensions.cs | 124 ++++++++++++++++++ 3 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs index cc77ca44..f53163ff 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -5,7 +5,9 @@ using System.IO; using System.Threading.Tasks; +using Microsoft.ApplicationInsights; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; using Serilog.Formatting.Compact; @@ -16,7 +18,25 @@ class Program { static async Task Main(string[] args) { - await CreateHostBuilder(args).Build().RunAsync(); + var host = CreateHostBuilder(args).Build(); + + TelemetryClient telemetryClient = null; + try + { + telemetryClient = host.Services.GetService(); + + telemetryClient.TrackTrace("Fabrikan Workflow Service is starting."); + + await host.RunAsync(); + } + finally + { + // before exit, flush the remaining data + telemetryClient?.Flush(); + + // flush is not blocking so wait a bit + Task.Delay(5000).Wait(); + } } private static IHostBuilder CreateHostBuilder(string[] args) @@ -47,4 +67,4 @@ private static IHostBuilder CreateHostBuilder(string[] args) .UseConsoleLifetime(); } } -} +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index cb36dbfc..a761ff8a 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -4,25 +4,21 @@ // ------------------------------------------------------------ using System; -using Fabrikam.Workflow.Service.RequestProcessing; -using Fabrikam.Workflow.Service.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Fabrikam.Workflow.Service.RequestProcessing; +using Fabrikam.Workflow.Service.Services; namespace Fabrikam.Workflow.Service { public static class ServiceStartup { - private const string AppInsightsInstrumentationKey = "ApplicationInsights-InstrumentationKey"; - public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { services.AddOptions(); // Configure AppInsights - services.AddApplicationInsightsKubernetesEnricher(); - services.AddApplicationInsightsTelemetry( - context.Configuration[AppInsightsInstrumentationKey]); + services.AddApplicationInsightsTelemetry(context.Configuration); services.Configure(context.Configuration); services.AddHostedService(); @@ -51,4 +47,4 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect .AddResiliencyPolicies(context.Configuration); } } -} +} \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs new file mode 100644 index 00000000..0fc472f0 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Linq; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.AspNetCore.TelemetryInitializers; +using Microsoft.ApplicationInsights.DependencyCollector; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation.ApplicationId; +using Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace Fabrikam.Workflow.Service +{ + /// + /// Application Insights setup class based on https://docs.microsoft.com/en-us/azure/azure-monitor/app/console + /// + /// + /// Telemetry Modules initialization as expected based on https://github.com/Microsoft/ApplicationInsights-aspnetcore/blob/04b5485d4a8aa498b2d99c60bdf8ca59bc9103fc/src/Microsoft.ApplicationInsights.AspNetCore/Implementation/TelemetryConfigurationOptions.cs#L27 + /// + internal static class TracingExtensions + { + private const string CustomKeyVaultAIppInsightsIKey = "ApplicationInsights-InstrumentationKey"; + + public static IServiceCollection AddApplicationInsightsTelemetry( + this IServiceCollection services, + IConfiguration configuration) + { + services.AddSingleton(s => + { + var telemetryConfig = TelemetryConfiguration.CreateDefault(); + + var config = s.GetService(); + var instrumentationKey = config[CustomKeyVaultAIppInsightsIKey]; + + if (!string.IsNullOrWhiteSpace(instrumentationKey)) + { + telemetryConfig.InstrumentationKey = instrumentationKey; + } + + // use processors + telemetryConfig + .DefaultTelemetrySink + .TelemetryProcessorChainBuilder + .Use((next) => + { + var processor = new QuickPulseTelemetryProcessor(next); + + var quickPulseModule = s.GetServices() + .OfType() + .Single(); + quickPulseModule.RegisterTelemetryProcessor(processor); + + return processor; + }); + + telemetryConfig + .DefaultTelemetrySink + .TelemetryProcessorChainBuilder + .Build(); + + // add initializers: https://github.com/Microsoft/ApplicationInsights-aspnetcore/pull/672 + telemetryConfig.TelemetryInitializers.Add( + new DomainNameRoleInstanceTelemetryInitializer()); + telemetryConfig.TelemetryInitializers.Add( + new HttpDependenciesParsingTelemetryInitializer()); + telemetryConfig.AddApplicationInsightsKubernetesEnricher( + applyOptions: null); + + // initialize all modules + foreach (var module in s.GetServices()) + { + module.Initialize(telemetryConfig); + } + + // other config: https://github.com/Microsoft/ApplicationInsights-aspnetcore/blob/de1af6235a4cc365d64cbc78db9bdd2d579a37ee/src/Microsoft.ApplicationInsights.AspNetCore/Implementation/TelemetryConfigurationOptionsSetup.cs#L129 + telemetryConfig.ApplicationIdProvider = + s.GetRequiredService(); + + return telemetryConfig; + }); + + services.AddSingleton(s => + { + var module = new DependencyTrackingTelemetryModule(); + + var excludedDomains = module.ExcludeComponentCorrelationHttpHeadersOnDomains; + excludedDomains.Add("core.windows.net"); + excludedDomains.Add("core.chinacloudapi.cn"); + excludedDomains.Add("core.cloudapi.de"); + excludedDomains.Add("core.usgovcloudapi.net"); + + if (module.EnableLegacyCorrelationHeadersInjection) + { + excludedDomains.Add("localhost"); + excludedDomains.Add("127.0.0.1"); + } + + var includedActivities = module.IncludeDiagnosticSourceActivities; + // TODO: in workflow scenario EventHubs activities may not be required. + includedActivities.Add("Microsoft.Azure.EventHubs"); + includedActivities.Add("Microsoft.Azure.ServiceBus"); + + return module; + }); + + services.AddSingleton< + ITelemetryModule, + QuickPulseTelemetryModule>(); + + services.TryAddSingleton< + IApplicationIdProvider, + ApplicationInsightsApplicationIdProvider>(); + + services.TryAddSingleton(); + + return services; + } + } +} \ No newline at end of file From f11757d4840133b8268cc071c45b443924af8f51 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 25 Feb 2019 20:08:34 +0000 Subject: [PATCH 077/246] Merged PR 653: upgrade ingestion to spring boot 2.1.2 upgrade mvn reference Related work items: #8250 --- k8s/ingestion.yaml | 2 +- src/shipping/ingestion/pom.xml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index c434e42b..be7aba34 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -60,7 +60,7 @@ spec: containerPort: 80 livenessProbe: httpGet: - path: /health + path: /actuator/health port: 80 initialDelaySeconds: 30 periodSeconds: 20 diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 82ee20ae..5a601cce 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -14,8 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 1.5.2.RELEASE - + 2.1.2.RELEASE @@ -24,7 +23,7 @@ 1.8 1.16.10 2.7.5 - 1.4.1.RELEASE + 2.1.2.RELEASE 1.1.3 1.7.12 From 57ee256b93d74e0f11eea56b65b4caa92b778cb5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 1 Mar 2019 14:48:36 +0000 Subject: [PATCH 078/246] Merged PR 670: add helm dependency add helm dependency partially cherry-picked from 9e48dc04d2bc02dacfef379a8d9bdd8f9b355b07: - [advance] Merged PR 668: Add deployment of app gateway in front of private ingress related work items: #8943 and others Related work items: #8943 --- deployment.md | 8 ++++++++ deploymentARM.md | 8 ++++++++ k8s/tiller-rbac.yaml | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 k8s/tiller-rbac.yaml diff --git a/deployment.md b/deployment.md index e5f43756..85617056 100644 --- a/deployment.md +++ b/deployment.md @@ -5,6 +5,7 @@ - Azure subscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) +- [Helm](https://docs.helm.sh/using_helm/#installing-helm) > Note: in linux systems, it is possible to run the docker command without prefacing > with sudo. For more information, please refer to [the Post-installation steps @@ -63,6 +64,13 @@ kubectl create namespace backend && \ kubectl create namespace frontend ``` +Setup Helm in the container + +```bash +kubectl apply -f $k8s/tiller-rbac.yaml +helm init --service-account tiller +``` + Create an Azure Container Registry instance. > Note: Azure Container Registory is not required. If you prefer, you can store the Docker images for this solution in another container registry. diff --git a/deploymentARM.md b/deploymentARM.md index c011c8f2..84fa205b 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -7,6 +7,7 @@ - Azure suscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) +- [Helm](https://docs.helm.sh/using_helm/#installing-helm) - [JQ](https://stedolan.github.io/jq/download/) > Note: in linux systems, it is possible to run the docker command without prefacing @@ -119,6 +120,13 @@ kubectl create namespace frontend ``` +Setup Helm in the container + +```bash +kubectl apply -f $K8S/tiller-rbac.yaml +helm init --service-account tiller +``` + Integrate Application Insights instance ```bash diff --git a/k8s/tiller-rbac.yaml b/k8s/tiller-rbac.yaml new file mode 100644 index 00000000..1fcf47dc --- /dev/null +++ b/k8s/tiller-rbac.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tiller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tiller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: tiller + namespace: kube-system From 1cd6feb5e8403ee42a051f0519fae20009f99c79 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 1 Mar 2019 20:06:21 +0000 Subject: [PATCH 079/246] Merged PR 671: change deployment to use helm for delivery for delivery change deployment to use helm for delivery for delivery solved: #8943 Related work items: #8943 --- .../delivery/templates/delivery-deploy.yaml | 50 +++++++------ .../templates}/delivery-identity.yaml | 14 +++- .../delivery-service-experimental.yaml | 31 ++++++++ .../templates/delivery-service-sxs.yaml | 28 +++++++ .../delivery/templates/delivery-service.yaml | 53 +++++++------ charts/delivery/values.yaml | 12 ++- deployment.md | 27 ++++--- deploymentARM.md | 35 ++++----- k8s/delivery.yaml | 75 ------------------- 9 files changed, 167 insertions(+), 158 deletions(-) rename {k8s => charts/delivery/templates}/delivery-identity.yaml (72%) create mode 100644 charts/delivery/templates/delivery-service-experimental.yaml create mode 100644 charts/delivery/templates/delivery-service-sxs.yaml delete mode 100644 k8s/delivery.yaml diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index edbaee6a..8b0ec1b9 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -1,3 +1,11 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery +################################################################################################### apiVersion: apps/v1beta2 kind: Deployment metadata: @@ -10,6 +18,7 @@ metadata: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "delivery.chart" . }} + aadpodidbinding: delivery annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: @@ -28,31 +37,30 @@ spec: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "delivery.chart" . }} + aadpodidbinding: delivery spec: + securityContext: + fsGroup: 1 containers: - - name: deliveryservice + - name: fabrikam-delivery image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOCDB_KEY - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf - key: CosmosDB_Key - - name: DOCDB_ENDPOINT - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf - key: CosmosDB_Endpoint - name: DOCDB_DATABASEID - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf - key: CosmosDB_DatabaseId + value: {{ .Values.cosmosdb.id }} - name: DOCDB_COLLECTIONID - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-dd-deliveryservice-cosmosdbconf - key: CosmosDB_CollectionId - ports: + value: {{ .Values.cosmosdb.collectionid }} + - name: KEY_VAULT_URI + value: {{ .Values.keyvault.uri }} + - name: no_proxy + value: 169.254.169.254 + ports: - name: service - containerPort: 80 + containerPort: 8080 + resources: + requests: + cpu: 1 + memory: 2G + limits: + cpu: 1 + memory: 3G diff --git a/k8s/delivery-identity.yaml b/charts/delivery/templates/delivery-identity.yaml similarity index 72% rename from k8s/delivery-identity.yaml rename to charts/delivery/templates/delivery-identity.yaml index f9ded1e7..f9141fa2 100644 --- a/k8s/delivery-identity.yaml +++ b/charts/delivery/templates/delivery-identity.yaml @@ -4,21 +4,27 @@ # ------------------------------------------------------------ ################################################################################################### -# Delivery service identityß +# Delivery service identity ################################################################################################### apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: name: delivery-identity + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "0" spec: type: 0 - ResourceID: "identityResourceId" - ClientID: "identityClientid" + ResourceID: {{ .Values.identity.resourceid }} + ClientID: {{ .Values.identity.clientid }} --- apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: name: delivery-identity-binding + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "1" spec: AzureIdentity: delivery-identity - Selector: delivery \ No newline at end of file + Selector: delivery diff --git a/charts/delivery/templates/delivery-service-experimental.yaml b/charts/delivery/templates/delivery-service-experimental.yaml new file mode 100644 index 00000000..b4b4ca3f --- /dev/null +++ b/charts/delivery/templates/delivery-service-experimental.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service experimental +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "delivery-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: delivery-experimental + labels: + app.kubernetes.io/name: delivery + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/managed-by: azurepipelines + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: delivery + app.kubernetes.io/instance: delivery-v0.1.0 +{{ end }} diff --git a/charts/delivery/templates/delivery-service-sxs.yaml b/charts/delivery/templates/delivery-service-sxs.yaml new file mode 100644 index 00000000..3d78c796 --- /dev/null +++ b/charts/delivery/templates/delivery-service-sxs.yaml @@ -0,0 +1,28 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service sxs +################################################################################################### +apiVersion: v1 +kind: Service +metadata: + name: {{ include "delivery.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index b6d704ec..5f5d5c4d 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -1,21 +1,32 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "delivery.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "delivery.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - clusterIP: None +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "delivery-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: delivery + labels: + app: delivery + app.kubernetes.io/component: backend + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/managed-by: azurepipelines + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: delivery + app.kubernetes.io/instance: delivery-v0.1.0 +{{ end }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index d6d3dabe..e177b734 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -1,10 +1,16 @@ # Default values for dronedelivery. replicaCount: 1 +identity: + clientid: + resourceid: +dockerregistry: image: repository: tag: pullPolicy: IfNotPresent -service: - type: ClusterIP -dockerregistry: +cosmosdb: + id: + collectionid: +keyvault: + uri: reason: unknown diff --git a/deployment.md b/deployment.md index 85617056..fdcf02b0 100644 --- a/deployment.md +++ b/deployment.md @@ -36,6 +36,7 @@ export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) export K8S=./microservices-reference-implementation/k8s +HELM_CHARTS=./microservices-reference-implementation/charts ``` Provision a Kubernetes cluster in AKS @@ -215,28 +216,26 @@ az keyvault set-policy --name $DELIVERY_KEYVAULT_NAME --secret-permissions get l # Allow the cluster to manage the identity to assign to pods az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DELIVERY_PRINCIPAL_RESOURCE_ID - -# Deploy the identity resources -cat $K8S/delivery-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $DELIVERY_PRINCIPAL_CLIENT_ID#g" > $K8S/delivery-identity-0.yaml -kubectl apply -f $K8S/delivery-identity-0.yaml ``` Deploy the Delivery service: ```bash -# Update the image tag and config values in the deployment YAML -sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ - sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ - sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ - sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" > $K8S/delivery-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/delivery-0.yaml +helm install $HELM_CHARTS/delivery/ \ + --set image.tag=0.1.0 \ + --set image.repository=delivery \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DELIVERY_PRINCIPAL_ID \ + --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ + --set cosmosdb.id=$DATABASE_NAME \ + --set cosmosdb.collectionid=$COLLECTION_NAME \ + --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ + --namespace backend \ + --name delivery-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status delivery-v0.1.0 ``` ## Deploy the Package service diff --git a/deploymentARM.md b/deploymentARM.md index 84fa205b..1d3c7dc3 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -43,6 +43,7 @@ export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) export K8S=./microservices-reference-implementation/k8s +HELM_CHARTS=./microservices-reference-implementation/charts ``` Infrastructure Prerequisites @@ -182,34 +183,28 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/delivery:0.1.0 ``` -Set up pod identity +Deploy the Delivery service: ```bash -# Extract outputs from deployment +# Extract pod identity outputs from deployment export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalClientId.value -o tsv) -# Deploy the identity resources -cat $K8S/delivery-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $DELIVERY_PRINCIPAL_CLIENT_ID#g" > $K8S/delivery-identity-0.yaml -kubectl apply -f $K8S/delivery-identity-0.yaml -``` - -Deploy the Delivery service: - -```bash -# Update the image tag and config values in the deployment YAML -sed "s#image:#image: $ACR_SERVER/delivery:0.1.0#g" $K8S/delivery.yaml | \ - sed "s/value: \"CosmosDB_DatabaseId\"/value: $DATABASE_NAME/g" | \ - sed "s/value: \"CosmosDB_CollectionId\"/value: $COLLECTION_NAME/g" | \ - sed "s#value: \"KeyVault_Name\"#value: $DELIVERY_KEYVAULT_URI#g" > $K8S/delivery-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/delivery-0.yaml +helm install $HELM_CHARTS/delivery/ \ + --set image.tag=0.1.0 \ + --set image.repository=delivery \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DELIVERY_PRINCIPAL_ID \ + --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ + --set cosmosdb.id=$DATABASE_NAME \ + --set cosmosdb.collectionid=$COLLECTION_NAME \ + --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ + --namespace backend \ + --name delivery-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status delivery-v0.1.0 ``` ## Deploy the Package service diff --git a/k8s/delivery.yaml b/k8s/delivery.yaml deleted file mode 100644 index 3f2535a8..00000000 --- a/k8s/delivery.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Delivery service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: delivery - labels: - app: delivery - bc: shipping - co: fabrikam -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app: delivery ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: delivery - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: delivery - version: 0.1.0 - bc: shipping - co: fabrikam - aadpodidbinding: delivery -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: delivery - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: delivery - version: 0.1.0 - bc: shipping - co: fabrikam - aadpodidbinding: delivery - annotations: - team: deliveryservice - spec: - securityContext: - fsGroup: 1 - containers: - - name: delivery - image: - env: - # Export K8s secrets - - name: DOCDB_DATABASEID - value: "CosmosDB_DatabaseId" - - name: DOCDB_COLLECTIONID - value: "CosmosDB_CollectionId" - - name: KEY_VAULT_URI - value: "KeyVault_Name" - - name: no_proxy - value: 169.254.169.254 - ports: - - name: service - containerPort: 8080 - resources: - requests: - cpu: 1 - memory: 2G - limits: - cpu: 1 - memory: 3G From 4ca6f971c8e6342641220d8b4998eed3418ba4d2 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Wed, 6 Mar 2019 17:10:54 +0000 Subject: [PATCH 080/246] Merged PR 669: add tls for ingestion add tls for ingestion CLI and ARM versions of the basic RA Additional changes: * use a variable for the root folder, so any changes are done in a single place * fixes wrong reference to deployment when extracting identity outputs * fixes wrong secret name in workflow deployment. Related work items: #8876, #8944 --- azuredeploy.json | 2 +- deployment.md | 56 +++++++++++++++++++++------------- deploymentARM.md | 75 +++++++++++++++++++++++++--------------------- k8s/ingestion.yaml | 15 ++++++++-- 4 files changed, 89 insertions(+), 59 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 4bda6453..bc1a6ed3 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -576,7 +576,7 @@ }, { "type": "secrets", - "name": "ApplicationInsights--InstrumentationKey", + "name": "ApplicationInsights-InstrumentationKey", "apiVersion": "2015-06-01", "properties": { "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" diff --git a/deployment.md b/deployment.md index fdcf02b0..fce9f200 100644 --- a/deployment.md +++ b/deployment.md @@ -35,8 +35,9 @@ export AI_NAME="${UNIQUE_APP_NAME_PREFIX}-appinsights" export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) -export K8S=./microservices-reference-implementation/k8s -HELM_CHARTS=./microservices-reference-implementation/charts +export PROJECT_ROOT=./microservices-reference-implementation +export K8S=$PROJECT_ROOT/k8s +export HELM_CHARTS=$PROJECT_ROOT/charts ``` Provision a Kubernetes cluster in AKS @@ -68,7 +69,7 @@ kubectl create namespace frontend Setup Helm in the container ```bash -kubectl apply -f $k8s/tiller-rbac.yaml +kubectl apply -f $K8S/tiller-rbac.yaml helm init --service-account tiller ``` @@ -162,7 +163,7 @@ az cosmosdb create \ Build the Delivery service ```bash -export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery +export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery ``` Build and publish the container image @@ -250,7 +251,7 @@ az cosmosdb create --name $COSMOSDB_NAME --kind MongoDB --resource-group $RESOUR Build the Package service ```bash -export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package +export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package # Build the docker image docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH @@ -305,7 +306,7 @@ export INGESTION_QUEUE_NAMESPACE_ENDPOINT=$(az servicebus namespace show --resou Build the workflow service ```bash -export WORKFLOW_PATH=./microservices-reference-implementation/src/shipping/workflow +export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow # Build the Docker image docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. @@ -394,7 +395,7 @@ export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule k Build the Ingestion service ```bash -export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion +export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion # Build the docker image docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH @@ -408,11 +409,30 @@ Deploy the Ingestion service ```bash # Deploy the ngnix ingress controller -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml - -# Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml +helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true + +# Obtain the load balancer ip address and assign a domain name +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) +export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" +export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) + +# Create a self-signed certificate for TLS +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls.crt \ + -keyout ingestion-ingress-tls.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" + +kubectl create secret tls ingestion-ingress-tls \ + --namespace backend \ + --key ingestion-ingress-tls.key \ + --cert ingestion-ingress-tls.crt + +# Update deployment YAML with image tag and the fqdn +cat $K8S/ingestion.yaml | \ + sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" | \ + sed "s#ingestion-host-name#$EXTERNAL_INGEST_FQDN#g" \ + > $K8S/ingestion-0.yaml # Create secret kubectl -n backend create secret generic ingestion-secrets \ @@ -433,7 +453,7 @@ kubectl get pods -n backend Build the dronescheduler services ```bash -export DRONE_PATH=microservices-reference-implementation/src/shipping/dronescheduler +export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler ``` Create KeyVault and secrets @@ -501,16 +521,10 @@ kubectl get pods -n backend You can send delivery requests to the ingestion service using the Swagger UI. -Get the public IP address of the Ingestion Service: - -```bash -export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend ingress/ingestion-ingress -o jsonpath="{.status.loadBalancer.ingress[0].ip}") -``` - -Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. +Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. ```bash -open "http://$EXTERNAL_IP_ADDRESS/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" ``` > We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. diff --git a/deploymentARM.md b/deploymentARM.md index 1d3c7dc3..98c61a0d 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -1,7 +1,5 @@ # Deploying the Reference Implementation - - ## Prerequisites - Azure suscription @@ -20,7 +18,7 @@ Clone or download this repo locally. git clone https://github.com/mspnp/microservices-reference-implementation.git ``` -> The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. +The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. ## Generate a SSH rsa public/private key pair @@ -42,8 +40,9 @@ export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) -export K8S=./microservices-reference-implementation/k8s -HELM_CHARTS=./microservices-reference-implementation/charts +export PROJECT_ROOT=./microservices-reference-implementation +export K8S=$PROJECT_ROOT/k8s +export HELM_CHARTS=$PROJECT_ROOT/charts ``` Infrastructure Prerequisites @@ -120,7 +119,6 @@ kubectl create namespace backend && \ kubectl create namespace frontend ``` - Setup Helm in the container ```bash @@ -169,7 +167,7 @@ export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n az Build the Delivery service ```bash -export DELIVERY_PATH=./microservices-reference-implementation/src/shipping/delivery +export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery ``` Build and publish the container image @@ -187,8 +185,8 @@ Deploy the Delivery service: ```bash # Extract pod identity outputs from deployment -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) # Deploy the service helm install $HELM_CHARTS/delivery/ \ @@ -218,7 +216,7 @@ export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeplo Build the Package service ```bash -export PACKAGE_PATH=microservices-reference-implementation/src/shipping/package +export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package # Build the docker image docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH @@ -260,7 +258,7 @@ export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n a Build the workflow service ```bash -export WORKFLOW_PATH=./microservices-reference-implementation/src/shipping/workflow +export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow # Build the Docker image docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. @@ -274,8 +272,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowPrincipalClientId.value -o tsv) +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) # Deploy the identity resources cat $K8S/workflow-identity.yaml | \ @@ -315,11 +313,7 @@ export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule k Build the Ingestion service ```bash -export INGESTION_PATH=./microservices-reference-implementation/src/shipping/ingestion - -# Build the app -docker build -t openjdk_and_mvn-build:8-jdk -f $INGESTION_PATH/Dockerfilemaven $INGESTION_PATH -docker run -it --rm -v $( cd "${INGESTION_PATH}" && pwd )/:/sln openjdk_and_mvn-build:8-jdk +export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion # Build the docker image docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH @@ -333,11 +327,30 @@ Deploy the Ingestion service ```bash # Deploy the ngnix ingress controller -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml - -# Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" $K8S/ingestion.yaml > $K8S/ingestion-0.yaml +helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true + +# Obtain the load balancer ip address and assign a domain name +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) +export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" +export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) + +# Create a self-signed certificate for TLS +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls.crt \ + -keyout ingestion-ingress-tls.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" + +kubectl create secret tls ingestion-ingress-tls \ + --namespace backend \ + --key ingestion-ingress-tls.key \ + --cert ingestion-ingress-tls.crt + +# Update deployment YAML with image tag and the fqdn +cat $K8S/ingestion.yaml | \ + sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" | \ + sed "s#ingestion-host-name#$EXTERNAL_INGEST_FQDN#g" \ + > $K8S/ingestion-0.yaml # Create secret kubectl -n backend create secret generic ingestion-secrets \ @@ -364,15 +377,15 @@ export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP Build the dronescheduler services ```bash -export DRONE_PATH=microservices-reference-implementation/src/shipping/dronescheduler +export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler ``` Create and set up pod identity ```bash # Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) # Deploy the identity resources cat $K8S/dronescheduler-identity.yaml | \ @@ -411,16 +424,10 @@ kubectl get pods -n backend You can send delivery requests to the ingestion service using the Swagger UI. -Get the public IP address of the Ingestion Service: - -```bash -export EXTERNAL_IP_ADDRESS=$(kubectl get --namespace backend ingress/ingestion-ingress -o jsonpath="{.status.loadBalancer.ingress[0].ip}") -``` - -Use a web browser to navigate to `http://[EXTERNAL_IP_ADDRESS]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. +Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. ```bash -open "http://$EXTERNAL_IP_ADDRESS/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" ``` > We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index be7aba34..db285645 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -28,9 +28,18 @@ kind: Ingress metadata: name: ingestion-ingress spec: - backend: - serviceName: ingestion - servicePort: 80 + tls: + - hosts: + - ingestion-host-name + secretName: ingestion-ingress-tls + rules: + - host: ingestion-host-name + http: + paths: + - path: / + backend: + serviceName: ingestion + servicePort: 80 --- apiVersion: extensions/v1beta1 kind: Deployment From c77c345277f7ee971d3ab49bae7db93e9d831af2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 6 Mar 2019 18:05:39 +0000 Subject: [PATCH 081/246] Merged PR 667: add AppInsights to Ingestion distributed tracing ingestion implementation - add AI - fine tuning SB - remove service mesh dep resolved: #8591 Related work items: #8591 --- deployment.md | 9 +- deploymentARM.md | 3 +- k8s/ingestion.yaml | 10 +- src/shipping/ingestion/Dockerfile | 6 +- src/shipping/ingestion/pom.xml | 14 +- .../configuration/ApplicationProperties.java | 27 +-- .../controller/IngestionController.java | 21 +- .../ingestion/service/IngestionImpl.java | 65 +++---- .../ingestion/util/ClientPool.java | 6 +- .../ingestion/util/ClientPoolImpl.java | 43 +++-- .../util/InstrumentedQueueClient.java | 14 ++ .../util/InstrumentedQueueClientImpl.java | 40 ++++ .../ingestion/util/ServiceBusTracing.java | 20 ++ .../ingestion/util/ServiceBusTracingImpl.java | 142 ++++++++++++++ .../src/main/resources/application.properties | 9 +- .../ingestion/IngestionControllerTest.java | 43 ----- .../dronedelivery/ingestion/IngestionIT.java | 10 +- .../InstrumentedQueueClientTest.java | 67 +++++++ .../dronedelivery/ingestion/TracingTest.java | 179 ++++++++++++++++++ .../configuration/TestAppConfig.java | 18 ++ 20 files changed, 586 insertions(+), 160 deletions(-) create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClient.java create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClientImpl.java create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracing.java create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/InstrumentedQueueClientTest.java create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/TracingTest.java create mode 100644 src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java diff --git a/deployment.md b/deployment.md index fce9f200..9bacb712 100644 --- a/deployment.md +++ b/deployment.md @@ -436,10 +436,11 @@ cat $K8S/ingestion.yaml | \ # Create secret kubectl -n backend create secret generic ingestion-secrets \ - --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ - --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ - --from-literal=queue_keyname=IngestionServiceAccessKey \ - --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} + --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ + --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ + --from-literal=queue_keyname=IngestionServiceAccessKey \ + --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ + --from-literal=appinsights-ikey=${AI_IKEY} # Deploy service kubectl --namespace backend apply -f $K8S/ingestion-0.yaml diff --git a/deploymentARM.md b/deploymentARM.md index 98c61a0d..d000e784 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -357,7 +357,8 @@ kubectl -n backend create secret generic ingestion-secrets \ --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ --from-literal=queue_keyname=${INGESTION_ACCESS_KEY_NAME} \ - --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} + --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ + --from-literal=appinsights-ikey=${AI_IKEY} # Deploy service kubectl --namespace backend apply -f $K8S/ingestion-0.yaml diff --git a/k8s/ingestion.yaml b/k8s/ingestion.yaml index db285645..4e7bd357 100644 --- a/k8s/ingestion.yaml +++ b/k8s/ingestion.yaml @@ -44,7 +44,7 @@ spec: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: ingestion + name: &ingestion-container_name ingestion labels: app: ingestion version: 0.1.0 @@ -100,3 +100,11 @@ spec: secretKeyRef: name: ingestion-secrets key: queue_keyvalue + - name: APPINSIGHTS_INSTRUMENTATIONKEY + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: appinsights-ikey + - name: CONTAINER_NAME + value: *ingestion-container_name + diff --git a/src/shipping/ingestion/Dockerfile b/src/shipping/ingestion/Dockerfile index 762bfcd2..f65cc83f 100644 --- a/src/shipping/ingestion/Dockerfile +++ b/src/shipping/ingestion/Dockerfile @@ -15,6 +15,10 @@ RUN mvn package -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.sk FROM base as final WORKDIR /app + +ADD https://github.com/Microsoft/ApplicationInsights-Java/releases/download/2.3.0/applicationinsights-agent-2.3.0.jar ./appinsights/applicationinsights-agent-2.3.0.jar +ADD /etc/AI-Agent.xml ./appinsights/AI-Agent.xml + COPY --from=build /usr/src/app/target ./ -ENTRYPOINT ["java","-jar","ingestion-0.1.0.jar"] +ENTRYPOINT ["java","-Djava.security.egdfile=file:/dev/./urandom","-javaagent:/app/appinsights/applicationinsights-agent-2.3.0.jar","-jar","ingestion-0.1.0.jar"] diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 5a601cce..7127cf19 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -135,7 +135,19 @@ log4j-core 2.9.1 - + + + com.microsoft.azure + applicationinsights-spring-boot-starter + 1.1.1 + + + + com.microsoft.azure + applicationinsights-web + 2.3.0 + + diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java index b0cd2f74..ac59bb19 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/ApplicationProperties.java @@ -1,8 +1,5 @@ package com.fabrikam.dronedelivery.ingestion.configuration; -import java.util.ArrayList; -import java.util.List; - import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; @@ -41,12 +38,6 @@ public class ApplicationProperties { private int threadPoolExecutorQueueSize = 0; private int threadPoolExecutorMaxPoolSize = 0; private int messageAmqpClientPoolSize = 0; - - // Istio properties for distributed tracing - private List serviceMeshHeaders = new ArrayList(); - - // Correlation header for breadcrumb trail - private String serviceMeshCorrelationHeader = "serviceMeshCorrelationHeader"; public String getNamespace() { return namespace; @@ -103,29 +94,13 @@ public int getThreadPoolExecutorMaxPoolSize() { public void setThreadPoolExecutorMaxPoolSize(int maxPoolSize) { this.threadPoolExecutorMaxPoolSize = maxPoolSize; } - - public List getServiceMeshHeaders() { - return serviceMeshHeaders; - } - - public void setServiceMeshHeaders(List serviceMeshHeaders) { - this.serviceMeshHeaders = serviceMeshHeaders; - } - + //To resolve ${} in @Value @Bean public static PropertySourcesPlaceholderConfigurer propertyConfigInDev() { return new PropertySourcesPlaceholderConfigurer(); } - public String getServiceMeshCorrelationHeader() { - return serviceMeshCorrelationHeader; - } - - public void setServiceMeshCorrelationHeader(String serviceMeshCorrelationHeader) { - this.serviceMeshCorrelationHeader = serviceMeshCorrelationHeader; - } - public int getMessageAmqpClientPoolSize() { return messageAmqpClientPoolSize; } diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java index 4d4898bd..c8c357ea 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java @@ -2,7 +2,6 @@ import org.springframework.web.bind.annotation.RestController; -import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; import com.fabrikam.dronedelivery.ingestion.models.*; import com.fabrikam.dronedelivery.ingestion.service.*; @@ -38,18 +37,12 @@ public class IngestionController { private final static Logger log = LogManager.getLogger(IngestionController.class); - private static final String CorrelationHeaderTag = "CorrelationId"; - - @Autowired - private ApplicationProperties appProps; - @Autowired private Ingestion ingestion; @Autowired - public IngestionController(Ingestion ingestion, ApplicationProperties appProps) { + public IngestionController(Ingestion ingestion) { this.ingestion = ingestion; - this.appProps = appProps; } @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Bad Format data") // 400 @@ -87,12 +80,9 @@ public void exHandlerJsonError(JsonProcessingException e) { public CompletableFuture> scheduleDeliveryAsync(HttpServletResponse response, @RequestBody ExternalDelivery externalDelivery, @RequestHeader HttpHeaders httpHeaders) { - // Extract the correlation id and log it - String correlationId = httpHeaders.getFirst(appProps.getServiceMeshCorrelationHeader()); String deliveryId = UUID.randomUUID().toString(); - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put(CorrelationHeaderTag, - correlationId).put("DeliveryId", deliveryId)) { + try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) { log.info("In schedule delivery action with delivery request {}", externalDelivery.toString()); // Exceptions handled by exception handler @@ -123,8 +113,7 @@ public CompletableFuture> cancelDeliveryAsync(HttpServlet // Exceptions handled by exception handler // making standard in the controller - String correlationId = httpHeaders.getFirst(appProps.getServiceMeshCorrelationHeader()); - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put(CorrelationHeaderTag, correlationId)) + try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) { log.info("In cancel delivery action with id: {}", deliveryId); ingestion.cancelDeliveryAsync(deliveryId.toString(), httpHeaders.toSingleValueMap()); @@ -138,9 +127,7 @@ public CompletableFuture> cancelDeliveryAsync(HttpServlet public CompletableFuture> rescheduleDeliveryAsync(HttpServletResponse response, @RequestBody ExternalRescheduledDelivery externalRescheduledDelivery, @PathVariable("id") String deliveryId, @RequestHeader HttpHeaders httpHeaders) { - String correlationId = httpHeaders.getFirst(appProps.getServiceMeshCorrelationHeader()); - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put(CorrelationHeaderTag, correlationId) - .put("DeliveryId", deliveryId)) { + try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) { log.info("In reschedule delivery action with delivery request: {}", externalRescheduledDelivery.toString()); diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java index 68962332..94f57e11 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/service/IngestionImpl.java @@ -1,90 +1,69 @@ package com.fabrikam.dronedelivery.ingestion.service; + import com.fabrikam.dronedelivery.ingestion.models.*; +import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.util.HashMap; import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; import com.fabrikam.dronedelivery.ingestion.util.ClientPool; import com.google.gson.Gson; import com.google.gson.GsonBuilder; - import com.microsoft.azure.servicebus.Message; import com.microsoft.azure.servicebus.primitives.ServiceBusException; @Service public class IngestionImpl implements Ingestion { - @Autowired private ClientPool clientPool; - - @Autowired - private ApplicationProperties appProps; @Autowired - public IngestionImpl(ClientPool clientPool, ApplicationProperties appProps) { + public IngestionImpl(ClientPool clientPool) { this.clientPool = clientPool; - this.appProps = appProps; } @Async @Override - public void scheduleDeliveryAsync(DeliveryBase delivery, Map httpHeaders) { + public void scheduleDeliveryAsync(DeliveryBase delivery, Map httpHeaders) { Message sendEvent = getMessage(delivery, httpHeaders); sendEvent.getProperties().put("operation", "delivery"); - try { - clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ServiceBusException e) { - throw new RuntimeException(e); - } + this.enqueuMessageAsync(sendEvent); } @Async @Override - public void cancelDeliveryAsync(String deliveryId, Map httpHeaders) { + public void cancelDeliveryAsync(String deliveryId, Map httpHeaders) { Message sendEvent = getMessage(deliveryId, httpHeaders); sendEvent.getProperties().put("operation", "cancel"); - try { - clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ServiceBusException e) { - throw new RuntimeException(e); - - } + this.enqueuMessageAsync(sendEvent); } @Async @Override - public void rescheduleDeliveryAsync(DeliveryBase rescheduledDelivery, Map httpHeaders) { + public void rescheduleDeliveryAsync(DeliveryBase rescheduledDelivery, Map httpHeaders) { Message sendEvent = getMessage(rescheduledDelivery, httpHeaders); sendEvent.getProperties().put("operation", "reschedule"); - try { - clientPool.getConnection().sendAsync(sendEvent).thenApply((Void) -> "result"); - } catch (InterruptedException | ServiceBusException e) { - throw new RuntimeException(e); - } + this.enqueuMessageAsync(sendEvent); } - - private Message getMessage(Object deliveryObj, Map httpHeaders){ + + private Message getMessage(Object deliveryObj, Map httpHeaders) { Gson gson = new GsonBuilder().create(); byte[] payloadBytes = gson.toJson(deliveryObj).getBytes(Charset.defaultCharset()); Message sendEvent = new Message(payloadBytes); - - Map eventProps = new HashMap(); - for(String header:appProps.getServiceMeshHeaders()){ - if(httpHeaders.containsKey(header)){ - eventProps.put(header, httpHeaders.get(header)); - } - } - - if(eventProps.size()>0){ - sendEvent.getProperties().putAll(eventProps); - } - + return sendEvent; } -} + + @Async + private void enqueuMessageAsync(Message message) { + try { + this.clientPool.getConnection().sendAsync(message).thenApply((Void) -> "result"); + } catch (InterruptedException | ServiceBusException | URISyntaxException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java index e533be38..4dfddec3 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPool.java @@ -2,12 +2,12 @@ import org.springframework.scheduling.annotation.Async; -import com.microsoft.azure.servicebus.QueueClient; +import java.net.URISyntaxException; + import com.microsoft.azure.servicebus.primitives.ServiceBusException; public interface ClientPool { @Async - public QueueClient getConnection() throws InterruptedException, ServiceBusException; - + public InstrumentedQueueClient getConnection() throws InterruptedException, ServiceBusException, URISyntaxException; } diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java index 812e6990..0304160d 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java @@ -1,5 +1,8 @@ package com.fabrikam.dronedelivery.ingestion.util; +import java.net.URI; +import java.net.URISyntaxException; + import com.fabrikam.dronedelivery.ingestion.configuration.*; import com.microsoft.azure.servicebus.QueueClient; @@ -14,40 +17,52 @@ @Service public class ClientPoolImpl implements ClientPool { - private final QueueClient[] queueClients; + private static final String SCHEME = "https"; + + private final InstrumentedQueueClient[] queueClients; private final String[] queueNames; private final ApplicationProperties appProperties; private final String nameSpace; private final String sasKeyName; private final String sasKey; - + private final ServiceBusTracing tracing; @Autowired - public ClientPoolImpl(ApplicationProperties appProps) { + public ClientPoolImpl(ApplicationProperties appProps, ServiceBusTracing tracing) { this.appProperties = appProps; - - - this.queueNames = System.getenv(appProperties.getEnvQueueName()).split(","); - nameSpace = System.getenv(appProperties.getEnvNameSpace()); + this.tracing = tracing; + + this.queueNames = System.getenv(appProperties.getEnvQueueName()).split(","); + nameSpace = System.getenv(appProperties.getEnvNameSpace()); sasKeyName = System.getenv(appProperties.getEnvsasKeyName()); sasKey = System.getenv(appProperties.getEnvsasKey()); - - this.queueClients = new QueueClient[this.appProperties.getMessageAmqpClientPoolSize()]; + + this.queueClients = new InstrumentedQueueClient[this.appProperties.getMessageAmqpClientPoolSize()]; } @Async @Override - public QueueClient getConnection() throws InterruptedException, ServiceBusException { + public InstrumentedQueueClient getConnection() + throws InterruptedException, ServiceBusException, URISyntaxException { int poolId = (int) (Math.random() * queueClients.length); int eventHubId = (int) (Math.random() * queueNames.length); if (queueClients[poolId] == null) { - ConnectionStringBuilder connectionString = new ConnectionStringBuilder(nameSpace, - queueNames[eventHubId], sasKeyName, sasKey); - queueClients[poolId] = new QueueClient(connectionString, ReceiveMode.PEEKLOCK); + ConnectionStringBuilder connectionString = new ConnectionStringBuilder(nameSpace, queueNames[eventHubId], + sasKeyName, sasKey); + + queueClients[poolId] = new InstrumentedQueueClientImpl( + new URI(SCHEME, + connectionString + .getEndpoint() + .getHost(), + "/", + null).toString(), + new QueueClient(connectionString, ReceiveMode.PEEKLOCK), + tracing); } return queueClients[poolId]; } -} +} \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClient.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClient.java new file mode 100644 index 00000000..139493fa --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClient.java @@ -0,0 +1,14 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +import java.util.concurrent.CompletableFuture; + +import com.microsoft.azure.servicebus.IMessage; + +public interface InstrumentedQueueClient { + CompletableFuture sendAsync(IMessage message); +} diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClientImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClientImpl.java new file mode 100644 index 00000000..65e62093 --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/InstrumentedQueueClientImpl.java @@ -0,0 +1,40 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +import java.util.concurrent.CompletableFuture; + +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.IQueueClient; + +public class InstrumentedQueueClientImpl implements InstrumentedQueueClient { + private final ServiceBusTracing tracing; + private final String endpoint; + private final IQueueClient client; + + public InstrumentedQueueClientImpl( + String endpoint, + IQueueClient client, + ServiceBusTracing tracing) { + this.endpoint = endpoint; + this.client = client; + this.tracing = tracing; + } + + @Override + public CompletableFuture sendAsync(IMessage message) { + String queueName = this.client.getQueueName(); + + return this.tracing.trackAndCorrelateServiceBusDependency( + endpoint, + queueName, + message, + (m) -> + { + return this.client.sendAsync(m); + }); + } +} diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracing.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracing.java new file mode 100644 index 00000000..23ea5653 --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracing.java @@ -0,0 +1,20 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; + +import com.microsoft.azure.servicebus.IMessage; + +public interface ServiceBusTracing { + + public CompletableFuture trackAndCorrelateServiceBusDependency( + String endpoint, + String queueName, + IMessage message, + Function> func); +} diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java new file mode 100644 index 00000000..a5849b2a --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java @@ -0,0 +1,142 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import com.microsoft.applicationinsights.TelemetryClient; +import com.microsoft.applicationinsights.extensibility.context.OperationContext; +import com.microsoft.applicationinsights.telemetry.Duration; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; +import com.microsoft.applicationinsights.telemetry.RequestTelemetry; +import com.microsoft.applicationinsights.web.internal.ThreadContext; +import com.microsoft.applicationinsights.web.internal.correlation.TelemetryCorrelationUtils; +import com.microsoft.azure.servicebus.IMessage; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ServiceBusTracingImpl implements ServiceBusTracing { + private static final long NANO_TO_MILI_SECONDS = TimeUnit.MILLISECONDS.toNanos(1); + + private static final String DIAGNOSTIC_ID_PROPERTY_NAME = "Diagnostic-Id"; + + private static final String SERVICE_BUS_REMOTE_DEPENDENCY_TYPE = "Azure Service Bus"; + + private static final String SERVICE_BUS_REMOTE_DEPENDENDENCY_NAME = "Send"; + + private static final String TARGET_REMOTE_DEPENDENDENCY_FORMAT = "%s | %s"; + + private final TelemetryClient telemetryClient; + + @Autowired + public ServiceBusTracingImpl(TelemetryClient telemetryClient) + { + this.telemetryClient = telemetryClient; + } + + @Override + public CompletableFuture trackAndCorrelateServiceBusDependency( + String endpoint, + String queueName, + IMessage message, + Function> func) { + + CompletableFuture result; + + String target = String.format( + Locale.US, + TARGET_REMOTE_DEPENDENDENCY_FORMAT, + endpoint, + queueName); + + propagateCorrelationProperties(message); + + final long start = System.nanoTime(); + result = func.apply(message); + result.whenComplete((r,t) -> { + final long finish = System.nanoTime(); + final long intervalMs = (finish - start) / NANO_TO_MILI_SECONDS; + boolean successful = false; + if (t == null) { + successful = true; + } + + telemetryClient.trackDependency( + createRemoteDependencyTelemetry( + target, + new Duration(intervalMs), + successful)); + + if(successful == false){ + RuntimeException runtimeEx = new RuntimeException(t); + telemetryClient.trackException(runtimeEx); + } + }); + + return result; + } + + private static void propagateCorrelationProperties(IMessage message) { + String parentId = ThreadContext + .getRequestTelemetryContext() + .getHttpRequestTelemetry() + .getId(); + + // propagate Service Bus required properties + // https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-end-to-end-tracing + // https://docs.microsoft.com/en-us/azure/azure-monitor/app/correlation#telemetry-correlation-in-the-java-sdk + message.getProperties().put(DIAGNOSTIC_ID_PROPERTY_NAME, parentId); + } + + private static RemoteDependencyTelemetry createRemoteDependencyTelemetry( + String target, + Duration duration, + boolean successful) { + String dependencyId = TelemetryCorrelationUtils + .generateChildDependencyId(); + + RemoteDependencyTelemetry dependencyTelemetry = + new RemoteDependencyTelemetry( + SERVICE_BUS_REMOTE_DEPENDENDENCY_NAME, + "", + duration, + successful); + + dependencyTelemetry.setId(dependencyId); + dependencyTelemetry.setType(SERVICE_BUS_REMOTE_DEPENDENCY_TYPE); + dependencyTelemetry.setTarget(target); + + RequestTelemetry requestTelemetry = ThreadContext + .getRequestTelemetryContext() + .getHttpRequestTelemetry(); + + String parentId = requestTelemetry.getId(); + String operationId = extractRootId(parentId); + + OperationContext operationContext = dependencyTelemetry + .getContext() + .getOperation(); + operationContext.setParentId(parentId); + operationContext.setId(operationId); + + return dependencyTelemetry; + } + + private static String extractRootId(String parentId) { + int rootEnd = parentId.indexOf('.'); + if (rootEnd < 0) { + rootEnd = parentId.length(); + } + + int rootStart = parentId.charAt(0) == '|' ? 1 : 0; + return parentId.substring(rootStart, rootEnd); + } +} \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/resources/application.properties b/src/shipping/ingestion/src/main/resources/application.properties index 5523f018..96f41dce 100644 --- a/src/shipping/ingestion/src/main/resources/application.properties +++ b/src/shipping/ingestion/src/main/resources/application.properties @@ -1,7 +1,7 @@ ############################################### # add environment variables # QUEUE_NAMESPACE QUEUE_NAME QUEUE_KEYNAME QUEUE_KEYVALUE -# To debug/run tests. Yaml file will have to provide +# To debug/run tests. Yaml file will have to provide # environment va ################################################ service.threadPoolExecutorQueueSize=10000 @@ -12,8 +12,11 @@ service.envNameSpace=QUEUE_NAMESPACE service.envQueueName=QUEUE_NAME service.envsasKeyName=QUEUE_KEYNAME service.envsasKey=QUEUE_KEYVALUE -service.serviceMeshHeaders=l5d-ctx-deadline,l5d-ctx-trace -service.serviceMeshCorrelationHeader=l5d-ctx-trace server.port:80 +logging.level.org.springframework.web=INFO +# Specify the name of your springboot application. This can be any logical name you would like to give to your app. +spring.application.name=${CONTAINER_NAME} +# Logging level [all, trace, info, warn, error, off]. Default value: error. +azure.application-insights.logger.level=info \ No newline at end of file diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java index 25fd7e18..69165a07 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionControllerTest.java @@ -107,8 +107,6 @@ public void ScheduleDeliveryIsAccepted() throws Exception { Mockito.doNothing().when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); - MvcResult resultActions = mockMvc.perform( post("/api/deliveryrequests") .contentType(MediaType.APPLICATION_JSON) @@ -121,7 +119,6 @@ public void ScheduleDeliveryIsAccepted() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -131,7 +128,6 @@ public void RescheduleDeliveryIsOk() throws Exception { Mockito.doNothing().when(ingestionimplMock) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); MvcResult resultActions = mockMvc.perform(patch("/api/deliveryrequests/" + deliveryId) @@ -144,7 +140,6 @@ public void RescheduleDeliveryIsOk() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -153,7 +148,6 @@ public void CancelDeliveryIsOk() throws Exception { Mockito.doNothing().when(ingestionimplMock) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); MvcResult resultActions = mockMvc .perform(delete("/api/deliveryrequests/" + deliveryId).contentType(MediaType.APPLICATION_JSON)) @@ -164,7 +158,6 @@ public void CancelDeliveryIsOk() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -184,7 +177,6 @@ public void scheduleDeliveryHandlesServiceBusException() throws Exception { .when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( post("/api/deliveryrequests") @@ -194,7 +186,6 @@ public void scheduleDeliveryHandlesServiceBusException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -204,7 +195,6 @@ public void schedulerDeliveryHandlesIllegalArgumentException() throws Exception .when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( post("/api/deliveryrequests") @@ -214,7 +204,6 @@ public void schedulerDeliveryHandlesIllegalArgumentException() throws Exception Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -224,7 +213,6 @@ public void scheduleDeliveryHandlesIOException() throws Exception { .when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( post("/api/deliveryrequests") @@ -234,7 +222,6 @@ public void scheduleDeliveryHandlesIOException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -244,7 +231,6 @@ public void scheduleDeliveryHandlesJsonProcessingException() throws Exception { .when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( post("/api/deliveryrequests") .contentType(MediaType.APPLICATION_JSON) @@ -253,8 +239,6 @@ public void scheduleDeliveryHandlesJsonProcessingException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -265,7 +249,6 @@ public void scheduleDeliveryHandlesMethodArgumentTypeMismatchException() throws .when(ingestionimplMock) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( post("/api/deliveryrequests") .contentType(MediaType.APPLICATION_JSON) @@ -274,8 +257,6 @@ public void scheduleDeliveryHandlesMethodArgumentTypeMismatchException() throws Mockito.verify(ingestionimplMock, times(1)) .scheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -288,7 +269,6 @@ public void rescheduleDeliveryHandlesServiceBusException() throws Exception { .when(ingestionimplMock) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -300,7 +280,6 @@ public void rescheduleDeliveryHandlesServiceBusException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -310,7 +289,6 @@ public void reschedulerDeliveryHandlesIllegalArgumentException() throws Exceptio .when(ingestionimplMock) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -323,7 +301,6 @@ public void reschedulerDeliveryHandlesIllegalArgumentException() throws Exceptio Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -333,7 +310,6 @@ public void rescheduleDeliveryHandlesIOException() throws Exception { .when(ingestionimplMock) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -345,7 +321,6 @@ public void rescheduleDeliveryHandlesIOException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -357,7 +332,6 @@ public void rescheduleDeliveryHandlesJsonProcessingException() throws Exception String deliveryId = externalRDelivery.getDeliveryId().toString(); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( patch("/api/deliveryrequests/" + deliveryId) .contentType(MediaType.APPLICATION_JSON) @@ -366,8 +340,6 @@ public void rescheduleDeliveryHandlesJsonProcessingException() throws Exception Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -378,7 +350,6 @@ public void rescheduleDeliveryHandlesMethodArgumentTypeMismatchException() throw .when(ingestionimplMock) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); mockMvc.perform( @@ -389,8 +360,6 @@ public void rescheduleDeliveryHandlesMethodArgumentTypeMismatchException() throw Mockito.verify(ingestionimplMock, times(1)) .rescheduleDeliveryAsync(Mockito.any(DeliveryBase.class), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -403,7 +372,6 @@ public void cancelscheduleDeliveryHandlesServiceBusException() throws Exception .when(ingestionimplMock) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -415,7 +383,6 @@ public void cancelscheduleDeliveryHandlesServiceBusException() throws Exception Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -425,7 +392,6 @@ public void cancelschedulerDeliveryHandlesIllegalArgumentException() throws Exce .when(ingestionimplMock) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -438,7 +404,6 @@ public void cancelschedulerDeliveryHandlesIllegalArgumentException() throws Exce Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -449,7 +414,6 @@ public void cancelscheduleDeliveryHandlesIOException() throws Exception { .when(ingestionimplMock) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); @@ -461,7 +425,6 @@ public void cancelscheduleDeliveryHandlesIOException() throws Exception { Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @Test @@ -473,7 +436,6 @@ public void cancelscheduleDeliveryHandlesJsonProcessingException() throws Except String deliveryId = externalRDelivery.getDeliveryId().toString(); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); mockMvc.perform( delete("/api/deliveryrequests/" + deliveryId) .contentType(MediaType.APPLICATION_JSON) @@ -482,8 +444,6 @@ public void cancelscheduleDeliveryHandlesJsonProcessingException() throws Except Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } @@ -494,7 +454,6 @@ public void cancelscheduleDeliveryHandlesMethodArgumentTypeMismatchException() t .when(ingestionimplMock) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - Mockito.when(appPropsMock.getServiceMeshCorrelationHeader()).thenReturn("header"); String deliveryId = externalRDelivery.getDeliveryId().toString(); mockMvc.perform( @@ -505,8 +464,6 @@ public void cancelscheduleDeliveryHandlesMethodArgumentTypeMismatchException() t Mockito.verify(ingestionimplMock, times(1)) .cancelDeliveryAsync(Mockito.anyString(), Mockito.anyMap()); - - Mockito.verify(appPropsMock, times(1)).getServiceMeshCorrelationHeader(); } diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java index 9376a9be..ec19d60b 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/IngestionIT.java @@ -9,7 +9,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -18,7 +17,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.web.context.WebApplicationContext; - import com.fabrikam.dronedelivery.ingestion.models.ConfirmationRequired; import com.fabrikam.dronedelivery.ingestion.models.ContainerSize; import com.fabrikam.dronedelivery.ingestion.models.ExternalDelivery; @@ -29,7 +27,9 @@ import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import org.springframework.test.context.junit4.SpringRunner; + import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -37,7 +37,11 @@ @RunWith(SpringRunner.class) @SpringBootTest public class IngestionIT { - + + static { + System.setProperty("CONTAINER_NAME", "test-container"); + } + @Autowired private WebApplicationContext wac; private MockMvc mockMvc; diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/InstrumentedQueueClientTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/InstrumentedQueueClientTest.java new file mode 100644 index 00000000..ec35467e --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/InstrumentedQueueClientTest.java @@ -0,0 +1,67 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion; + +import com.fabrikam.dronedelivery.ingestion.util.InstrumentedQueueClientImpl; +import com.fabrikam.dronedelivery.ingestion.util.ServiceBusTracing; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.IQueueClient; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +public class InstrumentedQueueClientTest { + + private static final String SB_ENDPOINT = "sbEndpoint"; + + private static final String SB_QUEUE_NAME = "sbQueueName"; + + private @Mock ServiceBusTracing tracingMock; + + private @Mock IMessage messageMock; + + private @InjectMocks InstrumentedQueueClientImpl instrumentedQueueClientImpl; + + private IQueueClient queueClientlMock; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + queueClientlMock = mock(IQueueClient.class); + when(queueClientlMock.getQueueName()).thenReturn(SB_QUEUE_NAME); + + instrumentedQueueClientImpl = + new InstrumentedQueueClientImpl( + SB_ENDPOINT, + queueClientlMock, + tracingMock); + } + + @Test + public void sendAsync_ThenWrappedByTrackAndCorrelatedWithProperInfo() throws Exception { + + // Arrange + + // Act + instrumentedQueueClientImpl.sendAsync(messageMock); + + // Assert + verify(queueClientlMock, times(1)).getQueueName(); + verify(tracingMock, times(1)) + .trackAndCorrelateServiceBusDependency( + eq(SB_ENDPOINT), + eq(SB_QUEUE_NAME), + eq(messageMock), + Mockito.any()); + } +} diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/TracingTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/TracingTest.java new file mode 100644 index 00000000..7493463d --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/TracingTest.java @@ -0,0 +1,179 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion; + +import com.fabrikam.dronedelivery.ingestion.util.ServiceBusTracingImpl; +import com.microsoft.applicationinsights.TelemetryClient; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; +import com.microsoft.applicationinsights.telemetry.RequestTelemetry; +import com.microsoft.applicationinsights.web.internal.RequestTelemetryContext; +import com.microsoft.applicationinsights.web.internal.ThreadContext; +import com.microsoft.azure.servicebus.IMessage; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.invocation.Invocation; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.*; + +import java.util.Map; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.function.Function; + + +public class TracingTest { + private static final String TRACK_DEPENDENCY_METHOD_NAME = "trackDependency"; + + private static final int DELAY_COMPLETION_MS = 100; + + private static final String SB_QUEUE_NAME = "sbQueueName"; + + private static final String SB_ENDPOINT = "sbEndpoint"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private @Mock IMessage messageMock; + + private @Mock CompletableFuture futureMock; + + private @Mock Function> functionMock; + + private @Mock Map mapMock; + + private @Mock TelemetryClient telemetryClientMock; + + private @Mock RequestTelemetryContext telemetryContextMock; + + private @InjectMocks ServiceBusTracingImpl tracing; + + private RequestTelemetry requestTelemetryFake; + + @Before + public void setUp() throws Exception { + + MockitoAnnotations.initMocks(this); + when(messageMock.getProperties()).thenReturn(mapMock); + + requestTelemetryFake = new RequestTelemetry(); + requestTelemetryFake.setId("42"); + + when(telemetryContextMock.getHttpRequestTelemetry()).thenReturn(requestTelemetryFake); + } + + @Test + public void trackAndCorrelate_ThenPropagateDiagnoticId() throws Exception { + + // Arrange + ThreadContext.setRequestTelemetryContext(telemetryContextMock); + when(functionMock.apply(eq(messageMock))).thenReturn(futureMock); + + // Act + tracing.trackAndCorrelateServiceBusDependency(SB_ENDPOINT, SB_QUEUE_NAME, messageMock, functionMock); + + // Assert + verify(messageMock, times(1)).getProperties(); + // TODO: validate proper Request-Id format + verify(mapMock, times(1)).put(eq("Diagnostic-Id"), eq("42")); + } + + @Test(timeout = DELAY_COMPLETION_MS * 10) + public void trackAndCorrelate_ThenApplyFunctionAndTrackDependency() throws Exception { + + // Arrange + ThreadContext.setRequestTelemetryContext(telemetryContextMock); + CompletableFuture result = new CompletableFuture<>(); + ArgumentCaptor argument = ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + // Act + tracing.trackAndCorrelateServiceBusDependency( + SB_ENDPOINT, + SB_QUEUE_NAME, + messageMock, + (m) -> { + try { + Thread.sleep(DELAY_COMPLETION_MS); + result.complete(null); + } catch (InterruptedException e) { + // shallowed exception is ok in here, please expect this test to timeout + } + + return result; + }); + + while (!existsTrackDepedency(mockingDetails(telemetryClientMock) + .getInvocations())){ + Thread.sleep(DELAY_COMPLETION_MS / 10); + } + + // Assert + verify(telemetryClientMock, times(1)) + .trackDependency(any(RemoteDependencyTelemetry.class)); + verify(telemetryClientMock).trackDependency(argument.capture()); + assertTrue(argument.getValue().getDuration().getTotalMilliseconds() >= DELAY_COMPLETION_MS); + assertEquals( + SB_ENDPOINT + " | "+ SB_QUEUE_NAME, + argument.getValue().getTarget()); + assertTrue(argument.getValue().getSuccess()); + } + + @Test(expected = CompletionException.class) + public void trackAndCorrelate_ThenThrownRuntimeExceptionWhenErr() throws Throwable { + // Arrange + ThreadContext.setRequestTelemetryContext(telemetryContextMock); + CompletableFuture result = new CompletableFuture<>(); + ArgumentCaptor argument = ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + + when(functionMock.apply(eq(messageMock))).thenReturn(result); + + // Act + tracing.trackAndCorrelateServiceBusDependency( + SB_ENDPOINT, + SB_QUEUE_NAME, + messageMock, + functionMock); + + try { + result.completeExceptionally(new RuntimeException()); + result.join(); + } + finally{ + // Assert + verify(functionMock, times(1)) + .apply(eq(messageMock)); + verify(telemetryClientMock, times(1)) + .trackException(any(RuntimeException.class)); + verify(telemetryClientMock, times(1)) + .trackDependency(any(RemoteDependencyTelemetry.class)); + verify(telemetryClientMock).trackDependency(argument.capture()); + assertEquals( + SB_ENDPOINT + " | "+ SB_QUEUE_NAME, + argument.getValue().getTarget()); + assertFalse(argument.getValue().getSuccess()); + } + } + + private static boolean existsTrackDepedency(Collection invcations) + { + for (Invocation invoke : invcations) { + if(invoke.getMethod().getName().equals(TRACK_DEPENDENCY_METHOD_NAME)) + { + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java new file mode 100644 index 00000000..a7135bc5 --- /dev/null +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java @@ -0,0 +1,18 @@ +package com.fabrikam.dronedelivery.ingestion.configuration; + +import com.microsoft.applicationinsights.TelemetryClient; + +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +public class TestAppConfig { + + @Bean + @Primary + public TelemetryClient getTelemetryClient() { + return Mockito.mock(TelemetryClient.class); + } +} From 5b6ee28f41b296730fc6dc8ab58a8826f5f1a10d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 6 Mar 2019 19:56:23 +0000 Subject: [PATCH 082/246] Merged PR 672: change deployment to use helm for dronescheduler - add chart for dronescheduler - add deployment instructions for using helm solved: #8990 Related work items: #8990 --- .../delivery/templates/delivery-service.yaml | 3 +- charts/dronescheduler/Chart.yaml | 4 ++ charts/dronescheduler/templates/NOTES.txt | 10 +++ charts/dronescheduler/templates/_helpers.tpl | 32 +++++++++ .../templates/dronescheduler-deploy.yaml | 62 +++++++++++++++++ .../templates}/dronescheduler-identity.yaml | 14 ++-- .../dronescheduler-service-experimental.yaml | 31 +++++++++ .../templates/dronescheduler-service-sxs.yaml | 28 ++++++++ .../templates/dronescheduler-service.yaml | 31 +++++++++ charts/dronescheduler/values.yaml | 13 ++++ deployment.md | 28 ++++---- deploymentARM.md | 32 ++++----- k8s/dronescheduler.yaml | 68 ------------------- 13 files changed, 248 insertions(+), 108 deletions(-) create mode 100644 charts/dronescheduler/Chart.yaml create mode 100644 charts/dronescheduler/templates/NOTES.txt create mode 100644 charts/dronescheduler/templates/_helpers.tpl create mode 100644 charts/dronescheduler/templates/dronescheduler-deploy.yaml rename {k8s => charts/dronescheduler/templates}/dronescheduler-identity.yaml (72%) create mode 100644 charts/dronescheduler/templates/dronescheduler-service-experimental.yaml create mode 100644 charts/dronescheduler/templates/dronescheduler-service-sxs.yaml create mode 100644 charts/dronescheduler/templates/dronescheduler-service.yaml create mode 100644 charts/dronescheduler/values.yaml delete mode 100644 k8s/dronescheduler.yaml diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 5f5d5c4d..fb234077 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -15,8 +15,7 @@ kind: Service metadata: name: delivery labels: - app: delivery - app.kubernetes.io/component: backend + app.kubernetes.io/app: delivery app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery app.kubernetes.io/managed-by: azurepipelines diff --git a/charts/dronescheduler/Chart.yaml b/charts/dronescheduler/Chart.yaml new file mode 100644 index 00000000..0df49fbf --- /dev/null +++ b/charts/dronescheduler/Chart.yaml @@ -0,0 +1,4 @@ +name: dronescheduler +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Scheduler Service diff --git a/charts/dronescheduler/templates/NOTES.txt b/charts/dronescheduler/templates/NOTES.txt new file mode 100644 index 00000000..c5dbfc98 --- /dev/null +++ b/charts/dronescheduler/templates/NOTES.txt @@ -0,0 +1,10 @@ +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +All the objects were created in the namespace {{ .Values.namespace }} + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} + $ helm get {{ .Release.Name }} diff --git a/charts/dronescheduler/templates/_helpers.tpl b/charts/dronescheduler/templates/_helpers.tpl new file mode 100644 index 00000000..1cd08a99 --- /dev/null +++ b/charts/dronescheduler/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "dronescheduler.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dronescheduler.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dronescheduler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml new file mode 100644 index 00000000..dab1fb87 --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -0,0 +1,62 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler +################################################################################################### +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "dronescheduler.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "dronescheduler.chart" . }} + aadpodidbinding: dronescheduler + annotations: + kubernetes.io/change-cause: {{ .Values.reason }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "dronescheduler.chart" . }} + aadpodidbinding: dronescheduler + spec: + securityContext: + fsGroup: 1 + containers: + - name: fabrikam-dronescheduler + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: KEY_VAULT_URI + value: {{ .Values.keyvault.uri }} + - name: no_proxy + value: 169.254.169.254 + ports: + - name: service + containerPort: 8080 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G diff --git a/k8s/dronescheduler-identity.yaml b/charts/dronescheduler/templates/dronescheduler-identity.yaml similarity index 72% rename from k8s/dronescheduler-identity.yaml rename to charts/dronescheduler/templates/dronescheduler-identity.yaml index 50b62529..ab7fc6e6 100644 --- a/k8s/dronescheduler-identity.yaml +++ b/charts/dronescheduler/templates/dronescheduler-identity.yaml @@ -4,21 +4,27 @@ # ------------------------------------------------------------ ################################################################################################### -# Drone Scheduler service identity +# Dronescheduler service identity ################################################################################################### apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: name: dronescheduler-identity + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "0" spec: type: 0 - ResourceID: "identityResourceId" - ClientID: "identityClientid" + ResourceID: {{ .Values.identity.resourceid }} + ClientID: {{ .Values.identity.clientid }} --- apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: name: dronescheduler-identity-binding + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "1" spec: AzureIdentity: dronescheduler-identity - Selector: dronescheduler \ No newline at end of file + Selector: dronescheduler diff --git a/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml b/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml new file mode 100644 index 00000000..dcafda2c --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler service experimental +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: dronescheduler-experimental + labels: + app.kubernetes.io/name: dronescheduler + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/managed-by: azurepipelines + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: dronescheduler + app.kubernetes.io/instance: dronescheduler-v0.1.0 +{{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml b/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml new file mode 100644 index 00000000..e41c4dd7 --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml @@ -0,0 +1,28 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler service +################################################################################################### +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dronescheduler.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedronescheduler + helm.sh/chart: {{ include "dronescheduler.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml new file mode 100644 index 00000000..b9e3eb37 --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler service +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: dronescheduler + labels: + app.kubernetes.io/app: dronescheduler + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/managed-by: azurepipelines + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: dronescheduler + app.kubernetes.io/instance: dronescheduler-v0.1.0 +{{ end }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml new file mode 100644 index 00000000..26065bd3 --- /dev/null +++ b/charts/dronescheduler/values.yaml @@ -0,0 +1,13 @@ +# Default values for dronescheduler. +replicaCount: 1 +identity: + clientid: + resourceid: +dockerregistry: +image: + repository: + tag: + pullPolicy: IfNotPresent +keyvault: + uri: +reason: unknown diff --git a/deployment.md b/deployment.md index 9bacb712..72aba51f 100644 --- a/deployment.md +++ b/deployment.md @@ -484,12 +484,6 @@ az keyvault set-policy --name $DRONESCHEDULER_KEYVAULT_NAME --secret-permissions # Allow the cluster to manage the identity to assign to pods az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID - -# Deploy the identity resources -cat $K8S/dronescheduler-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" > $K8S/dronescheduler-identity-0.yaml -kubectl apply -f $K8S/dronescheduler-identity-0.yaml ``` Build and publish the container image @@ -503,19 +497,21 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/dronescheduler:0.1.0 ``` -Deploy the dronescheduler services: - +Deploy the dronescheduler service: ```bash -# Update the image tag in the deployment YAML -cat $K8S/dronescheduler.yaml | \ - sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" | \ - sed "s#value: \"KeyVault_Name\"#value: $DRONESCHEDULER_KEYVAULT_URI#g" > $K8S/dronescheduler-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml +helm install $HELM_CHARTS/dronescheduler/ \ + --set image.tag=0.1.0 \ + --set image.repository=dronescheduler \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_ID \ + --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ + --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ + --namespace backend \ + --name dronescheduler-v0.1.0 -## Verify all services are running: -kubectl get pods -n backend +# Verify the pod is created +helm status dronescheduler-v0.1.0 ``` ## Validate the application is running diff --git a/deploymentARM.md b/deploymentARM.md index d000e784..4871457e 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -385,14 +385,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) - -# Deploy the identity resources -cat $K8S/dronescheduler-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" > $K8S/dronescheduler-identity-0.yaml -kubectl apply -f $K8S/dronescheduler-identity-0.yaml +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) ``` Build and publish the container image @@ -406,19 +400,21 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/dronescheduler:0.1.0 ``` -Deploy the dronescheduler services: - +Deploy the dronescheduler service: ```bash -# Update the image tag in the deployment YAML -cat $K8S/dronescheduler.yaml | \ - sed "s#image:#image: $ACR_SERVER/dronescheduler:0.1.0#g" | \ - sed "s#value: \"KeyVault_Name\"#value: $DRONESCHEDULER_KEYVAULT_URI#g" > $K8S/dronescheduler-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/dronescheduler-0.yaml +helm install $HELM_CHARTS/dronescheduler/ \ + --set image.tag=0.1.0 \ + --set image.repository=dronescheduler \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_ID \ + --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ + --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ + --namespace backend \ + --name dronescheduler-v0.1.0 -## Verify all services are running: -kubectl get pods -n backend +# Verify the pod is created +helm status dronescheduler-v0.1.0 ``` ## Validate the application is running diff --git a/k8s/dronescheduler.yaml b/k8s/dronescheduler.yaml deleted file mode 100644 index 50318788..00000000 --- a/k8s/dronescheduler.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Drone service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: dronescheduler - labels: - app: dronescheduler - bc: dronemgmt - co: fabrikam -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app: dronescheduler ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: dronescheduler - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: dronescheduler - version: 0.1.0 - bc: dronemgmt - co: fabrikam - aadpodidbinding: dronescheduler -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: dronescheduler - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: dronescheduler - version: 0.1.0 - bc: dronemgmt - co: fabrikam - aadpodidbinding: dronescheduler - annotations: - team: droneschedulerservice - spec: - containers: - - name: dronescheduler - image: - env: - - name: KEY_VAULT_URI - value: "KeyVault_Name" - - name: no_proxy - value: 169.254.169.254 - ports: - - name: service - containerPort: 8080 - resources: - requests: - cpu: 1 - memory: 1G - limits: - cpu: 1 - memory: 1G From 694322c6f07c464629546e88a2b68f1df4196cbf Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 7 Mar 2019 18:44:08 +0000 Subject: [PATCH 083/246] Merged PR 676: change deployment to use helm for package change deployment to use helm for package solved: #8989 Related work items: #8989 --- charts/package/templates/package-deploy.yaml | 23 +++++-- .../package-service-experimental.yaml | 33 ++++++++++ .../templates/package-service-sxs.yaml | 29 +++++++++ charts/package/templates/package-service.yaml | 54 +++++++++------ charts/package/values.yaml | 8 +-- deployment.md | 12 ++-- deploymentARM.md | 12 ++-- k8s/package.yml | 65 ------------------- 8 files changed, 130 insertions(+), 106 deletions(-) create mode 100644 charts/package/templates/package-service-experimental.yaml create mode 100644 charts/package/templates/package-service-sxs.yaml delete mode 100644 k8s/package.yml diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index a19dda8a..f7a8903c 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -1,3 +1,11 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package +################################################################################################### apiVersion: apps/v1beta2 kind: Deployment metadata: @@ -8,7 +16,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronepackage + app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "package.chart" . }} annotations: kubernetes.io/change-cause: {{ .Values.reason }} @@ -30,21 +38,24 @@ spec: helm.sh/chart: {{ include "package.chart" . }} spec: containers: - - name: &package-container_name package + - name: &package-container_name fabrikam-package image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CONNECTION_STRING valueFrom: secretKeyRef: name: package-secrets - key: {{ .Release.Name }}-mongodb-pwd + key: mongodb-pwd + - name: COLLECTION_NAME + value: packages - name: APPINSIGHTS_INSTRUMENTATIONKEY valueFrom: secretKeyRef: name: package-secrets - key: {{ .Release.Name }}-appinsights-ikey - - name: COLLECTION_NAME - value: packages + key: appinsights-ikey + - name: LOG_LEVEL + value: {{ .Values.log.level }} - name: CONTAINER_NAME value: *package-container_name ports: diff --git a/charts/package/templates/package-service-experimental.yaml b/charts/package/templates/package-service-experimental.yaml new file mode 100644 index 00000000..ab528da5 --- /dev/null +++ b/charts/package/templates/package-service-experimental.yaml @@ -0,0 +1,33 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package service experimental +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "package-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: package-experimental + labels: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "package.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{ end }} diff --git a/charts/package/templates/package-service-sxs.yaml b/charts/package/templates/package-service-sxs.yaml new file mode 100644 index 00000000..4b532e7a --- /dev/null +++ b/charts/package/templates/package-service-sxs.yaml @@ -0,0 +1,29 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package service sxs +################################################################################################### + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "package.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "package.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index 07eedb12..f31c9b27 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -1,21 +1,33 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "package.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronepackage - helm.sh/chart: {{ include "package.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - clusterIP: None +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# package service +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "package-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: package + labels: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "package.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{ end }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 1c8e550b..a997bfc1 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -1,10 +1,10 @@ -# Default values for package. +# Default values for package service. replicaCount: 1 +dockerregistry: image: repository: tag: pullPolicy: IfNotPresent -service: - type: ClusterIP -dockerregistry: reason: unknown +log: + level: error diff --git a/deployment.md b/deployment.md index 72aba51f..c3f77b23 100644 --- a/deployment.md +++ b/deployment.md @@ -264,9 +264,6 @@ docker push $ACR_SERVER/package:0.1.0 Deploy the Package service ```bash -# Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml - # Create secret export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') kubectl -n backend create \ @@ -275,10 +272,15 @@ kubectl -n backend create \ --from-literal=appinsights-ikey=$AI_IKEY # Deploy service -kubectl --namespace backend apply -f $K8S/package-0.yml +helm install $HELM_CHARTS/package/ \ + --set image.tag=0.1.0 \ + --set image.repository=package \ + --set dockerregistry=$ACR_SERVER \ + --namespace backend \ + --name package-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status package-v0.1.0 ``` ## Deploy the Workflow service diff --git a/deploymentARM.md b/deploymentARM.md index 4871457e..e93a832e 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -229,9 +229,6 @@ docker push $ACR_SERVER/package:0.1.0 Deploy the Package service ```bash -# Update deployment YAML with image tage -sed "s#image:#image: $ACR_SERVER/package:0.1.0#g" $K8S/package.yml > $K8S/package-0.yml - # Create secret # Note: Connection strings cannot be exported as outputs in ARM deployments export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') @@ -241,10 +238,15 @@ kubectl -n backend create \ --from-literal=appinsights-ikey=$AI_IKEY # Deploy service -kubectl --namespace backend apply -f $K8S/package-0.yml +helm install $HELM_CHARTS/package/ \ + --set image.tag=0.1.0 \ + --set image.repository=package \ + --set dockerregistry=$ACR_SERVER \ + --namespace backend \ + --name package-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status package-v0.1.0 ``` ## Deploy the Workflow service diff --git a/k8s/package.yml b/k8s/package.yml deleted file mode 100644 index 0b534248..00000000 --- a/k8s/package.yml +++ /dev/null @@ -1,65 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: package - labels: - app: package - version: 0.1.0 - bc: shipping - co: fabrikam -spec: - replicas: 1 - selector: - matchLabels: - app: package - template: - metadata: - labels: - app: package - version: 0.1.0 - bc: shipping - co: fabrikam - spec: - containers: - - name: &package-container_name package - image: - env: - - name: CONNECTION_STRING - valueFrom: - secretKeyRef: - name: package-secrets - key: mongodb-pwd - - name: COLLECTION_NAME - value: packages - - name: APPINSIGHTS_INSTRUMENTATIONKEY - valueFrom: - secretKeyRef: - name: package-secrets - key: appinsights-ikey - - name: LOG_LEVEL - value: error - - name: CONTAINER_NAME - value: *package-container_name - ports: - - containerPort: 80 - imagePullPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: package - labels: - app: package - bc: shipping - co: fabrikam -spec: - selector: - app: package - ports: - - name: http - port: 80 From d6e38ae15651d77213eb448b34551ab528ab734a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Mar 2019 19:29:53 +0000 Subject: [PATCH 084/246] Merged PR 673: bug fixes delivery bug fixes delivery remove fixed names and instances from delivery --- .../templates/delivery-service-experimental.yaml | 10 ++++++---- charts/delivery/templates/delivery-service.yaml | 10 ++++++---- deployment.md | 2 +- deploymentARM.md | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/charts/delivery/templates/delivery-service-experimental.yaml b/charts/delivery/templates/delivery-service-experimental.yaml index b4b4ca3f..52600d65 100644 --- a/charts/delivery/templates/delivery-service-experimental.yaml +++ b/charts/delivery/templates/delivery-service-experimental.yaml @@ -15,10 +15,12 @@ kind: Service metadata: name: delivery-experimental labels: - app.kubernetes.io/name: delivery + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - app.kubernetes.io/managed-by: azurepipelines helm.sh/chart: {{ include "delivery.chart" . }} spec: ports: @@ -26,6 +28,6 @@ spec: port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: delivery - app.kubernetes.io/instance: delivery-v0.1.0 + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} {{ end }} diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index fb234077..55e546b4 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -15,10 +15,12 @@ kind: Service metadata: name: delivery labels: - app.kubernetes.io/app: delivery + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - app.kubernetes.io/managed-by: azurepipelines helm.sh/chart: {{ include "delivery.chart" . }} spec: ports: @@ -26,6 +28,6 @@ spec: port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: delivery - app.kubernetes.io/instance: delivery-v0.1.0 + app.kubernetes.io/name: {{ template "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} {{ end }} diff --git a/deployment.md b/deployment.md index c3f77b23..4c12272a 100644 --- a/deployment.md +++ b/deployment.md @@ -227,7 +227,7 @@ helm install $HELM_CHARTS/delivery/ \ --set image.tag=0.1.0 \ --set image.repository=delivery \ --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DELIVERY_PRINCIPAL_ID \ + --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ diff --git a/deploymentARM.md b/deploymentARM.md index e93a832e..2167f910 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -193,7 +193,7 @@ helm install $HELM_CHARTS/delivery/ \ --set image.tag=0.1.0 \ --set image.repository=delivery \ --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DELIVERY_PRINCIPAL_ID \ + --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ From 3192969b66615bb9a75ca140c9c81dc5eb9dc41d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Mar 2019 19:30:28 +0000 Subject: [PATCH 085/246] Merged PR 674: bug fixes dronescheduler remove fixed names and instances from dronescheduler --- .../dronescheduler-service-experimental.yaml | 12 +++++++----- .../templates/dronescheduler-service-sxs.yaml | 2 +- .../templates/dronescheduler-service.yaml | 12 +++++++----- deployment.md | 2 +- deploymentARM.md | 2 +- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml b/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml index dcafda2c..9ba17ad6 100644 --- a/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml @@ -15,17 +15,19 @@ kind: Service metadata: name: dronescheduler-experimental labels: - app.kubernetes.io/name: dronescheduler + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - app.kubernetes.io/managed-by: azurepipelines - helm.sh/chart: {{ include "delivery.chart" . }} + helm.sh/chart: {{ include "dronescheduler.chart" . }} spec: ports: - name: http port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: dronescheduler - app.kubernetes.io/instance: dronescheduler-v0.1.0 + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} {{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml b/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml index e41c4dd7..59d80f37 100644 --- a/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml @@ -16,7 +16,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedronescheduler + app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "dronescheduler.chart" . }} spec: ports: diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index b9e3eb37..fd2bdd01 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -15,17 +15,19 @@ kind: Service metadata: name: dronescheduler labels: - app.kubernetes.io/app: dronescheduler + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - app.kubernetes.io/managed-by: azurepipelines - helm.sh/chart: {{ include "delivery.chart" . }} + helm.sh/chart: {{ include "dronescheduler.chart" . }} spec: ports: - name: http port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: dronescheduler - app.kubernetes.io/instance: dronescheduler-v0.1.0 + app.kubernetes.io/name: {{ template "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} {{ end }} diff --git a/deployment.md b/deployment.md index 4c12272a..4590dd86 100644 --- a/deployment.md +++ b/deployment.md @@ -506,7 +506,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set image.tag=0.1.0 \ --set image.repository=dronescheduler \ --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_ID \ + --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --namespace backend \ diff --git a/deploymentARM.md b/deploymentARM.md index 2167f910..aac1cd2e 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -409,7 +409,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set image.tag=0.1.0 \ --set image.repository=dronescheduler \ --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_ID \ + --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --namespace backend \ From 4eb03ee0d3f9917063c406e55ed1d97506bb829d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Mar 2019 22:02:51 +0000 Subject: [PATCH 086/246] Merged PR 677: change deployment to use helm for workflow change deployment to use helm for workflow solved: #8991 Related work items: #8991 --- .../workflow/templates/workflow-deploy.yaml | 55 ++++++++++++- .../templates}/workflow-identity.yaml | 14 +++- charts/workflow/values.yaml | 23 +++++- deployment.md | 30 ++++--- deploymentARM.md | 32 ++++---- k8s/workflow.yaml | 81 ------------------- 6 files changed, 115 insertions(+), 120 deletions(-) rename {k8s => charts/workflow/templates}/workflow-identity.yaml (72%) delete mode 100644 k8s/workflow.yaml diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index a47b1640..b7435399 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -1,3 +1,11 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Workflow +################################################################################################### apiVersion: apps/v1beta2 kind: Deployment metadata: @@ -10,6 +18,7 @@ metadata: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "workflow.chart" . }} + aadpodidbinding: workflow annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: @@ -28,9 +37,51 @@ spec: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "workflow.chart" . }} + aadpodidbinding: workflow spec: securityContext: fsGroup: 1 containers: - - name: workflowservice - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} \ No newline at end of file + - name: fabrikam-workflow + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: workflow + mountPath: /kvmnt + readOnly: true + env: + - name: SERVICE_URI_DELIVERY + value: {{ .Values.serviceuri.delivery }} + - name: SERVICE_URI_DRONE + value: {{ .Values.serviceuri.drone }} + - name: SERVICE_URI_PACKAGE + value: {{ .Values.serviceuri.package }} + - name: SERVICEREQUEST__MAXRETRIES + value: "{{ .Values.servicerequest.maxretries }}" + - name: SERVICEREQUEST__CIRCUITBREAKERTHRESHOLD + value: "{{ .Values.servicerequest.circuitbreakerthreshold }}" + - name: SERVICEREQUEST__CIRCUITBREAKERSAMPLINGPERIODSECONDS + value: "{{ .Values.servicerequest.circuitbreakersamplingperiodseconds }}" + - name: SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT + value: "{{ .Values.servicerequest.circuitbreakerminimumthroughput }}" + - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATION + value: "{{ .Values.servicerequest.circuitbreakerbreakduration }}" + - name: SERVICEREQUEST__MAXBULKHEADSIZE + value: "{{ .Values.servicerequest.maxbulkheadsize }}" + - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE + value: "{{ .Values.servicerequest.maxbulkheadqueuesize }}" + - name: no_proxy + value: 169.254.169.254 + volumes: + - name: workflow + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: {{ .Values.keyvault.name }} + keyvaultobjectnames: QueueName;QueueEndpoint;ApplicationInsights-InstrumentationKey + keyvaultobjecttypes: secret;secret;secret + keyvaultobjectversions: "" + resourcegroup: {{ .Values.keyvault.resourcegroup }} + subscriptionid: {{ .Values.keyvault.subscriptionid }} + tenantid: {{ .Values.keyvault.tenantid }} diff --git a/k8s/workflow-identity.yaml b/charts/workflow/templates/workflow-identity.yaml similarity index 72% rename from k8s/workflow-identity.yaml rename to charts/workflow/templates/workflow-identity.yaml index 752ee8f5..c9b4836c 100644 --- a/k8s/workflow-identity.yaml +++ b/charts/workflow/templates/workflow-identity.yaml @@ -4,21 +4,27 @@ # ------------------------------------------------------------ ################################################################################################### -# Delivery service identityß +# Workflow service identity ################################################################################################### apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: name: workflow-identity + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "0" spec: type: 0 - ResourceID: "identityResourceId" - ClientID: "identityClientid" + ResourceID: {{ .Values.identity.resourceid }} + ClientID: {{ .Values.identity.clientid }} --- apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: name: workflow-identity-binding + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "1" spec: AzureIdentity: workflow-identity - Selector: workflow \ No newline at end of file + Selector: workflow diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 6882f354..0668cc7e 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -1,8 +1,29 @@ # Default values for workflow. replicaCount: 1 +dockerregistry: +identity: + clientid: + resourceid: image: repository: tag: pullPolicy: IfNotPresent -dockerregistry: reason: unknown +serviceuri: + delivery: http://delivery/api/Deliveries/ + drone: http://dronescheduler/api/DroneDeliveries/ + package: http://package/api/packages/ +servicerequest: + maxretries: 3 + circuitbreakerthreshold: 0.5 + circuitbreakersamplingperiodseconds: 5 + circuitbreakerminimumthroughput: 20 + circuitbreakerbreakduration: 30 + maxbulkheadsize: 100 + maxbulkheadqueuesize: 25 +keyvault: + name: + resourcegroup: + subscriptionid: + tenantid: + diff --git a/deployment.md b/deployment.md index 4590dd86..c330af20 100644 --- a/deployment.md +++ b/deployment.md @@ -5,7 +5,7 @@ - Azure subscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) -- [Helm](https://docs.helm.sh/using_helm/#installing-helm) +- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) > Note: in linux systems, it is possible to run the docker command without prefacing > with sudo. For more information, please refer to [the Post-installation steps @@ -354,29 +354,27 @@ az role assignment create --role Contributor --assignee $WORKFLOW_PRINCIPAL_ID - # Allow the cluster to manage the identity to assign to pods az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $WORKFLOW_PRINCIPAL_RESOURCE_ID - -# Deploy the identity resources -cat $K8S/workflow-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $WORKFLOW_PRINCIPAL_CLIENT_ID#g" > $K8S/workflow-identity-0.yaml -kubectl apply -f $K8S/workflow-identity-0.yaml ``` Deploy the Workflow service: ```bash -# Update the image tag and config values in the deployment YAML -sed "s#image:#image: $ACR_SERVER/workflow:0.1.0#g" $K8S/workflow.yaml | \ - sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ - sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ - sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ - sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $WORKFLOW_KEYVAULT_NAME#g" > $K8S/workflow-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/workflow-0.yaml +helm install $HELM_CHARTS/workflow/ \ + --set image.tag=0.1.0 \ + --set image.repository=workflow \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ + --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ + --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ + --set keyvault.resourcegroup=$RESOURCE_GROUP \ + --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ + --set keyvault.tenantid=$TENANT_ID \ + --namespace backend \ + --name workflow-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status workflow-v0.1.0 ``` ## Deploy the Ingestion service diff --git a/deploymentARM.md b/deploymentARM.md index aac1cd2e..24611c21 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -5,7 +5,7 @@ - Azure suscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) -- [Helm](https://docs.helm.sh/using_helm/#installing-helm) +- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) - [JQ](https://stedolan.github.io/jq/download/) > Note: in linux systems, it is possible to run the docker command without prefacing @@ -146,6 +146,8 @@ kubectl apply -f $K8S/k8s-rbac-ai.yaml Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol +Note: the tested nmi version was 1.4. It enables namespaced pod identity. + ```bash # setup AAD pod identity kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml @@ -276,29 +278,27 @@ Create and set up pod identity # Extract outputs from deployment export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) - -# Deploy the identity resources -cat $K8S/workflow-identity.yaml | \ - sed "s#ResourceID: \"identityResourceId\"#ResourceID: $WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#ClientID: \"identityClientid\"#ClientID: $WORKFLOW_PRINCIPAL_CLIENT_ID#g" > $K8S/workflow-identity-0.yaml -kubectl apply -f $K8S/workflow-identity-0.yaml ``` Deploy the Workflow service: ```bash -# Update the image tag and config values in the deployment YAML -sed "s#image:#image: $ACR_SERVER/workflow:0.1.0#g" $K8S/workflow.yaml | \ - sed "s#resourcegroup: \"keyVaultResourceGroup\"#resourcegroup: $RESOURCE_GROUP#g" | \ - sed "s#subscriptionid: \"keyVaultSubscriptionId\"#subscriptionid: $SUBSCRIPTION_ID#g" | \ - sed "s#tenantid: \"keyVaultTenantId\"#tenantid: $TENANT_ID#g" | \ - sed "s#keyvaultname: \"keyVaultName\"#keyvaultname: $WORKFLOW_KEYVAULT_NAME#g" > $K8S/workflow-0.yaml - # Deploy the service -kubectl --namespace backend apply -f $K8S/workflow-0.yaml +helm install $HELM_CHARTS/workflow/ \ + --set image.tag=0.1.0 \ + --set image.repository=workflow \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ + --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ + --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ + --set keyvault.resourcegroup=$RESOURCE_GROUP \ + --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ + --set keyvault.tenantid=$TENANT_ID \ + --namespace backend \ + --name workflow-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status workflow-v0.1.0 ``` ## Deploy the Ingestion service diff --git a/k8s/workflow.yaml b/k8s/workflow.yaml deleted file mode 100644 index 99be3705..00000000 --- a/k8s/workflow.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Workflow service -################################################################################################### -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: workflow - labels: # In API version apps/v1beta2, .metadata.labels no longer default to .spec.template.metadata.labels. - app: workflow - version: 0.1.0 - bc: shipping - co: fabrikam - aadpodidbinding: workflow -spec: - replicas: 1 - selector: # In API version apps/v1beta2, .spec.selector no longer default to .spec.template.metadata.labels. - matchLabels: # spec.selector is immutable after creation of the Deployment in apps/v1beta2. - app: workflow - template: # A Deployment may terminate Pods whose labels match the selector if their template is different from .spec.template - metadata: - labels: - app: workflow - version: 0.1.0 - bc: shipping - co: fabrikam - aadpodidbinding: workflow - annotations: - team: workflowservice - spec: - securityContext: - fsGroup: 1 - containers: - - name: workflow - image: - volumeMounts: - - name: workflow - mountPath: /kvmnt - readOnly: true - env: - - name: CONFIGURATION_FOLDER - value: /kvmnt - - name: SERVICE_URI_DELIVERY - value: http://delivery/api/Deliveries/ - - name: SERVICE_URI_DRONE - value: http://dronescheduler/api/DroneDeliveries/ - - name: SERVICE_URI_PACKAGE - value: http://package/api/packages/ - - name: SERVICEREQUEST__MAXRETRIES - value: "3" - - name: SERVICEREQUEST__CIRCUITBREAKERTHRESHOLD - value: "0.5" - - name: SERVICEREQUEST__CIRCUITBREAKERSAMPLINGPERIODSECONDS - value: "5" - - name: SERVICEREQUEST__CIRCUITBREAKERMINIMUMTHROUGHPUT - value: "20" - - name: SERVICEREQUEST__CIRCUITBREAKERBREAKDURATIONSECONDS - value: "30" - - name: SERVICEREQUEST__MAXBULKHEADSIZE - value: "100" - - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE - value: "25" - - name: no_proxy - value: 169.254.169.254 - volumes: - - name: workflow - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "keyVaultName" - keyvaultobjectnames: QueueName;QueueEndpoint;ApplicationInsights-InstrumentationKey - keyvaultobjecttypes: secret;secret;secret - keyvaultobjectversions: "" - resourcegroup: "keyVaultResourceGroup" - subscriptionid: "keyVaultSubscriptionId" - tenantid: "keyVaultTenantId" \ No newline at end of file From ee707b492d0199161baf208977bdd8666bf475b7 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 11 Mar 2019 12:38:28 +0000 Subject: [PATCH 087/246] Merged PR 675: use access policies for workflow access to sb instead of MSI use access policies for workflow access to sb instead of MSI Related work items: #8942 --- azuredeploy.json | 64 +++++++++++-------- .../workflow/templates/workflow-deploy.yaml | 4 +- deployment.md | 11 +++- deploymentARM.md | 3 + .../WorkflowService.cs | 13 +++- .../WorkflowServiceOptions.cs | 4 ++ 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index bc1a6ed3..50364ebb 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -110,18 +110,8 @@ }, "kubernetesVersion": { "type": "string", - "defaultValue": "1.12.4", - "allowedValues": [ - "1.9.10", - "1.9.11", - "1.10.9", - "1.10.12", - "1.11.5", - "1.11.6", - "1.12.4" - ], "metadata": { - "description": "The version of Kubernetes." + "description": "The version of Kubernetes. It must be supported in the target location." } }, "deliveryRedisStorageType": { @@ -200,10 +190,9 @@ "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(resourceGroup().name, '-ds-kv')]", "workflowKeyVaultName": "[concat(resourceGroup().name, '-wf-kv')]", + "workflowServiceAccessKey": "WorkflowServiceAccessKey", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", - "contributorRoleObjectId": "b24988ac-6180-42a0-ab88-20f7382dd24c", - "contributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('contributorRoleObjectId'))]", "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]" }, @@ -384,6 +373,19 @@ "dependsOn": [ "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" ] + }, + { + "name": "[variables('workflowServiceAccessKey')]", + "type": "AuthorizationRules", + "apiVersion": "2017-04-01", + "properties": { + "rights": [ + "Listen" + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + ] } ] }, @@ -574,6 +576,29 @@ "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" ] }, + { + "type": "secrets", + "name": "QueueAccessPolicyName", + "apiVersion": "2015-06-01", + "properties": { + "value": "[variables('workflowServiceAccessKey')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + ] + }, + { + "type": "secrets", + "name": "QueueAccessPolicyKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "[listkeys(resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('ingestionSBNamespace'), variables('workflowServiceAccessKey')), '2017-04-01').primaryKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", + "[resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('ingestionSBNamespace'), variables('workflowServiceAccessKey'))]" + ] + }, { "type": "secrets", "name": "ApplicationInsights-InstrumentationKey", @@ -614,19 +639,6 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]" ] }, - { - "type": "Microsoft.ServiceBus/namespaces/providers/roleAssignments", - "name": "[concat(variables('ingestionSBNamespace'), '/Microsoft.Authorization/', guid('workflow-sb', resourceGroup().id))]", - "apiVersion": "2017-05-01", - "properties": { - "roleDefinitionId": "[variables('contributorRoleId')]", - "principalId": "[parameters('workflowPrincipalId')]", - "scope": "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" - ] - }, { "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", "name": "[concat(variables('workflowKeyVaultName'), '/Microsoft.Authorization/', guid('workflow-kv', resourceGroup().id))]", diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index b7435399..7cf769db 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -79,8 +79,8 @@ spec: options: usepodidentity: "true" keyvaultname: {{ .Values.keyvault.name }} - keyvaultobjectnames: QueueName;QueueEndpoint;ApplicationInsights-InstrumentationKey - keyvaultobjecttypes: secret;secret;secret + keyvaultobjectnames: QueueName;QueueEndpoint;QueueAccessPolicyName;QueueAccessPolicyKey;ApplicationInsights-InstrumentationKey + keyvaultobjecttypes: secret;secret;secret;secret;secret keyvaultobjectversions: "" resourcegroup: {{ .Values.keyvault.resourcegroup }} subscriptionid: {{ .Values.keyvault.subscriptionid }} diff --git a/deployment.md b/deployment.md index c330af20..ae45dc96 100644 --- a/deployment.md +++ b/deployment.md @@ -300,9 +300,15 @@ az servicebus queue create --name $INGESTION_QUEUE_NAME \ --namespace-name $INGESTION_QUEUE_NAMESPACE \ --resource-group $RESOURCE_GROUP +az servicebus namespace authorization-rule create --namespace-name $INGESTION_QUEUE_NAMESPACE \ + --name WorkflowServiceAccessKey \ + --resource-group $RESOURCE_GROUP \ + --rights Listen + export INGESTION_QUEUE_ID=$(az servicebus queue show --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_QUEUE_NAME --query "id" --output tsv) export INGESTION_QUEUE_NAMESPACE_ID=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "id" --output tsv) export INGESTION_QUEUE_NAMESPACE_ENDPOINT=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "serviceBusEndpoint" --output tsv) +export WORKFLOW_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name WorkflowServiceAccessKey --query primaryKey -o tsv) ``` Build the workflow service @@ -326,6 +332,8 @@ export WORKFLOW_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-workflow-kv" az keyvault create --name $WORKFLOW_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueName --value ${INGESTION_QUEUE_NAME} az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueEndpoint --value ${INGESTION_QUEUE_NAMESPACE_ENDPOINT} +az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueAccessPolicyName --value WorkflowServiceAccessKey +az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueAccessPolicyKey --value ${WORKFLOW_ACCESS_KEY_VALUE} az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name ApplicationInsights-InstrumentationKey --value ${AI_IKEY} export WORKFLOW_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "id" --output tsv) @@ -349,9 +357,6 @@ export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURC az role assignment create --role Reader --assignee $WORKFLOW_PRINCIPAL_ID --scope $WORKFLOW_KEYVAULT_ID az keyvault set-policy --name $WORKFLOW_KEYVAULT_NAME --secret-permissions get list --spn $WORKFLOW_PRINCIPAL_CLIENT_ID -# Grant the identity access to the ingestion queue -az role assignment create --role Contributor --assignee $WORKFLOW_PRINCIPAL_ID --scope $INGESTION_QUEUE_NAMESPACE_ID - # Allow the cluster to manage the identity to assign to pods az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $WORKFLOW_PRINCIPAL_RESOURCE_ID ``` diff --git a/deploymentARM.md b/deploymentARM.md index 24611c21..f1eb9a75 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -80,10 +80,13 @@ until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do ech until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done # Deploy all other resources +# The version of kubernetes must be supported in the target region +export KUBERNETES_VERSION='1.12.6' az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ servicePrincipalId=${SP_OBJECT_ID} \ + kubernetesVersion=${KUBERNETES_VERSION} \ sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ deliveryIdName=${DELIVERY_ID_NAME} \ deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs index e1ebd215..0adba19e 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowService.cs @@ -10,7 +10,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.ServiceBus; -using Microsoft.Azure.ServiceBus.Primitives; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -46,8 +45,16 @@ public WorkflowService(IOptions options, ILogger options) { - TokenProvider tokenProvider = TokenProvider.CreateManagedServiceIdentityTokenProvider(); - return new QueueClient(options.Value.QueueEndpoint, options.Value.QueueName, tokenProvider, receiveMode: ReceiveMode.PeekLock, transportType: TransportType.Amqp) + var connectionStringBuilder = new ServiceBusConnectionStringBuilder + { + Endpoint = options.Value.QueueEndpoint, + EntityPath = options.Value.QueueName, + SasKeyName = options.Value.QueueAccessPolicyName, + SasKey = options.Value.QueueAccessPolicyKey, + TransportType = TransportType.Amqp + }; + + return new QueueClient(connectionStringBuilder) { PrefetchCount = options.Value.PrefetchCount }; diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs index c9bebec8..8eb21164 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs @@ -17,6 +17,10 @@ public WorkflowServiceOptions() public string QueueName { get; set; } + public string QueueAccessPolicyName { get; set; } + + public string QueueAccessPolicyKey { get; set; } + public int MaxConcurrency { get; internal set; } public int PrefetchCount { get; internal set; } From f1868daa5d91c984359c6a3862967ec6b57cdcb2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 16:28:31 +0000 Subject: [PATCH 088/246] Merged PR 682: remove unused secrets script file remove unused secrets script file --- src/shipping/package/build/create-secrets.sh | 15 --------------- src/shipping/package/package-service.md | 8 -------- 2 files changed, 23 deletions(-) delete mode 100644 src/shipping/package/build/create-secrets.sh diff --git a/src/shipping/package/build/create-secrets.sh b/src/shipping/package/build/create-secrets.sh deleted file mode 100644 index c0a65d05..00000000 --- a/src/shipping/package/build/create-secrets.sh +++ /dev/null @@ -1,15 +0,0 @@ -#! /bin/bash - -CosmosDB=$1 -ResourceGroup=$2 - -# Get the CosmosDB connection string -ConnectionString=`az cosmosdb list-connection-strings --name $CosmosDB --resource-group $ResourceGroup --query "connectionStrings[0].connectionString"` - -# Trim quotes -ConnectionString="${ConnectionString%\"}" -ConnectionString="${ConnectionString#\"}" - -# Create the app secret -echo 'Creating k8s secrets' -kubectl create secret generic package-secrets --from-literal=mongodb-pwd=$ConnectionString \ No newline at end of file diff --git a/src/shipping/package/package-service.md b/src/shipping/package/package-service.md index 62bcdf91..b269ca88 100644 --- a/src/shipping/package/package-service.md +++ b/src/shipping/package/package-service.md @@ -13,11 +13,3 @@ ``` docker build -f ./Dockerfile -t /package-service . ``` - -## Provision database and create secrets - -1. In Azure, create a Cosmos DB database with MongoDB API -2. Install Azure CLI 2.0 -3. Run `az login` -4. Run create-secrets.sh - From d56e472153b70b6979199de76a64ea75932803e8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 16:36:43 +0000 Subject: [PATCH 089/246] Merged PR 685: add missing secrets in package chart add missing secrets in package chart --- charts/package/templates/package-secrets.yaml | 16 ++++++++++++++++ deployment.md | 6 ++---- deploymentARM.md | 6 ++---- 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 charts/package/templates/package-secrets.yaml diff --git a/charts/package/templates/package-secrets.yaml b/charts/package/templates/package-secrets.yaml new file mode 100644 index 00000000..8c44cb70 --- /dev/null +++ b/charts/package/templates/package-secrets.yaml @@ -0,0 +1,16 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package secrets +################################################################################################### +kind: Secret +apiVersion: v1 +metadata: + name: package-secrets +type: Opaque +data: + appinsights-ikey: {{ .Values.secrets.appinsights.ikey | b64enc }} + mongodb-pwd: {{ .Values.secrets.mongo.pwd | b64enc }} diff --git a/deployment.md b/deployment.md index ae45dc96..c8b0134d 100644 --- a/deployment.md +++ b/deployment.md @@ -266,15 +266,13 @@ Deploy the Package service ```bash # Create secret export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') -kubectl -n backend create \ - secret generic package-secrets \ - --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION \ - --from-literal=appinsights-ikey=$AI_IKEY # Deploy service helm install $HELM_CHARTS/package/ \ --set image.tag=0.1.0 \ --set image.repository=package \ + --set secrets.appinsights.ikey=$AI_IKEY \ + --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ --set dockerregistry=$ACR_SERVER \ --namespace backend \ --name package-v0.1.0 diff --git a/deploymentARM.md b/deploymentARM.md index f1eb9a75..3d65935d 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -237,15 +237,13 @@ Deploy the Package service # Create secret # Note: Connection strings cannot be exported as outputs in ARM deployments export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') -kubectl -n backend create \ - secret generic package-secrets \ - --from-literal=mongodb-pwd=$COSMOSDB_CONNECTION \ - --from-literal=appinsights-ikey=$AI_IKEY # Deploy service helm install $HELM_CHARTS/package/ \ --set image.tag=0.1.0 \ --set image.repository=package \ + --set secrets.appinsights.ikey=$AI_IKEY \ + --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ --set dockerregistry=$ACR_SERVER \ --namespace backend \ --name package-v0.1.0 From 51e217c3f032517b2a31b1bcc2d094f27882c0af Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 17:02:48 +0000 Subject: [PATCH 090/246] Merged PR 684: update docker-compose support in package update docker-compose support in package - remove unused docker-compose files - update dev docker-compose file to work with new dockerfile - streamline dev env --- src/shipping/package/app/initializer.ts | 23 +++++++++++++++---- src/shipping/package/build/dev.dockerfile | 5 ---- .../package/build/docker-compose.ci.build.yml | 11 --------- ...er-compose.yml => docker-compose.dev.yaml} | 15 ++++++++---- src/shipping/package/down.sh | 4 ++-- src/shipping/package/package-service.md | 14 ++++------- src/shipping/package/up.sh | 2 +- 7 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 src/shipping/package/build/dev.dockerfile delete mode 100644 src/shipping/package/build/docker-compose.ci.build.yml rename src/shipping/package/{build/docker-compose.yml => docker-compose.dev.yaml} (61%) mode change 100644 => 100755 src/shipping/package/down.sh mode change 100644 => 100755 src/shipping/package/up.sh diff --git a/src/shipping/package/app/initializer.ts b/src/shipping/package/app/initializer.ts index f1231687..bd9e6c4c 100644 --- a/src/shipping/package/app/initializer.ts +++ b/src/shipping/package/app/initializer.ts @@ -34,10 +34,25 @@ export class PackageServiceInitializer } private static initAppInsights(cloudRole = "package") { - appInsights.setup(); - appInsights.defaultClient.context.tags[appInsights.defaultClient.context.keys.cloudRole] = cloudRole; - appInsights.start(); - console.log('Application Insights started'); + if (!process.env.APPINSIGHTS_INSTRUMENTATIONKEY && + process.env.NODE_ENV === 'development') { + const logger = console; + process.stderr.write('Skipping app insights setup - in development mode with no ikey set\n'); + appInsights. + defaultClient = { + trackEvent: logger.log.bind(console, 'trackEvent'), + trackException: logger.error.bind(console, 'trackException'), + trackMetric: logger.log.bind(console, 'trackMetric'), + }; + } else if (process.env.APPINSIGHTS_INSTRUMENTATIONKEY) { + appInsights.setup(); + appInsights.defaultClient.context.tags[appInsights.defaultClient.context.keys.cloudRole] = cloudRole; + process.stdout.write('App insights setup - configuring client\n'); + appInsights.start(); + process.stdout.write('Application Insights started'); + } else { + throw new Error('No app insights setup. A key must be specified in non-development environments.'); + } } } diff --git a/src/shipping/package/build/dev.dockerfile b/src/shipping/package/build/dev.dockerfile deleted file mode 100644 index 100065a5..00000000 --- a/src/shipping/package/build/dev.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM node:8.0.0 - -RUN apt-get update && \ - apt-get install -y tmux supervisor inotify-tools && \ - rm -rf /var/lib/apt/lists; rm /tmp/*; apt-get autoremove -y diff --git a/src/shipping/package/build/docker-compose.ci.build.yml b/src/shipping/package/build/docker-compose.ci.build.yml deleted file mode 100644 index 53936632..00000000 --- a/src/shipping/package/build/docker-compose.ci.build.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Build package service -version: '3' - -services: - ci-build: - image: node:8.0.0 - volumes: - - ..:/src - working_dir: /src - command: /bin/bash -c "npm install && npm run build" - diff --git a/src/shipping/package/build/docker-compose.yml b/src/shipping/package/docker-compose.dev.yaml similarity index 61% rename from src/shipping/package/build/docker-compose.yml rename to src/shipping/package/docker-compose.dev.yaml index 6201d572..dae26097 100644 --- a/src/shipping/package/build/docker-compose.yml +++ b/src/shipping/package/docker-compose.dev.yaml @@ -3,20 +3,25 @@ services: app: build: context: . - dockerfile: dev.dockerfile + dockerfile: Dockerfile + depends_on: + - mongo + restart: on-failure command: bash ports: - "7080:80" - "7443:443" - volumes: - - ..:/app tty: true stdin_open: true - working_dir: /app + environment: + - NODE_ENV=development + - CONNECTION_STRING=mongodb://packagedb:27017/local + - COLLECTION_NAME=packages + - LOG_LEVEL = "debug" networks: dronedelivery: aliases: - - packageservice + - package mongo: image: mongo:3.5.6 ports: diff --git a/src/shipping/package/down.sh b/src/shipping/package/down.sh old mode 100644 new mode 100755 index ec8aafb1..06fbf71b --- a/src/shipping/package/down.sh +++ b/src/shipping/package/down.sh @@ -1,6 +1,6 @@ -docker-compose -p drone-package -f ./build/docker-compose.yml down +docker-compose -p drone-package -f ./docker-compose.dev.yaml down # cleanup the network if there are not containers using it if [ "$(docker network inspect dronedelivery --format "{{range .Containers}}T{{end}}")" == "" ]; then docker network rm dronedelivery -fi \ No newline at end of file +fi diff --git a/src/shipping/package/package-service.md b/src/shipping/package/package-service.md index b269ca88..067aace5 100644 --- a/src/shipping/package/package-service.md +++ b/src/shipping/package/package-service.md @@ -2,14 +2,10 @@ ## Create local development environment -1. Install Docker +1. Install Docker and docker-compose 2. From a bash CLI navigate to the project root and type `./up.sh` -3. From the app environment type `npm install` -4. To run the app in the local dev environment, type `npm start` -5. The app listens on port 80. In the dev environment, this is mapped to localhost:7080 +3. try ```curl -X PUT --header 'Accept: application/json' 'http://localhost:7080/api/packages/42'``` -## Build docker image - -``` -docker build -f ./Dockerfile -t /package-service . -``` +> Known issue: +> ```'failed to connect to server [packagedb:27017] on first connect [MongoError: connect ECONNREFUSED 172.24.0.2:27017]'``` +> First time the package service starts, it might fail connecting to mongo. Package would automatically restart if needed. diff --git a/src/shipping/package/up.sh b/src/shipping/package/up.sh old mode 100644 new mode 100755 index d6e969c2..8ba8275f --- a/src/shipping/package/up.sh +++ b/src/shipping/package/up.sh @@ -8,6 +8,6 @@ if ! docker network ls | grep -q dronedelivery; then docker network create dronedelivery fi -docker-compose -p drone-package -f ./build/docker-compose.yml up -d +docker-compose -p drone-package -f ./docker-compose.dev.yaml up -d docker attach dronepackage_app_1 From cbd0bc08e7d3d9b92af2ad21a1457c86f761caba Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 17:28:42 +0000 Subject: [PATCH 091/246] Merged PR 678: change deployment to use helm for ingestion change deployment to use helm for ingestion create helm chart for ingestion solved: #8992 #8993 Related work items: #8992, #8993 --- charts/ingestion/Chart.yaml | 4 + charts/ingestion/templates/NOTES.txt | 10 + charts/ingestion/templates/_helpers.tpl | 32 +++ .../ingestion/templates/ingestion-deploy.yaml | 193 ++++++++---------- .../templates/ingestion-ingress.yaml | 32 +++ .../ingestion-secret-ingress-queue.yaml | 11 + .../ingestion-secret-ingress-tls.yaml | 10 + .../ingestion-service-experimental.yaml | 33 +++ .../templates/ingestion-service-sxs.yaml | 28 +++ .../templates/ingestion-service.yaml | 33 +++ charts/ingestion/values.yaml | 8 + deployment.md | 41 ++-- deploymentARM.md | 41 ++-- 13 files changed, 324 insertions(+), 152 deletions(-) create mode 100644 charts/ingestion/Chart.yaml create mode 100644 charts/ingestion/templates/NOTES.txt create mode 100644 charts/ingestion/templates/_helpers.tpl rename k8s/ingestion.yaml => charts/ingestion/templates/ingestion-deploy.yaml (56%) create mode 100644 charts/ingestion/templates/ingestion-ingress.yaml create mode 100644 charts/ingestion/templates/ingestion-secret-ingress-queue.yaml create mode 100644 charts/ingestion/templates/ingestion-secret-ingress-tls.yaml create mode 100644 charts/ingestion/templates/ingestion-service-experimental.yaml create mode 100644 charts/ingestion/templates/ingestion-service-sxs.yaml create mode 100644 charts/ingestion/templates/ingestion-service.yaml create mode 100644 charts/ingestion/values.yaml diff --git a/charts/ingestion/Chart.yaml b/charts/ingestion/Chart.yaml new file mode 100644 index 00000000..aa6d03d4 --- /dev/null +++ b/charts/ingestion/Chart.yaml @@ -0,0 +1,4 @@ +name: ingestion +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Ingestion Service diff --git a/charts/ingestion/templates/NOTES.txt b/charts/ingestion/templates/NOTES.txt new file mode 100644 index 00000000..c5dbfc98 --- /dev/null +++ b/charts/ingestion/templates/NOTES.txt @@ -0,0 +1,10 @@ +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +All the objects were created in the namespace {{ .Values.namespace }} + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} + $ helm get {{ .Release.Name }} diff --git a/charts/ingestion/templates/_helpers.tpl b/charts/ingestion/templates/_helpers.tpl new file mode 100644 index 00000000..fb15086a --- /dev/null +++ b/charts/ingestion/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ingestion.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ingestion.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ingestion.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/k8s/ingestion.yaml b/charts/ingestion/templates/ingestion-deploy.yaml similarity index 56% rename from k8s/ingestion.yaml rename to charts/ingestion/templates/ingestion-deploy.yaml index 4e7bd357..eac51d7f 100644 --- a/k8s/ingestion.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -1,110 +1,83 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################## -# Ingestion service -################################################################################################## -apiVersion: v1 -kind: Service -metadata: - name: ingestion - labels: - app: ingestion - bc: shipping - co: fabrikam -spec: - ports: - - port: 80 - name: http - selector: - app: ingestion - type: - ClusterIP ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ingestion-ingress -spec: - tls: - - hosts: - - ingestion-host-name - secretName: ingestion-ingress-tls - rules: - - host: ingestion-host-name - http: - paths: - - path: / - backend: - serviceName: ingestion - servicePort: 80 ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: &ingestion-container_name ingestion - labels: - app: ingestion - version: 0.1.0 - bc: shipping - co: fabrikam -spec: - replicas: 1 - template: - metadata: - labels: - app: ingestion - version: 0.1.0 - bc: shipping - co: fabrikam - spec: - containers: - - name: ingestion - image: - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 80 - livenessProbe: - httpGet: - path: /actuator/health - port: 80 - initialDelaySeconds: 30 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /api/probe - port: 80 - initialDelaySeconds: 30 - periodSeconds: 20 - env: - - name: QUEUE_NAMESPACE - valueFrom: - secretKeyRef: - name: ingestion-secrets - key: queue_namespace - - name: QUEUE_NAME - valueFrom: - secretKeyRef: - name: ingestion-secrets - key: queue_name - - name: QUEUE_KEYNAME - valueFrom: - secretKeyRef: - name: ingestion-secrets - key: queue_keyname - - name: QUEUE_KEYVALUE - valueFrom: - secretKeyRef: - name: ingestion-secrets - key: queue_keyvalue - - name: APPINSIGHTS_INSTRUMENTATIONKEY - valueFrom: - secretKeyRef: - name: ingestion-secrets - key: appinsights-ikey - - name: CONTAINER_NAME - value: *ingestion-container_name - +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################## +# Ingestion +################################################################################################## +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "ingestion.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} + annotations: + kubernetes.io/change-cause: {{ .Values.reason }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} + spec: + containers: + - name: &ingestion-container_name fabrikam-ingestion + image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + httpGet: + path: /actuator/health + port: 80 + initialDelaySeconds: 30 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /api/probe + port: 80 + initialDelaySeconds: 30 + periodSeconds: 20 + env: + - name: QUEUE_NAMESPACE + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: queue_namespace + - name: QUEUE_NAME + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: queue_name + - name: QUEUE_KEYNAME + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: queue_keyname + - name: QUEUE_KEYVALUE + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: queue_keyvalue + - name: APPINSIGHTS_INSTRUMENTATIONKEY + valueFrom: + secretKeyRef: + name: ingestion-secrets + key: appinsights-ikey + - name: CONTAINER_NAME + value: *ingestion-container_name diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml new file mode 100644 index 00000000..a22d79a1 --- /dev/null +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -0,0 +1,32 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingestion ingress +################################################################################################### + +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingestion-ingress +spec: + tls: + {{- range .Values.ingress.hosts }} + {{- if .tls }} + - hosts: + - {{ .name }} + secretName: {{ .tlsSecretName }} + {{ end }} + {{ end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + - path: {{ default "/" .path }} + backend: + serviceName: "{{ .serviceName }}" + servicePort: http + {{ end }} diff --git a/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml b/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml new file mode 100644 index 00000000..d7e6bbd9 --- /dev/null +++ b/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml @@ -0,0 +1,11 @@ +kind: Secret +apiVersion: v1 +metadata: + name: ingestion-secrets +type: Opaque +data: + appinsights-ikey: {{ .Values.secrets.appinsights.ikey | b64enc }} + queue_keyname: {{ .Values.secrets.queue.keyname | b64enc }} + queue_keyvalue: {{ .Values.secrets.queue.keyvalue | b64enc }} + queue_name: {{ .Values.secrets.queue.name | b64enc }} + queue_namespace: {{ .Values.secrets.queue.namespace | b64enc }} diff --git a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml new file mode 100644 index 00000000..e95393b6 --- /dev/null +++ b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml @@ -0,0 +1,10 @@ +{{- range .Values.ingress.tls.secrets }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +{{ end }} diff --git a/charts/ingestion/templates/ingestion-service-experimental.yaml b/charts/ingestion/templates/ingestion-service-experimental.yaml new file mode 100644 index 00000000..4c24b666 --- /dev/null +++ b/charts/ingestion/templates/ingestion-service-experimental.yaml @@ -0,0 +1,33 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingestion service experimental +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "ingestion-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: ingestion-experimental + labels: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{ end }} diff --git a/charts/ingestion/templates/ingestion-service-sxs.yaml b/charts/ingestion/templates/ingestion-service-sxs.yaml new file mode 100644 index 00000000..36d2ae0b --- /dev/null +++ b/charts/ingestion/templates/ingestion-service-sxs.yaml @@ -0,0 +1,28 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingestion service sxs +################################################################################################### +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ingestion.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml new file mode 100644 index 00000000..4527ac4d --- /dev/null +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -0,0 +1,33 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingestion service +################################################################################################### + +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +{{ if (eq .Release.Name "ingestion-v0.1.0") }} +apiVersion: v1 +kind: Service +metadata: + name: ingestion + labels: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: azurepipelines + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ template "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{ end }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml new file mode 100644 index 00000000..e2060990 --- /dev/null +++ b/charts/ingestion/values.yaml @@ -0,0 +1,8 @@ +# Default values for ingestion. +replicaCount: 1 +dockerregistry: +image: + repository: + tag: + pullPolicy: IfNotPresent +reason: unknown diff --git a/deployment.md b/deployment.md index c8b0134d..696d9a7e 100644 --- a/deployment.md +++ b/deployment.md @@ -419,6 +419,7 @@ until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-contr export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) +INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # Create a self-signed certificate for TLS openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ @@ -426,30 +427,28 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ingestion-ingress-tls.key \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" -kubectl create secret tls ingestion-ingress-tls \ - --namespace backend \ - --key ingestion-ingress-tls.key \ - --cert ingestion-ingress-tls.crt - -# Update deployment YAML with image tag and the fqdn -cat $K8S/ingestion.yaml | \ - sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" | \ - sed "s#ingestion-host-name#$EXTERNAL_INGEST_FQDN#g" \ - > $K8S/ingestion-0.yaml - -# Create secret -kubectl -n backend create secret generic ingestion-secrets \ - --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ - --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ - --from-literal=queue_keyname=IngestionServiceAccessKey \ - --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ - --from-literal=appinsights-ikey=${AI_IKEY} - # Deploy service -kubectl --namespace backend apply -f $K8S/ingestion-0.yaml +helm install $HELM_CHARTS/ingestion/ \ + --set image.tag=0.1.0 \ + --set image.repository=ingestion \ + --set dockerregistry=$ACR_SERVER \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=ingestion \ + --set ingress.hosts[0].tls=true \ + --set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ + --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ + --set secrets.appinsights.ikey=${AI_IKEY} \ + --set secrets.queue.keyname=IngestionServiceAccessKey \ + --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ + --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ + --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ + --namespace backend \ + --name ingestion-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status ingestion-v0.1.0 ``` ## Deploy DroneScheduler service diff --git a/deploymentARM.md b/deploymentARM.md index 3d65935d..0a010938 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -337,6 +337,7 @@ until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-contr export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) +INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # Create a self-signed certificate for TLS openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ @@ -344,30 +345,28 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ingestion-ingress-tls.key \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" -kubectl create secret tls ingestion-ingress-tls \ - --namespace backend \ - --key ingestion-ingress-tls.key \ - --cert ingestion-ingress-tls.crt - -# Update deployment YAML with image tag and the fqdn -cat $K8S/ingestion.yaml | \ - sed "s#image:#image: $ACR_SERVER/ingestion:0.1.0#g" | \ - sed "s#ingestion-host-name#$EXTERNAL_INGEST_FQDN#g" \ - > $K8S/ingestion-0.yaml - -# Create secret -kubectl -n backend create secret generic ingestion-secrets \ - --from-literal=queue_namespace=${INGESTION_QUEUE_NAMESPACE} \ - --from-literal=queue_name=${INGESTION_QUEUE_NAME} \ - --from-literal=queue_keyname=${INGESTION_ACCESS_KEY_NAME} \ - --from-literal=queue_keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ - --from-literal=appinsights-ikey=${AI_IKEY} - # Deploy service -kubectl --namespace backend apply -f $K8S/ingestion-0.yaml +helm install $HELM_CHARTS/ingestion/ \ + --set image.tag=0.1.0 \ + --set image.repository=ingestion \ + --set dockerregistry=$ACR_SERVER \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=ingestion \ + --set ingress.hosts[0].tls=true \ + --set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ + --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ + --set secrets.appinsights.ikey=${AI_IKEY} \ + --set secrets.queue.keyname=IngestionServiceAccessKey \ + --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ + --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ + --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ + --namespace backend \ + --name ingestion-v0.1.0 # Verify the pod is created -kubectl get pods -n backend +helm status ingestion-v0.1.0 ``` ## Deploy DroneScheduler service From 9ed7e9c5d90cd4d95676592a2f5f695e250762cb Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 17:39:01 +0000 Subject: [PATCH 092/246] Merged PR 683: remove docker-compose support remove docker-compose support --- .../delivery/Fabrikam.DroneDelivery.sln | 6 ---- .../delivery/docker-compose.ci.build.yml | 9 ------ src/shipping/delivery/docker-compose.dcproj | 18 ----------- .../delivery/docker-compose.override.yml | 32 ------------------- src/shipping/delivery/docker-compose.yml | 8 ----- 5 files changed, 73 deletions(-) delete mode 100644 src/shipping/delivery/docker-compose.ci.build.yml delete mode 100644 src/shipping/delivery/docker-compose.dcproj delete mode 100644 src/shipping/delivery/docker-compose.override.yml delete mode 100644 src/shipping/delivery/docker-compose.yml diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.sln b/src/shipping/delivery/Fabrikam.DroneDelivery.sln index 95d5273f..d355f4ad 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.sln +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.sln @@ -5,8 +5,6 @@ VisualStudioVersion = 15.0.26730.15 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.DeliveryService", "Fabrikam.DroneDelivery.DeliveryService\Fabrikam.DroneDelivery.DeliveryService.csproj", "{D01A5347-3D4F-44F4-98C2-AD73D363631B}" EndProject -Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{2670610E-2495-42AA-BB4C-915D0B74E5CE}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.Common", "Fabrikam.DroneDelivery.Common\Fabrikam.DroneDelivery.Common.csproj", "{530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{4D81DB69-724B-44C2-A779-66778C21722E}" @@ -23,10 +21,6 @@ Global {D01A5347-3D4F-44F4-98C2-AD73D363631B}.Debug|Any CPU.Build.0 = Debug|Any CPU {D01A5347-3D4F-44F4-98C2-AD73D363631B}.Release|Any CPU.ActiveCfg = Release|Any CPU {D01A5347-3D4F-44F4-98C2-AD73D363631B}.Release|Any CPU.Build.0 = Release|Any CPU - {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2670610E-2495-42AA-BB4C-915D0B74E5CE}.Release|Any CPU.Build.0 = Release|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Debug|Any CPU.Build.0 = Debug|Any CPU {530D7BEC-E8B0-4B0A-B608-EAC10E4E858E}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/src/shipping/delivery/docker-compose.ci.build.yml b/src/shipping/delivery/docker-compose.ci.build.yml deleted file mode 100644 index ab96d707..00000000 --- a/src/shipping/delivery/docker-compose.ci.build.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3.4' - -services: - ci-build: - image: microsoft/dotnet:2.2-sdk - volumes: - - .:/src - working_dir: /src - command: /bin/bash -c "dotnet restore ./Fabrikam.DroneDelivery.sln && dotnet test Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj && dotnet publish ./Fabrikam.DroneDelivery.sln -c Release -o ./obj/Docker/publish" diff --git a/src/shipping/delivery/docker-compose.dcproj b/src/shipping/delivery/docker-compose.dcproj deleted file mode 100644 index 791f1205..00000000 --- a/src/shipping/delivery/docker-compose.dcproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - 2670610e-2495-42aa-bb4c-915d0b74e5ce - True - http://localhost:{ServicePort}/api/values - fabrikam.dronedelivery.deliveryservice - Linux - 2.1 - - - - - docker-compose.yml - - - - \ No newline at end of file diff --git a/src/shipping/delivery/docker-compose.override.yml b/src/shipping/delivery/docker-compose.override.yml deleted file mode 100644 index b58ab7d7..00000000 --- a/src/shipping/delivery/docker-compose.override.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3.4' - -services: - delivery: - environment: - - ASPNETCORE_ENVIRONMENT=Development - ports: - - "80" - - dronescheduler: - environment: - - ASPNETCORE_ENVIRONMENT=Development - ports: - - "80" - - deliveryscheduler: - environment: - - ASPNETCORE_ENVIRONMENT=Development - ports: - - "80" - - account: - environment: - - ASPNETCORE_ENVIRONMENT=Development - ports: - - "80" - - thirdparty: - environment: - - ASPNETCORE_ENVIRONMENT=Development - ports: - - "80" diff --git a/src/shipping/delivery/docker-compose.yml b/src/shipping/delivery/docker-compose.yml deleted file mode 100644 index b179717e..00000000 --- a/src/shipping/delivery/docker-compose.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '3.4' - -services: - delivery: - image: fabrikam.dronedelivery.deliveryservice - build: - context: ./Fabrikam.DroneDelivery.DeliveryService - dockerfile: Dockerfile From 374ae56854ee0d782302ac0aa900bdd32537f26b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 11 Mar 2019 20:15:17 +0000 Subject: [PATCH 093/246] Merged PR 686: propagte replica count based on today's SU discussion: propagate replica count. - team can configure its desired replica count for a particular service. - DevOps could scale-up/down number of replicas outside CD pipeline if required. --- charts/delivery/templates/delivery-deploy.yaml | 2 +- charts/dronescheduler/templates/dronescheduler-deploy.yaml | 2 +- charts/ingestion/templates/ingestion-deploy.yaml | 2 +- charts/package/templates/package-deploy.yaml | 2 +- charts/workflow/templates/workflow-deploy.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 8b0ec1b9..1fe38914 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -22,7 +22,7 @@ metadata: annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: - replicas: 1 + replicas: {{ default 1 .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "delivery.name" . }} diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index dab1fb87..aeffb81a 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -22,7 +22,7 @@ metadata: annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: - replicas: 1 + replicas: {{ default 1 .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "dronescheduler.name" . }} diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index eac51d7f..54a8e281 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -21,7 +21,7 @@ metadata: annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: - replicas: 1 + replicas: {{ default 1 .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "ingestion.name" . }} diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index f7a8903c..def4a8ae 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -21,7 +21,7 @@ metadata: annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: - replicas: 1 + replicas: {{ default 1 .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "package.name" . }} diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 7cf769db..f65926f7 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -22,7 +22,7 @@ metadata: annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: - replicas: 1 + replicas: {{ default 1 .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "workflow.name" . }} From 483c640eb24cca1cd6d8642cdc3d7913c9133db4 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Wed, 13 Mar 2019 14:14:08 +0000 Subject: [PATCH 094/246] Merged PR 688: Update dev dependencies for package service update dev dependencies, migrate gulp script to match Related work items: #8562 --- .../workflow/templates/workflow-deploy.yaml | 2 + deploymentARM.md | 2 +- src/shipping/package/gulpfile.js | 42 +++++++++---------- src/shipping/package/package.json | 22 +++++----- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index f65926f7..2e39e58c 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -50,6 +50,8 @@ spec: mountPath: /kvmnt readOnly: true env: + - name: CONFIGURATION_FOLDER + value: /kvmnt - name: SERVICE_URI_DELIVERY value: {{ .Values.serviceuri.delivery }} - name: SERVICE_URI_DRONE diff --git a/deploymentARM.md b/deploymentARM.md index 0a010938..0c69a5e5 100644 --- a/deploymentARM.md +++ b/deploymentARM.md @@ -335,7 +335,7 @@ helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-contr # Obtain the load balancer ip address and assign a domain name until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls diff --git a/src/shipping/package/gulpfile.js b/src/shipping/package/gulpfile.js index 162f5acf..d0d53755 100644 --- a/src/shipping/package/gulpfile.js +++ b/src/shipping/package/gulpfile.js @@ -4,16 +4,16 @@ var nodemon = require('gulp-nodemon'); var del = require('del'); var mocha = require('gulp-mocha'); -gulp.task('set-env', function() { +gulp.task('set-env', gulp.series(function(done) { process.env.CONNECTION_STRING = "mongodb://packagedb:27017/local"; process.env.COLLECTION_NAME = "packages"; process.env.CORRELATION_HEADER = "x-b3-traceid"; process.env.LOG_LEVEL = "debug" - return; -}); + done(); +})); // Generate types that can be used when working with api definitions -gulp.task ('create-api-classes', function() { +gulp.task ('create-api-classes', gulp.series(function() { let generated = "// GENERATED FILE - DO NOT EDIT\n\n"; @@ -47,9 +47,14 @@ gulp.task ('create-api-classes', function() { return console.log(err); } }); -}); +})); -gulp.task('build', ['clean'], function () { +// delete all build output files +gulp.task('clean', gulp.series(function (done) { + return del(['.bin/'], done); + })); + +gulp.task('build', gulp.series('clean', function () { gulp.src('**/*.json', {'cwd':'app'}) .pipe(gulp.dest('.bin/app')); @@ -59,34 +64,29 @@ gulp.task('build', ['clean'], function () { .pipe(tsProject()); return tsResult.js.pipe(gulp.dest('.bin/app')); -}); +})); -gulp.task('serve', ['build', 'set-env'], function (cb) { +gulp.task('serve', gulp.series('build', 'set-env', function (cb) { return nodemon({ script: '.bin/app/main.js', ext: 'ts json', watch: 'app', env: { "PORT": 80 }, tasks: ["build"], - env: { 'NODE_ENV': 'dev' } + env: { 'NODE_ENV': 'development' } }) -}); - -// delete all build output files -gulp.task('clean', function (done) { - return del(['.bin/'], done); -}); +})); -gulp.task('build-int-test', ['build'], function () { +gulp.task('build-int-test', gulp.series('build', function () { var tsProject = ts.createProject('tsconfig.json'); var tsResult = gulp.src("test/**/*.ts", {'base':'.'}) .pipe(tsProject()); return tsResult.js.pipe(gulp.dest('.bin')); -}); +})); -gulp.task('int-test', ['build-int-test', 'set-env'], function() { - gulp.src("test/**/*.js", { cwd: '.bin' }) - .pipe(mocha({ reporter: 'list' })); -}); +gulp.task('int-test', gulp.series('build-int-test', 'set-env', function() { + return gulp.src("test/**/*.js", { cwd: '.bin' }) + .pipe(mocha({ reporter: 'list', exit: true })); +})); diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 643191a9..1e0e573f 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -38,21 +38,21 @@ "winston": "^2.4.0" }, "devDependencies": { - "@types/koa": "^2.0.46", + "@types/koa": "^2.0.48", "@types/koa-bodyparser": "^5.0.1", - "@types/mocha": "^5.2.5", - "@types/node": "^10.12.2", + "@types/mocha": "^5.2.6", + "@types/node": "^11.10.5", "@types/winston": "^2.4.4", - "del": "^3.0.0", - "gulp": "^3.9.1", + "del": "^4.0.0", + "gulp": "^4.0.0", "gulp-cli": "^2.0.1", "gulp-mocha": "^6.0.0", - "gulp-nodemon": "^2.4.1", - "gulp-typescript": "^4.0.2", + "gulp-nodemon": "^2.4.2", + "gulp-typescript": "^5.0.0", "gulp-util": "^3.0.8", - "mocha": "^5.2.0", - "nodemon": "^1.18.6", - "npm-check-updates": "^2.14.2", - "typescript": "^3.1.6" + "mocha": "^6.0.2", + "nodemon": "^1.18.10", + "npm-check-updates": "^3.0.2", + "typescript": "^3.3.3333" } } From 190bde1096f7a5025f066eaa3f7ce3f3b2c8f9c7 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 13 Mar 2019 16:24:30 +0000 Subject: [PATCH 095/246] Merged PR 681: bug fix logger competing depedencies issue bug fix logger competing depedencies issue - reduce image ~70MiB :) -> yet to be improved - remove unused/competing dependencies - SLF4J instead of specific logger API (Log4J) - send telemetry to AI using Logback for more information, https://github.com/mspnp/microservices-reference-implementation/issues/121 solved: #9022 Related work items: #9022 --- .../ingestion/templates/ingestion-deploy.yaml | 2 + src/shipping/ingestion/etc/AI-Agent.xml | 15 +++++ src/shipping/ingestion/pom.xml | 67 +++---------------- .../configuration/AsyncExceptionHandler.java | 6 +- .../controller/IngestionController.java | 35 ++++++---- .../src/main/resources/application.properties | 2 +- .../ingestion/src/main/resources/log4j2.xml | 14 ---- .../src/main/resources/logback-spring.xml | 12 ++++ 8 files changed, 63 insertions(+), 90 deletions(-) create mode 100755 src/shipping/ingestion/etc/AI-Agent.xml delete mode 100644 src/shipping/ingestion/src/main/resources/log4j2.xml create mode 100644 src/shipping/ingestion/src/main/resources/logback-spring.xml diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 54a8e281..5a3a23bf 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -79,5 +79,7 @@ spec: secretKeyRef: name: ingestion-secrets key: appinsights-ikey + - name: APPINSIGHTS_LOGGERLEVEL + value: {{ default "error" .Values.appinsights.loggerlevel }} - name: CONTAINER_NAME value: *ingestion-container_name diff --git a/src/shipping/ingestion/etc/AI-Agent.xml b/src/shipping/ingestion/etc/AI-Agent.xml new file mode 100755 index 00000000..53acb593 --- /dev/null +++ b/src/shipping/ingestion/etc/AI-Agent.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 7127cf19..602df5ab 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -24,18 +24,12 @@ 1.16.10 2.7.5 2.1.2.RELEASE - 1.1.3 - 1.7.12 org.springframework.boot - spring-boot-starter-data-rest - - - org.springframework.boot - spring-boot-starter-web-services + spring-boot-starter-web @@ -50,7 +44,6 @@ 1.0.5 - org.mockito mockito-core @@ -58,41 +51,10 @@ test - - org.powermock - powermock-module-junit4 - 1.7.1 - test - - - org.powermock - powermock-api-mockito2 - 1.7.1 - test - - - - junit junit - - com.microsoft.azure - azure-storage - 5.0.0 - - - - javax.ejb - ejb-api - 3.0 - - - org.apache.openejb - openejb-core - 4.7.4 - org.springframework.boot @@ -116,25 +78,6 @@ azure-servicebus 1.2.8 - - org.springframework.boot - spring-boot-starter-batch - - - org.springframework.boot - spring-boot-configuration-processor - true - - - org.apache.logging.log4j - log4j-api - 2.9.1 - - - org.apache.logging.log4j - log4j-core - 2.9.1 - com.microsoft.azure @@ -147,6 +90,12 @@ applicationinsights-web 2.3.0 + + + com.microsoft.azure + applicationinsights-logging-logback + 2.3.0 + @@ -194,4 +143,4 @@ - + \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java index e7ef22a1..a5fc4ec6 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/configuration/AsyncExceptionHandler.java @@ -1,14 +1,14 @@ package com.fabrikam.dronedelivery.ingestion.configuration; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; import java.lang.reflect.Method; public class AsyncExceptionHandler implements AsyncUncaughtExceptionHandler { - private final static Logger log = LogManager.getLogger(AsyncExceptionHandler.class); + private final static Logger log = LoggerFactory.getLogger(AsyncExceptionHandler.class); @Override public void handleUncaughtException(final Throwable throwable, final Method method, final Object... obj) { diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java index c8c357ea..189372d7 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/controller/IngestionController.java @@ -10,8 +10,10 @@ import javax.servlet.http.HttpServletResponse; -import org.apache.logging.log4j.CloseableThreadContext; -import org.apache.logging.log4j.LogManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -30,12 +32,10 @@ import java.io.IOException; import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.logging.log4j.Logger; - @RestController public class IngestionController { - private final static Logger log = LogManager.getLogger(IngestionController.class); + private final static Logger log = LoggerFactory.getLogger(IngestionController.class); @Autowired private Ingestion ingestion; @@ -81,8 +81,10 @@ public CompletableFuture> scheduleDeliveryAsync @RequestBody ExternalDelivery externalDelivery, @RequestHeader HttpHeaders httpHeaders) { String deliveryId = UUID.randomUUID().toString(); - - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) { + + try { + MDC.put("DeliveryId", deliveryId); + log.info("In schedule delivery action with delivery request {}", externalDelivery.toString()); // Exceptions handled by exception handler @@ -100,6 +102,8 @@ public CompletableFuture> scheduleDeliveryAsync // Extract the headers as a map and pass on to eventhub message // dumper ingestion.scheduleDeliveryAsync(delivery, httpHeaders.toSingleValueMap()); + } finally { + MDC.remove("DeliveryId"); } return CompletableFuture.completedFuture(new ResponseEntity<>(externalDelivery, HttpStatus.ACCEPTED)); @@ -113,12 +117,15 @@ public CompletableFuture> cancelDeliveryAsync(HttpServlet // Exceptions handled by exception handler // making standard in the controller - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) - { + try { + MDC.put("DeliveryId", deliveryId); + log.info("In cancel delivery action with id: {}", deliveryId); ingestion.cancelDeliveryAsync(deliveryId.toString(), httpHeaders.toSingleValueMap()); + } finally { + MDC.remove("DeliveryId"); } - + return CompletableFuture.completedFuture(new ResponseEntity<>(deliveryId, HttpStatus.NO_CONTENT)); } @@ -126,8 +133,8 @@ public CompletableFuture> cancelDeliveryAsync(HttpServlet @ResponseBody public CompletableFuture> rescheduleDeliveryAsync(HttpServletResponse response, @RequestBody ExternalRescheduledDelivery externalRescheduledDelivery, @PathVariable("id") String deliveryId, @RequestHeader HttpHeaders httpHeaders) { - - try (final CloseableThreadContext.Instance ctc = CloseableThreadContext.put("DeliveryId", deliveryId)) { + try { + MDC.put("DeliveryId", deliveryId); log.info("In reschedule delivery action with delivery request: {}", externalRescheduledDelivery.toString()); @@ -141,6 +148,8 @@ public CompletableFuture> rescheduleDeliveryAsync(HttpSer externalRescheduledDelivery.getDeadline(), externalRescheduledDelivery.getPickupTime()); ingestion.rescheduleDeliveryAsync(rescheduledDelivery, httpHeaders.toSingleValueMap()); + } finally { + MDC.remove("DeliveryId"); } return CompletableFuture.completedFuture(new ResponseEntity<>(deliveryId, HttpStatus.OK)); @@ -153,4 +162,4 @@ public CompletableFuture> probeDeliveryAsync() { //implement logic to test readiness return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.OK)); } -} +} \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/resources/application.properties b/src/shipping/ingestion/src/main/resources/application.properties index 96f41dce..9c7cdcb9 100644 --- a/src/shipping/ingestion/src/main/resources/application.properties +++ b/src/shipping/ingestion/src/main/resources/application.properties @@ -19,4 +19,4 @@ logging.level.org.springframework.web=INFO spring.application.name=${CONTAINER_NAME} # Logging level [all, trace, info, warn, error, off]. Default value: error. -azure.application-insights.logger.level=info \ No newline at end of file +azure.application-insights.logger.level=${APPINSIGHTS_LOGGERLEVEL} \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/resources/log4j2.xml b/src/shipping/ingestion/src/main/resources/log4j2.xml deleted file mode 100644 index 70cd2f93..00000000 --- a/src/shipping/ingestion/src/main/resources/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/shipping/ingestion/src/main/resources/logback-spring.xml b/src/shipping/ingestion/src/main/resources/logback-spring.xml new file mode 100644 index 00000000..1ac77874 --- /dev/null +++ b/src/shipping/ingestion/src/main/resources/logback-spring.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + \ No newline at end of file From 519a9a75c7813379dc365548cba93babc9f66df1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 14 Mar 2019 18:50:46 +0000 Subject: [PATCH 096/246] Merged PR 689: remove manual deployment steps from today's team agreement, removing the manual deployment steps in favor of ARM only as this version is more streamlined. --- README.md | 2 +- deployment.md | 303 +++++++++++-------------------- deploymentARM.md | 460 ----------------------------------------------- 3 files changed, 104 insertions(+), 661 deletions(-) delete mode 100644 deploymentARM.md diff --git a/README.md b/README.md index c7a71354..8a08259f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The Drone Delivery application is a sample application that consists of several ## Deployment -To deploy the solution, follow the steps listed [here](./deployment.md) to get deep understanding on the infrastructure or [just use ARM templates](./deploymentARM.md). +To deploy the solution, follow the steps listed [here](./deployment.md). diff --git a/deployment.md b/deployment.md index 696d9a7e..67cdacc9 100644 --- a/deployment.md +++ b/deployment.md @@ -6,6 +6,7 @@ - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) - [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) +- [JQ](https://stedolan.github.io/jq/download/) > Note: in linux systems, it is possible to run the docker command without prefacing > with sudo. For more information, please refer to [the Post-installation steps @@ -19,18 +20,22 @@ git clone https://github.com/mspnp/microservices-reference-implementation.git The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -## Create the Kubernetes cluster +## Generate a SSH rsa public/private key pair + +the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). +> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. + +## Azure Resources Provisioning Set environment variables. ```bash -export LOCATION=[YOUR_LOCATION_HERE] +export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] +export SSH_PRIVATE_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PRIVAYE_KEY_FILE_HERE] -export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] +export LOCATION=[YOUR_LOCATION_HERE] -export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ -export CLUSTER_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" && \ -export AI_NAME="${UNIQUE_APP_NAME_PREFIX}-appinsights" +export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) @@ -40,78 +45,95 @@ export K8S=$PROJECT_ROOT/k8s export HELM_CHARTS=$PROJECT_ROOT/charts ``` -Provision a Kubernetes cluster in AKS +Infrastructure Prerequisites ```bash # Log in to Azure az login -# Create a resource group for AKS -az group create --name $RESOURCE_GROUP --location $LOCATION +# Create a resource group and service principal for AKS +az group create --name $RESOURCE_GROUP --location $LOCATION && \ +export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ +export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ +export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ +export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) +``` -# Create the AKS cluster -az aks create --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --node-count 4 --enable-addons monitoring --generate-ssh-keys +Deployment -# Install kubectl -sudo az aks install-cli +> Note: this deployment might take up to 20 minutes -# Get the Kubernetes cluster credentials -az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +```bash +# Deploy the managed identities +# These are deployed first in a separate template to avoid propagation delays with AAD +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file azuredeploy-identities.json +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) -# Get the cluster's principal -export CLUSTER_SERVICE_PRINCIPAL=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query servicePrincipalProfile.clientId --output tsv) +# Wait for AAD propagation +until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done -# Create namespaces -kubectl create namespace backend && \ -kubectl create namespace frontend +# Deploy all other resources +# The version of kubernetes must be supported in the target region +export KUBERNETES_VERSION='1.12.6' +az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ +--parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + servicePrincipalId=${SP_OBJECT_ID} \ + kubernetesVersion=${KUBERNETES_VERSION} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ + deliveryIdName=${DELIVERY_ID_NAME} \ + deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ + droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ + droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ + workflowIdName=${WORKFLOW_ID_NAME} \ + workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} ``` -Setup Helm in the container - +Get outputs from Azure Deploy ```bash -kubectl apply -f $K8S/tiller-rbac.yaml -helm init --service-account tiller +# Shared +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) ``` -Create an Azure Container Registry instance. - -> Note: Azure Container Registory is not required. If you prefer, you can store the Docker images for this solution in another container registry. - +Enable Azure Monitoring for the AKS cluster ```bash -export ACR_NAME=[YOUR_CONTAINER_REGISTRY_NAME_HERE] +az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +``` -# Create the ACR instance -az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic +Download kubectl and create a k8s namespace +```bash +# Install kubectl +sudo az aks install-cli -# Log in to ACR -az acr login --name $ACR_NAME +# Get the Kubernetes cluster credentials +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -# Get the ACR login server name -export ACR_SERVER=$(az acr show -g $RESOURCE_GROUP -n $ACR_NAME --query "loginServer" -o tsv) +# Create namespaces +kubectl create namespace backend && \ +kubectl create namespace frontend ``` -Grant the cluster access to the registry. +Setup Helm in the container ```bash -# Acquire the necessary IDs -export ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP --query "id" --output tsv) - -# Grant the cluster read access to the registry -az role assignment create --assignee $CLUSTER_SERVICE_PRINCIPAL --role Reader --scope $ACR_ID +kubectl apply -f $K8S/tiller-rbac.yaml +helm init --service-account tiller ``` -Create an Application Insights instance +Integrate Application Insights instance ```bash -# Create AppInsights instance -az resource create \ - --resource-group $RESOURCE_GROUP \ - --resource-type "Microsoft.Insights/components" \ - --name $AI_NAME \ - --location $LOCATION \ - --properties '{"Application_Type":"other"}' - # Acquire Instrumentation Key +export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) export AI_IKEY=$(az resource show \ -g $RESOURCE_GROUP \ -n $AI_NAME \ @@ -127,6 +149,8 @@ kubectl apply -f $K8S/k8s-rbac-ai.yaml Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol +Note: the tested nmi version was 1.4. It enables namespaced pod identity. + ```bash # setup AAD pod identity kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml @@ -136,28 +160,13 @@ kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-fl ## Deploy the Delivery service -Provision Azure resources +Extract resource details from deployment ```bash -export REDIS_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-redis" && \ -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-service-cosmosdb" && \ +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ -export COLLECTION_NAME="${DATABASE_NAME}-col" - -# Create Azure Redis Cache -az redis create --location $LOCATION \ - --name $REDIS_NAME \ - --resource-group $RESOURCE_GROUP \ - --sku Premium \ - --vm-size P4 - -# Create Cosmos DB account with DocumentDB API -az cosmosdb create \ - --name $COSMOSDB_NAME \ - --kind GlobalDocumentDB \ - --resource-group $RESOURCE_GROUP \ - --max-interval 10 \ - --max-staleness-prefix 200 +export COLLECTION_NAME="${DATABASE_NAME}-col" && \ +export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) ``` Build the Delivery service @@ -177,51 +186,13 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/delivery:0.1.0 ``` -Create KeyVault and secrets - -```bash -# Create the vault -export DELIVERY_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-delivery-kv" -az keyvault create --name $DELIVERY_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION - -export DELIVERY_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "id" --output tsv) -export DELIVERY_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DELIVERY_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) - -# Retrieve resource details -export REDIS_ENDPOINT=$(az redis show --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query hostName -o tsv) -export REDIS_KEY=$(az redis list-keys --name $REDIS_NAME --resource-group $RESOURCE_GROUP --query primaryKey -o tsv) -export COSMOSDB_KEY=$(az cosmosdb list-keys --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query primaryMasterKey -o tsv) -export COSMOSDB_ENDPOINT=$(az cosmosdb show --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query documentEndpoint -o tsv) - -# Create secrets -az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Key --value ${COSMOSDB_KEY} # (consider using encoding base64 to keep the actual values) -az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name CosmosDB-Endpoint --value ${COSMOSDB_ENDPOINT} -az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-Endpoint --value ${REDIS_ENDPOINT} -az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name Redis-AccessKey --value ${REDIS_KEY} -az keyvault secret set --vault-name $DELIVERY_KEYVAULT_NAME --name ApplicationInsights--InstrumentationKey --value ${AI_IKEY} -``` - -Create and set up pod identity - -```bash -# Create the identity and extract properties -export DELIVERY_PRINCIPAL_NAME=delivery -az identity create --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "id" --output tsv) -export DELIVERY_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "principalId" --output tsv) -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DELIVERY_PRINCIPAL_NAME --query "clientId" --output tsv) - -# Grant the identity access to the KeyVault -az role assignment create --role Reader --assignee $DELIVERY_PRINCIPAL_ID --scope $DELIVERY_KEYVAULT_ID -az keyvault set-policy --name $DELIVERY_KEYVAULT_NAME --secret-permissions get list --spn $DELIVERY_PRINCIPAL_CLIENT_ID - -# Allow the cluster to manage the identity to assign to pods -az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DELIVERY_PRINCIPAL_RESOURCE_ID -``` - Deploy the Delivery service: ```bash +# Extract pod identity outputs from deployment +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) + # Deploy the service helm install $HELM_CHARTS/delivery/ \ --set image.tag=0.1.0 \ @@ -241,11 +212,10 @@ helm status delivery-v0.1.0 ## Deploy the Package service -Provision Azure resources +Extract resource details from deployment ```bash -export COSMOSDB_NAME="${UNIQUE_APP_NAME_PREFIX}-package-service-cosmosdb" -az cosmosdb create --name $COSMOSDB_NAME --kind MongoDB --resource-group $RESOURCE_GROUP +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) ``` Build the Package service @@ -265,6 +235,7 @@ Deploy the Package service ```bash # Create secret +# Note: Connection strings cannot be exported as outputs in ARM deployments export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') # Deploy service @@ -283,30 +254,10 @@ helm status package-v0.1.0 ## Deploy the Workflow service -Provision Azure resources +Extract resource details from deployment ```bash -export INGESTION_QUEUE_NAMESPACE="${UNIQUE_APP_NAME_PREFIX}-ingestion-ns" -export INGESTION_QUEUE_NAME="${UNIQUE_APP_NAME_PREFIX}-ingestion" - -az servicebus namespace create --location $LOCATION \ - --name $INGESTION_QUEUE_NAMESPACE \ - --resource-group $RESOURCE_GROUP \ - --sku Standard - -az servicebus queue create --name $INGESTION_QUEUE_NAME \ - --namespace-name $INGESTION_QUEUE_NAMESPACE \ - --resource-group $RESOURCE_GROUP - -az servicebus namespace authorization-rule create --namespace-name $INGESTION_QUEUE_NAMESPACE \ - --name WorkflowServiceAccessKey \ - --resource-group $RESOURCE_GROUP \ - --rights Listen - -export INGESTION_QUEUE_ID=$(az servicebus queue show --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_QUEUE_NAME --query "id" --output tsv) -export INGESTION_QUEUE_NAMESPACE_ID=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "id" --output tsv) -export INGESTION_QUEUE_NAMESPACE_ENDPOINT=$(az servicebus namespace show --resource-group $RESOURCE_GROUP --name $INGESTION_QUEUE_NAMESPACE --query "serviceBusEndpoint" --output tsv) -export WORKFLOW_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name WorkflowServiceAccessKey --query primaryKey -o tsv) +export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) ``` Build the workflow service @@ -322,41 +273,12 @@ az acr login --name $ACR_NAME docker push $ACR_SERVER/workflow:0.1.0 ``` - -Create KeyVault and secrets - -```bash -export WORKFLOW_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-workflow-kv" -az keyvault create --name $WORKFLOW_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION -az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueName --value ${INGESTION_QUEUE_NAME} -az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueEndpoint --value ${INGESTION_QUEUE_NAMESPACE_ENDPOINT} -az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueAccessPolicyName --value WorkflowServiceAccessKey -az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name QueueAccessPolicyKey --value ${WORKFLOW_ACCESS_KEY_VALUE} -az keyvault secret set --vault-name $WORKFLOW_KEYVAULT_NAME --name ApplicationInsights-InstrumentationKey --value ${AI_IKEY} - -export WORKFLOW_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "id" --output tsv) -export WORKFLOW_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $WORKFLOW_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) -``` - Create and set up pod identity - -> Note: after creating the identity, please wait for some time before assigning Reader role to the Workflow Principal Id. Otherwise it's possible to experience the following error: ```No matches in graph database for 'your-principal-id'``` - ```bash -# Create the identity and extract properties -export WORKFLOW_PRINCIPAL_NAME="workflow" -az identity create --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "id" --output tsv) -export WORKFLOW_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "principalId" --output tsv) -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $WORKFLOW_PRINCIPAL_NAME --query "clientId" --output tsv) - -# Grant the identity access to the KeyVault -az role assignment create --role Reader --assignee $WORKFLOW_PRINCIPAL_ID --scope $WORKFLOW_KEYVAULT_ID -az keyvault set-policy --name $WORKFLOW_KEYVAULT_NAME --secret-permissions get list --spn $WORKFLOW_PRINCIPAL_CLIENT_ID - -# Allow the cluster to manage the identity to assign to pods -az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $WORKFLOW_PRINCIPAL_RESOURCE_ID +# Extract outputs from deployment +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) ``` Deploy the Workflow service: @@ -382,17 +304,13 @@ helm status workflow-v0.1.0 ## Deploy the Ingestion service -Provision Azure resources +Extract resource details from deployment ```bash -# Create authorization rule to the ingestion queue -az servicebus namespace authorization-rule create --namespace-name $INGESTION_QUEUE_NAMESPACE \ - --name IngestionServiceAccessKey \ - --resource-group $RESOURCE_GROUP \ - --rights Send - -# Get access key -export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name IngestionServiceAccessKey --query primaryKey -o tsv) +export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ +export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) +export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) ``` Build the Ingestion service @@ -417,7 +335,7 @@ helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-contr # Obtain the load balancer ip address and assign a domain name until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${UNIQUE_APP_NAME_PREFIX}-ingest" +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls @@ -453,39 +371,24 @@ helm status ingestion-v0.1.0 ## Deploy DroneScheduler service -Build the dronescheduler services +Extract resource details from deployment ```bash -export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler +export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) ``` -Create KeyVault and secrets +Build the dronescheduler services ```bash -export DRONESCHEDULER_KEYVAULT_NAME="${UNIQUE_APP_NAME_PREFIX}-drone-kv" -az keyvault create --name $DRONESCHEDULER_KEYVAULT_NAME --resource-group $RESOURCE_GROUP --location $LOCATION -az keyvault secret set --vault-name $DRONESCHEDULER_KEYVAULT_NAME --name ApplicationInsights--InstrumentationKey --value ${AI_IKEY} - -export DRONESCHEDULER_KEYVAULT_ID=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_KEYVAULT_NAME --query "id" --output tsv) -export DRONESCHEDULER_KEYVAULT_URI=$(az keyvault show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_KEYVAULT_NAME --query "properties.vaultUri" --output tsv) +export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler ``` Create and set up pod identity ```bash -# Create the identity and extract properties -export DRONESCHEDULER_PRINCIPAL_NAME=dronescheduler -az identity create --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "id" --output tsv) -export DRONESCHEDULER_PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "principalId" --output tsv) -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $DRONESCHEDULER_PRINCIPAL_NAME --query "clientId" --output tsv) - -# Grant the identity access to the KeyVault -az role assignment create --role Reader --assignee $DRONESCHEDULER_PRINCIPAL_ID --scope $DRONESCHEDULER_KEYVAULT_ID -az keyvault set-policy --name $DRONESCHEDULER_KEYVAULT_NAME --secret-permissions get list --spn $DRONESCHEDULER_PRINCIPAL_CLIENT_ID - -# Allow the cluster to manage the identity to assign to pods -az role assignment create --role "Managed Identity Operator" --assignee $CLUSTER_SERVICE_PRINCIPAL --scope $DRONESCHEDULER_PRINCIPAL_RESOURCE_ID +# Extract outputs from deployment +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) ``` Build and publish the container image diff --git a/deploymentARM.md b/deploymentARM.md deleted file mode 100644 index 0c69a5e5..00000000 --- a/deploymentARM.md +++ /dev/null @@ -1,460 +0,0 @@ -# Deploying the Reference Implementation - -## Prerequisites - -- Azure suscription -- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) -- [Docker](https://docs.docker.com/) -- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) -- [JQ](https://stedolan.github.io/jq/download/) - -> Note: in linux systems, it is possible to run the docker command without prefacing -> with sudo. For more information, please refer to [the Post-installation steps -> for linux](https://docs.docker.com/install/linux/linux-postinstall/) - -Clone or download this repo locally. - -```bash -git clone https://github.com/mspnp/microservices-reference-implementation.git -``` - -The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. - -## Generate a SSH rsa public/private key pair - -the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). -> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. - -## Azure Resources Provisioning - -Set environment variables. - -```bash -export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] -export SSH_PRIVATE_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PRIVAYE_KEY_FILE_HERE] - -export LOCATION=[YOUR_LOCATION_HERE] - -export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] - -export SUBSCRIPTION_ID=$(az account show --query id --output tsv) -export TENANT_ID=$(az account show --query tenantId --output tsv) - -export PROJECT_ROOT=./microservices-reference-implementation -export K8S=$PROJECT_ROOT/k8s -export HELM_CHARTS=$PROJECT_ROOT/charts -``` - -Infrastructure Prerequisites - -```bash -# Log in to Azure -az login - -# Create a resource group and service principal for AKS -az group create --name $RESOURCE_GROUP --location $LOCATION && \ -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ -export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ -export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ -export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) -``` - -Deployment - -> Note: this deployment might take up to 20 minutes - -```bash -# Deploy the managed identities -# These are deployed first in a separate template to avoid propagation delays with AAD -az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file azuredeploy-identities.json -export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) -export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) -export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) -export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) -export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) -export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) - -# Wait for AAD propagation -until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done -until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done -until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done - -# Deploy all other resources -# The version of kubernetes must be supported in the target region -export KUBERNETES_VERSION='1.12.6' -az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ ---parameters servicePrincipalClientId=${SP_APP_ID} \ - servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ - servicePrincipalId=${SP_OBJECT_ID} \ - kubernetesVersion=${KUBERNETES_VERSION} \ - sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ - deliveryIdName=${DELIVERY_ID_NAME} \ - deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ - droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ - droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ - workflowIdName=${WORKFLOW_ID_NAME} \ - workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} -``` - -Get outputs from Azure Deploy -```bash -# Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) -``` - -Enable Azure Monitoring for the AKS cluster -```bash -az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -``` - -Download kubectl and create a k8s namespace -```bash -# Install kubectl -sudo az aks install-cli - -# Get the Kubernetes cluster credentials -az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME - -# Create namespaces -kubectl create namespace backend && \ -kubectl create namespace frontend -``` - -Setup Helm in the container - -```bash -kubectl apply -f $K8S/tiller-rbac.yaml -helm init --service-account tiller -``` - -Integrate Application Insights instance - -```bash -# Acquire Instrumentation Key -export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) -export AI_IKEY=$(az resource show \ - -g $RESOURCE_GROUP \ - -n $AI_NAME \ - --resource-type "Microsoft.Insights/components" \ - --query properties.InstrumentationKey \ - -o tsv) - -# add RBAC for AppInsights -kubectl apply -f $K8S/k8s-rbac-ai.yaml -``` - -## Setup AAD pod identity and key vault flexvol infrastructure - -Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol - -Note: the tested nmi version was 1.4. It enables namespaced pod identity. - -```bash -# setup AAD pod identity -kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml - -kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml -``` - -## Deploy the Delivery service - -Extract resource details from deployment - -```bash -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ -export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ -export COLLECTION_NAME="${DATABASE_NAME}-col" && \ -export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) -``` - -Build the Delivery service - -```bash -export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery -``` - -Build and publish the container image - -```bash -# Build the Docker image -docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/. - -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/delivery:0.1.0 -``` - -Deploy the Delivery service: - -```bash -# Extract pod identity outputs from deployment -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) - -# Deploy the service -helm install $HELM_CHARTS/delivery/ \ - --set image.tag=0.1.0 \ - --set image.repository=delivery \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ - --set cosmosdb.id=$DATABASE_NAME \ - --set cosmosdb.collectionid=$COLLECTION_NAME \ - --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ - --namespace backend \ - --name delivery-v0.1.0 - -# Verify the pod is created -helm status delivery-v0.1.0 -``` - -## Deploy the Package service - -Extract resource details from deployment - -```bash -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) -``` - -Build the Package service - -```bash -export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package - -# Build the docker image -docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/package:0.1.0 -``` - -Deploy the Package service - -```bash -# Create secret -# Note: Connection strings cannot be exported as outputs in ARM deployments -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') - -# Deploy service -helm install $HELM_CHARTS/package/ \ - --set image.tag=0.1.0 \ - --set image.repository=package \ - --set secrets.appinsights.ikey=$AI_IKEY \ - --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ - --set dockerregistry=$ACR_SERVER \ - --namespace backend \ - --name package-v0.1.0 - -# Verify the pod is created -helm status package-v0.1.0 -``` - -## Deploy the Workflow service - -Extract resource details from deployment - -```bash -export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) -``` - -Build the workflow service - -```bash -export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow - -# Build the Docker image -docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. - -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/workflow:0.1.0 -``` - -Create and set up pod identity - -```bash -# Extract outputs from deployment -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) -``` - -Deploy the Workflow service: - -```bash -# Deploy the service -helm install $HELM_CHARTS/workflow/ \ - --set image.tag=0.1.0 \ - --set image.repository=workflow \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ - --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ - --set keyvault.resourcegroup=$RESOURCE_GROUP \ - --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ - --set keyvault.tenantid=$TENANT_ID \ - --namespace backend \ - --name workflow-v0.1.0 - -# Verify the pod is created -helm status workflow-v0.1.0 -``` - -## Deploy the Ingestion service - -Extract resource details from deployment - -```bash -export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ -export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) -export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) -export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) -``` - -Build the Ingestion service - -```bash -export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion - -# Build the docker image -docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/ingestion:0.1.0 -``` - -Deploy the Ingestion service - -```bash -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true - -# Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done -export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" -export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) -INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls - -# Create a self-signed certificate for TLS -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -out ingestion-ingress-tls.crt \ - -keyout ingestion-ingress-tls.key \ - -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" - -# Deploy service -helm install $HELM_CHARTS/ingestion/ \ - --set image.tag=0.1.0 \ - --set image.repository=ingestion \ - --set dockerregistry=$ACR_SERVER \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=ingestion \ - --set ingress.hosts[0].tls=true \ - --set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \ - --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ - --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ - --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ - --set secrets.appinsights.ikey=${AI_IKEY} \ - --set secrets.queue.keyname=IngestionServiceAccessKey \ - --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ - --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ - --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ - --namespace backend \ - --name ingestion-v0.1.0 - -# Verify the pod is created -helm status ingestion-v0.1.0 -``` - -## Deploy DroneScheduler service - -Extract resource details from deployment - -```bash -export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) -``` - -Build the dronescheduler services - -```bash -export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler -``` - -Create and set up pod identity - -```bash -# Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) -``` - -Build and publish the container image - -```bash -# Build the Docker image -docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ - -# Push the images to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/dronescheduler:0.1.0 -``` - -Deploy the dronescheduler service: -```bash -# Deploy the service -helm install $HELM_CHARTS/dronescheduler/ \ - --set image.tag=0.1.0 \ - --set image.repository=dronescheduler \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ - --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ - --namespace backend \ - --name dronescheduler-v0.1.0 - -# Verify the pod is created -helm status dronescheduler-v0.1.0 -``` - -## Validate the application is running - -You can send delivery requests to the ingestion service using the Swagger UI. - -Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. - -```bash -open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" -``` - -> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. - -## Optional steps - -Follow these steps to add logging and monitoring capabilities to the solution. - -Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch - -```bash -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \ -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \ -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml -``` - -Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd - -```bash -# The example elasticsearch yaml files deploy a service named "elasticsearch" -wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ -sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml - -# Commenting out X-Pack credentials for demo purposes. -# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads -sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ -sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ -sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ -sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ -kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml -``` From b36a35afb0ce4a867d2beb21152d273079cb7ff1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 15 Mar 2019 14:45:03 +0000 Subject: [PATCH 097/246] Merged PR 692: fix evaluation error fix evaluation error and align values telemetry section with other charts --- charts/ingestion/templates/ingestion-deploy.yaml | 2 +- charts/ingestion/values.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 5a3a23bf..38f7ef93 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -80,6 +80,6 @@ spec: name: ingestion-secrets key: appinsights-ikey - name: APPINSIGHTS_LOGGERLEVEL - value: {{ default "error" .Values.appinsights.loggerlevel }} + value: {{ default "error" .Values.telemetry.level | quote }} - name: CONTAINER_NAME value: *ingestion-container_name diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index e2060990..5e49237e 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -6,3 +6,5 @@ image: tag: pullPolicy: IfNotPresent reason: unknown +telemetry: + level: "error" From 5660222b4bc4bd51d8e86e144bea7edc1a2535ea Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 15 Mar 2019 18:29:12 +0000 Subject: [PATCH 098/246] Merged PR 693: add AI telemetry for dronescheduler using ILoggerBuilder add AI telemetry for dronescheduler using ILoggerBuilder - upgrade to AppInsights 2.6.1 and Microsoft.Extensions.Logging.ApplicationInsights 2.9.1 - solve incompatibility issue between AppInsishgts for Kuberentes and Microsoft.Extensions.Logging.ApplicationInsights - add ILogger to program - update config to align with new AppInsights options - make telemetry level configurable solved: #9049 Related work items: #9049 --- .../templates/dronescheduler-deploy.yaml | 2 + charts/dronescheduler/values.yaml | 2 + ...brikam.DroneDelivery.DroneScheduler.csproj | 3 +- .../Program.cs | 15 +++- .../Startup.cs | 3 - .../TracingExtensions.cs | 69 +++++++++++++++++++ .../appsettings.Development.json | 5 ++ .../appsettings.json | 8 ++- 8 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index aeffb81a..296ae6d0 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -48,6 +48,8 @@ spec: env: - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} + - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 ports: diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 26065bd3..ea9d1dc4 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -11,3 +11,5 @@ image: keyvault: uri: reason: unknown +telemetry: + level: "Error" diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj index dbbe2589..af4ebbf9 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj @@ -10,7 +10,8 @@ - + + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs index 4107979b..d9c11b33 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs @@ -8,8 +8,12 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Serilog; namespace MockDroneScheduler { @@ -17,12 +21,21 @@ public class Program { public static void Main(string[] args) { - CreateWebHostBuilder(args).Build().Run(); + var host = CreateWebHostBuilder(args).Build(); + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Fabrikam Dronescheduler Service is starting."); + + host.Run(); } public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) .UseStartup() + .ConfigureLogging((hostingContext, loggingBuilder) => + { + loggingBuilder.AddApplicationInsights(hostingContext.Configuration); + loggingBuilder.AddSerilog(dispose: true); + }) .UseUrls("http://0.0.0.0:8080"); } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs index e92730b5..acea915b 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs @@ -47,9 +47,6 @@ public void ConfigureServices(IServiceCollection services) services.AddApplicationInsightsKubernetesEnricher(); services.AddApplicationInsightsTelemetry(Configuration); - services.AddLogging(loggingBuilder => - loggingBuilder.AddSerilog(dispose: true)); - // Add framework services. services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs new file mode 100644 index 00000000..629c7e96 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.ApplicationInsights; + +namespace MockDroneScheduler +{ + public static class TracingExtensions + { + private const string AppInsightsSectionName = "ApplicationInsightsLogger"; + + public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this IServiceCollection services) + { + services.Configure( + (config) => + config.AddApplicationInsightsKubernetesEnricher( + applyOptions: null) + ); + + return services; + } + + public static ILoggingBuilder AddApplicationInsights( + this ILoggingBuilder loggingBuilder, + IConfiguration configuration) + { + var aiOptions = + configuration + .GetSection(AppInsightsSectionName) + ?.Get() ?? + new ApplicationInsightsLoggerExtendedOptions(); + + loggingBuilder.AddFilter + ( + "", + aiOptions.TelemetryLogLevel); + + loggingBuilder.AddApplicationInsights( + o => + { + o.IncludeScopes = + aiOptions.IncludeScopes; + o.TrackExceptionsAsExceptionTelemetry = + aiOptions.TrackExceptionsAsExceptionTelemetry; + }); + + return loggingBuilder; + } + + private class ApplicationInsightsLoggerExtendedOptions + : ApplicationInsightsLoggerOptions + { + public ApplicationInsightsLoggerExtendedOptions() + : base() + { + this.TelemetryLogLevel = LogLevel.Warning; + } + + public LogLevel TelemetryLogLevel + { get; set; } + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json index fa8ce71a..3e5f5a85 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json @@ -6,5 +6,10 @@ "System": "Information", "Microsoft": "Information" } + }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "None", + "IncludeScopes": false, + "TrackExceptionsAsExceptionTelemetry": false } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json index 815fe795..8e99c47d 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json @@ -3,12 +3,14 @@ "IncludeScopes": false, "LogLevel": { "Default": "Information" - }, - "CorrelationHeaderKey": "l5d-ctx-trace" + } + }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "Error" }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], "WriteTo": [] } -} \ No newline at end of file +} From b7902413103303762118316b0a1c32ed8b4141ce Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 15 Mar 2019 18:51:40 +0000 Subject: [PATCH 099/246] Merged PR 687: add telemetry to delivery add telemetry to delivery Related work items: #9046 --- .../delivery/templates/delivery-deploy.yaml | 2 + charts/delivery/values.yaml | 2 + ...rikam.DroneDelivery.DeliveryService.csproj | 3 +- .../Program.cs | 15 +++- .../Startup.cs | 6 +- .../TracingExtensions.cs | 69 +++++++++++++++++++ .../appsettings.Development.json | 5 ++ .../appsettings.json | 5 +- 8 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 1fe38914..d1a42748 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -52,6 +52,8 @@ spec: value: {{ .Values.cosmosdb.collectionid }} - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} + - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 ports: diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index e177b734..494a260b 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -13,4 +13,6 @@ cosmosdb: collectionid: keyvault: uri: +telemetry: + level: "Error" reason: unknown diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index 19e2ae82..70a3a033 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -10,7 +10,8 @@ - + + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs index c783747d..eaab3942 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs @@ -3,8 +3,12 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Serilog; namespace Fabrikam.DroneDelivery.DeliveryService { @@ -12,12 +16,21 @@ public class Program { public static void Main(string[] args) { - CreateWebHostBuilder(args).Build().Run(); + var host = CreateWebHostBuilder(args).Build(); + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Fabrikam Delivery Service is starting."); + + host.Run(); } public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) .UseStartup() + .ConfigureLogging((hostingContext, loggingBuilder) => + { + loggingBuilder.AddApplicationInsights(hostingContext.Configuration); + loggingBuilder.AddSerilog(dispose: true); + }) .UseUrls("http://0.0.0.0:8080"); } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index eaa42504..66ef9f84 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -16,7 +16,6 @@ using Fabrikam.DroneDelivery.DeliveryService.Middlewares.Builder; using Serilog; using Serilog.Formatting.Compact; -using Fabrikam.DroneDelivery.Common; namespace Fabrikam.DroneDelivery.DeliveryService { @@ -32,7 +31,7 @@ public Startup(IHostingEnvironment env) var buildConfig = builder.Build(); - if(buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) + if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) { builder.AddAzureKeyVault(keyVaultUri); } @@ -51,9 +50,6 @@ public void ConfigureServices(IServiceCollection services) services.AddApplicationInsightsKubernetesEnricher(); services.AddApplicationInsightsTelemetry(Configuration); - services.AddLogging(loggingBuilder => - loggingBuilder.AddSerilog(dispose: true)); - // Add framework services. services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs new file mode 100644 index 00000000..838daef0 --- /dev/null +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.ApplicationInsights; + +namespace Fabrikam.DroneDelivery.DeliveryService +{ + public static class TracingExtensions + { + private const string AppInsightsSectionName = "ApplicationInsightsLogger"; + + public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this IServiceCollection services) + { + services.Configure( + (config) => + config.AddApplicationInsightsKubernetesEnricher( + applyOptions: null) + ); + + return services; + } + + public static ILoggingBuilder AddApplicationInsights( + this ILoggingBuilder loggingBuilder, + IConfiguration configuration) + { + var aiOptions = + configuration + .GetSection(AppInsightsSectionName) + ?.Get() ?? + new ApplicationInsightsLoggerExtendedOptions(); + + loggingBuilder.AddFilter + ( + "", + aiOptions.TelemetryLogLevel); + + loggingBuilder.AddApplicationInsights( + o => + { + o.IncludeScopes = + aiOptions.IncludeScopes; + o.TrackExceptionsAsExceptionTelemetry = + aiOptions.TrackExceptionsAsExceptionTelemetry; + }); + + return loggingBuilder; + } + + private class ApplicationInsightsLoggerExtendedOptions + : ApplicationInsightsLoggerOptions + { + public ApplicationInsightsLoggerExtendedOptions() + : base() + { + this.TelemetryLogLevel = LogLevel.Warning; + } + + public LogLevel TelemetryLogLevel + { get; set; } + } + } +} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json index fa8ce71a..3e5f5a85 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json @@ -6,5 +6,10 @@ "System": "Information", "Microsoft": "Information" } + }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "None", + "IncludeScopes": false, + "TrackExceptionsAsExceptionTelemetry": false } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json index a2bb0628..3b7c8685 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json @@ -5,6 +5,9 @@ "Default": "Information" } }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "Error" + }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], @@ -19,4 +22,4 @@ //} ] } -} \ No newline at end of file +} From 3289bab0ce12094a9b86673099e67748536c4bd0 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 15 Mar 2019 19:00:34 +0000 Subject: [PATCH 100/246] Merged PR 691: add AI telemetry for workflow using ILoggerBuilder add AI telemetry for workflow using ILoggerBuilder - upgrade to AppInsights 2.6.1 and Microsoft.Extensions.Logging.ApplicationInsights 2.9.1 - pipe telemetryconfiguration setup and config properly, this way configuration is shared between telemetry & distributed tracing - solve incompatibility issue between AppInsishgts for Kuberentes and Microsoft.Extensions.Logging.ApplicationInsights using a new better approach - clean up program since new pipe approach enables it - opt-in kubernetes enricher as it looks more like its canonical sample + scale for others implementations where this could be used - update config to align with new AppInsights options - make telemetry level configurable Important: in case it is not that clear from [47da2](https://pnp.visualstudio.com/roadmap/_git/aks-ra/commit/47da251f616a1b94ef4837e378ffc61bb6af778e)..[83eb7](https://pnp.visualstudio.com/roadmap/_git/aks-ra/commit/83eb7959bf9fad335fd89aa779ba65c51ab5460b) is about to configure the channel implementation that sends telemetry to application insights and share telemetry configuration between logger and telemetry client. solved: #9056 Related work items: #9056 --- .../workflow/templates/workflow-deploy.yaml | 2 + charts/workflow/values.yaml | 3 +- .../Fabrikam.Workflow.Service.csproj | 3 +- .../Fabrikam.Workflow.Service/Program.cs | 24 +- .../ServiceStartup.cs | 3 +- .../TracingExtensions.cs | 224 ++++++++++++++---- .../appsettings.Development.json | 16 ++ .../appsettings.json | 6 + 8 files changed, 211 insertions(+), 70 deletions(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 2e39e58c..95bdfd95 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -72,6 +72,8 @@ spec: value: "{{ .Values.servicerequest.maxbulkheadsize }}" - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE value: "{{ .Values.servicerequest.maxbulkheadqueuesize }}" + - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 volumes: diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 0668cc7e..7b2d9225 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -26,4 +26,5 @@ keyvault: resourcegroup: subscriptionid: tenantid: - +telemetry: + level: "Error" diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index cc9b26c3..08e604dc 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -7,7 +7,8 @@ - + + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs index f53163ff..07d7af27 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Serilog; using Serilog.Formatting.Compact; @@ -20,23 +21,10 @@ static async Task Main(string[] args) { var host = CreateHostBuilder(args).Build(); - TelemetryClient telemetryClient = null; - try - { - telemetryClient = host.Services.GetService(); + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Fabrikan Workflow Service is starting."); - telemetryClient.TrackTrace("Fabrikan Workflow Service is starting."); - - await host.RunAsync(); - } - finally - { - // before exit, flush the remaining data - telemetryClient?.Flush(); - - // flush is not blocking so wait a bit - Task.Delay(5000).Wait(); - } + await host.RunAsync(); } private static IHostBuilder CreateHostBuilder(string[] args) @@ -57,6 +45,8 @@ private static IHostBuilder CreateHostBuilder(string[] args) }) .ConfigureLogging((context, builder) => { + builder.AddApplicationInsights(context.Configuration); + var serilogBuilder = new LoggerConfiguration() .ReadFrom.Configuration(context.Configuration) .WriteTo.Console(new CompactJsonFormatter()); @@ -67,4 +57,4 @@ private static IHostBuilder CreateHostBuilder(string[] args) .UseConsoleLifetime(); } } -} \ No newline at end of file +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index a761ff8a..a98e48aa 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -18,6 +18,7 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect services.AddOptions(); // Configure AppInsights + services.AddApplicationInsightsKubernetesEnricher(); services.AddApplicationInsightsTelemetry(context.Configuration); services.Configure(context.Configuration); @@ -47,4 +48,4 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect .AddResiliencyPolicies(context.Configuration); } } -} \ No newline at end of file +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs index 0fc472f0..f83551a7 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs @@ -3,16 +3,23 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.AspNetCore.TelemetryInitializers; +using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DependencyCollector; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility.Implementation.ApplicationId; using Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse; +using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.ApplicationInsights; +using Microsoft.Extensions.Options; namespace Fabrikam.Workflow.Service { @@ -24,24 +31,164 @@ namespace Fabrikam.Workflow.Service /// internal static class TracingExtensions { - private const string CustomKeyVaultAIppInsightsIKey = "ApplicationInsights-InstrumentationKey"; + private const string AppInsightsSectionName = "ApplicationInsightsLogger"; public static IServiceCollection AddApplicationInsightsTelemetry( this IServiceCollection services, IConfiguration configuration) { - services.AddSingleton(s => + // add initializers + services.AddSingleton< + ITelemetryInitializer, + DomainNameRoleInstanceTelemetryInitializer>(); + services.AddSingleton< + ITelemetryInitializer, + HttpDependenciesParsingTelemetryInitializer>(); + + // add modules + services.AddSingleton(s => { - var telemetryConfig = TelemetryConfiguration.CreateDefault(); + var module = new DependencyTrackingTelemetryModule(); + + var excludedDomains = module.ExcludeComponentCorrelationHttpHeadersOnDomains; + excludedDomains.Add("core.windows.net"); + excludedDomains.Add("core.chinacloudapi.cn"); + excludedDomains.Add("core.cloudapi.de"); + excludedDomains.Add("core.usgovcloudapi.net"); + + if (module.EnableLegacyCorrelationHeadersInjection) + { + excludedDomains.Add("localhost"); + excludedDomains.Add("127.0.0.1"); + } + + var includedActivities = module.IncludeDiagnosticSourceActivities; + includedActivities.Add("Microsoft.Azure.ServiceBus"); + + return module; + }); + + services.AddSingleton< + ITelemetryModule, + QuickPulseTelemetryModule>(); + + // add others + services.TryAddSingleton< + IApplicationIdProvider, + ApplicationInsightsApplicationIdProvider>(); + + services.TryAddSingleton< + ITelemetryChannel, + ServerTelemetryChannel>(); + + services.TryAddSingleton(); + + services.AddSingleton(provider => + provider.GetService>().Value); - var config = s.GetService(); - var instrumentationKey = config[CustomKeyVaultAIppInsightsIKey]; + services.AddOptions(); + services.AddSingleton, TelemetryConfigurationOptions>(); + services.AddSingleton, TelemetryConfigurationOptionsSetup>(); + + return services; + } + + public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this IServiceCollection services) + { + services.Configure( + (config) => + config.AddApplicationInsightsKubernetesEnricher( + applyOptions: null) + ); + + return services; + } + + public static ILoggingBuilder AddApplicationInsights( + this ILoggingBuilder loggingBuilder, + IConfiguration configuration) + { + var aiOptions = + configuration + .GetSection(AppInsightsSectionName) + ?.Get() ?? + new ApplicationInsightsLoggerExtendedOptions(); + + loggingBuilder.AddFilter + ( + "", + aiOptions.TelemetryLogLevel); + + loggingBuilder.AddApplicationInsights( + o => + { + o.IncludeScopes = + aiOptions.IncludeScopes; + o.TrackExceptionsAsExceptionTelemetry = + aiOptions.TrackExceptionsAsExceptionTelemetry; + }); + + return loggingBuilder; + } + + private class ApplicationInsightsLoggerExtendedOptions + : ApplicationInsightsLoggerOptions + { + public ApplicationInsightsLoggerExtendedOptions() + : base() + { + this.TelemetryLogLevel = LogLevel.Warning; + } + public LogLevel TelemetryLogLevel + { get; set; } + } + + private class TelemetryConfigurationOptionsSetup : IConfigureOptions + { + private const string CustomKeyVaultAppInsightsIKey = "ApplicationInsights-InstrumentationKey"; + private const string AppInsightsDeveloperMode = "ApplicationInsights:DeveloperMode"; + + private readonly IConfiguration _configuration; + private readonly IServiceProvider _serviceProvider; + private readonly IEnumerable _initializers; + private readonly IEnumerable _modules; + private readonly ITelemetryChannel _telemetryChannel; + + public TelemetryConfigurationOptionsSetup( + IServiceProvider serviceProvider, + IEnumerable initializers, + IEnumerable modules, + ITelemetryChannel telemetryChannel, + IConfiguration configuration) + { + this._serviceProvider = serviceProvider; + this._initializers = initializers; + this._modules = modules; + this._telemetryChannel = telemetryChannel; + this._configuration = configuration; + } + + public void Configure(TelemetryConfiguration telemetryConfig) + { + // flex volume + var instrumentationKey = _configuration[CustomKeyVaultAppInsightsIKey]; if (!string.IsNullOrWhiteSpace(instrumentationKey)) { telemetryConfig.InstrumentationKey = instrumentationKey; } + // Fallback to default channel (InMemoryChannel) created by base sdk if no channel is found in DI + telemetryConfig.TelemetryChannel = + this._telemetryChannel + ?? telemetryConfig.TelemetryChannel; + + if(bool.TryParse(_configuration[AppInsightsDeveloperMode], out bool developerMode)) + this._telemetryChannel.DeveloperMode = developerMode; + + (telemetryConfig.TelemetryChannel as ITelemetryModule) + ?.Initialize(telemetryConfig); + // use processors telemetryConfig .DefaultTelemetrySink @@ -50,9 +197,10 @@ public static IServiceCollection AddApplicationInsightsTelemetry( { var processor = new QuickPulseTelemetryProcessor(next); - var quickPulseModule = s.GetServices() - .OfType() - .Single(); + var quickPulseModule = _serviceProvider + .GetServices() + .OfType() + .Single(); quickPulseModule.RegisterTelemetryProcessor(processor); return processor; @@ -64,61 +212,37 @@ public static IServiceCollection AddApplicationInsightsTelemetry( .Build(); // add initializers: https://github.com/Microsoft/ApplicationInsights-aspnetcore/pull/672 - telemetryConfig.TelemetryInitializers.Add( - new DomainNameRoleInstanceTelemetryInitializer()); - telemetryConfig.TelemetryInitializers.Add( - new HttpDependenciesParsingTelemetryInitializer()); - telemetryConfig.AddApplicationInsightsKubernetesEnricher( - applyOptions: null); + foreach (var initializers in _initializers) + { + telemetryConfig.TelemetryInitializers.Add(initializers); + } // initialize all modules - foreach (var module in s.GetServices()) + foreach (var module in _modules) { module.Initialize(telemetryConfig); } // other config: https://github.com/Microsoft/ApplicationInsights-aspnetcore/blob/de1af6235a4cc365d64cbc78db9bdd2d579a37ee/src/Microsoft.ApplicationInsights.AspNetCore/Implementation/TelemetryConfigurationOptionsSetup.cs#L129 telemetryConfig.ApplicationIdProvider = - s.GetRequiredService(); - - return telemetryConfig; - }); + _serviceProvider.GetRequiredService(); + } + } - services.AddSingleton(s => + private class TelemetryConfigurationOptions : IOptions + { + public TelemetryConfigurationOptions(IEnumerable> configureOptions) { - var module = new DependencyTrackingTelemetryModule(); + this.Value = TelemetryConfiguration.CreateDefault(); - var excludedDomains = module.ExcludeComponentCorrelationHttpHeadersOnDomains; - excludedDomains.Add("core.windows.net"); - excludedDomains.Add("core.chinacloudapi.cn"); - excludedDomains.Add("core.cloudapi.de"); - excludedDomains.Add("core.usgovcloudapi.net"); - - if (module.EnableLegacyCorrelationHeadersInjection) + var configureOptionsArray = configureOptions.ToArray(); + foreach (var c in configureOptionsArray) { - excludedDomains.Add("localhost"); - excludedDomains.Add("127.0.0.1"); + c.Configure(this.Value); } + } - var includedActivities = module.IncludeDiagnosticSourceActivities; - // TODO: in workflow scenario EventHubs activities may not be required. - includedActivities.Add("Microsoft.Azure.EventHubs"); - includedActivities.Add("Microsoft.Azure.ServiceBus"); - - return module; - }); - - services.AddSingleton< - ITelemetryModule, - QuickPulseTelemetryModule>(); - - services.TryAddSingleton< - IApplicationIdProvider, - ApplicationInsightsApplicationIdProvider>(); - - services.TryAddSingleton(); - - return services; + public TelemetryConfiguration Value { get; } } } -} \ No newline at end of file +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json new file mode 100644 index 00000000..bc80d3c4 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json @@ -0,0 +1,16 @@ +{ + "Logging": { + "IncludeScopes": false, + "LogLevel": { + "Default": "Information" + } + }, + "ApplicationInsights": { + "DeveloperMode": true + }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "Debug", + "IncludeScopes": false, + "TrackExceptionsAsExceptionTelemetry": true + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json index 484bbe90..c51d2805 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json @@ -5,6 +5,12 @@ "Default": "Information" } }, + "ApplicationInsights": { + "DeveloperMode": false + }, + "ApplicationInsightsLogger": { + "TelemetryLogLevel": "Error" + }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], From 1effad2dca28a9fc2d25b396b83000cdb870753d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 18 Mar 2019 21:44:31 +0000 Subject: [PATCH 101/246] Merged PR 696: change AppInsights telemetry log level configuration to use provider configuration change AppInsights telemetry log level configuration to use provider configuration - delivery - dronescheduler - workflow --- .../delivery/templates/delivery-deploy.yaml | 2 +- .../templates/dronescheduler-deploy.yaml | 2 +- .../workflow/templates/workflow-deploy.yaml | 2 +- .../Program.cs | 2 +- .../TracingExtensions.cs | 45 ------------------- .../appsettings.Development.json | 10 ++--- .../appsettings.json | 8 ++-- .../Program.cs | 2 +- .../TracingExtensions.cs | 45 ------------------- .../appsettings.Development.json | 10 ++--- .../appsettings.json | 8 ++-- .../Fabrikam.Workflow.Service/Program.cs | 3 +- .../TracingExtensions.cs | 43 ------------------ .../appsettings.Development.json | 10 ++--- .../appsettings.json | 8 ++-- 15 files changed, 37 insertions(+), 163 deletions(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index d1a42748..702d80ae 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -52,7 +52,7 @@ spec: value: {{ .Values.cosmosdb.collectionid }} - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} - - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + - name: LOGGING__ApplicationInsights__LOGLEVEL__DEFAULT value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index 296ae6d0..72a30a66 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -48,7 +48,7 @@ spec: env: - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} - - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + - name: LOGGING__ApplicationInsights__LOGLEVEL__DEFAULT value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 95bdfd95..86095234 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -72,7 +72,7 @@ spec: value: "{{ .Values.servicerequest.maxbulkheadsize }}" - name: SERVICEREQUEST__MAXBULKHEADQUEUESIZE value: "{{ .Values.servicerequest.maxbulkheadqueuesize }}" - - name: APPLICATIONINSIGHTSLOGGER__TELEMETRYLOGLEVEL + - name: LOGGING__ApplicationInsights__LOGLEVEL__DEFAULT value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs index eaab3942..35ca56da 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs @@ -28,7 +28,7 @@ public static IWebHostBuilder CreateWebHostBuilder(string[] args) => .UseStartup() .ConfigureLogging((hostingContext, loggingBuilder) => { - loggingBuilder.AddApplicationInsights(hostingContext.Configuration); + loggingBuilder.AddApplicationInsights(); loggingBuilder.AddSerilog(dispose: true); }) .UseUrls("http://0.0.0.0:8080"); diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs index 838daef0..28eea3d3 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/TracingExtensions.cs @@ -4,17 +4,12 @@ // ------------------------------------------------------------ using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.ApplicationInsights; namespace Fabrikam.DroneDelivery.DeliveryService { public static class TracingExtensions { - private const string AppInsightsSectionName = "ApplicationInsightsLogger"; - public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this IServiceCollection services) { services.Configure( @@ -25,45 +20,5 @@ public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this return services; } - - public static ILoggingBuilder AddApplicationInsights( - this ILoggingBuilder loggingBuilder, - IConfiguration configuration) - { - var aiOptions = - configuration - .GetSection(AppInsightsSectionName) - ?.Get() ?? - new ApplicationInsightsLoggerExtendedOptions(); - - loggingBuilder.AddFilter - ( - "", - aiOptions.TelemetryLogLevel); - - loggingBuilder.AddApplicationInsights( - o => - { - o.IncludeScopes = - aiOptions.IncludeScopes; - o.TrackExceptionsAsExceptionTelemetry = - aiOptions.TrackExceptionsAsExceptionTelemetry; - }); - - return loggingBuilder; - } - - private class ApplicationInsightsLoggerExtendedOptions - : ApplicationInsightsLoggerOptions - { - public ApplicationInsightsLoggerExtendedOptions() - : base() - { - this.TelemetryLogLevel = LogLevel.Warning; - } - - public LogLevel TelemetryLogLevel - { get; set; } - } } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json index 3e5f5a85..6c53351c 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.Development.json @@ -1,15 +1,15 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "None" + } + }, "LogLevel": { "Default": "Debug", "System": "Information", "Microsoft": "Information" } - }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "None", - "IncludeScopes": false, - "TrackExceptionsAsExceptionTelemetry": false } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json index 3b7c8685..9dcddd40 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/appsettings.json @@ -1,13 +1,15 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "Error" + } + }, "LogLevel": { "Default": "Information" } }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "Error" - }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs index d9c11b33..749a53f7 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs @@ -33,7 +33,7 @@ public static IWebHostBuilder CreateWebHostBuilder(string[] args) => .UseStartup() .ConfigureLogging((hostingContext, loggingBuilder) => { - loggingBuilder.AddApplicationInsights(hostingContext.Configuration); + loggingBuilder.AddApplicationInsights(); loggingBuilder.AddSerilog(dispose: true); }) .UseUrls("http://0.0.0.0:8080"); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs index 629c7e96..48faa7e3 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs @@ -4,17 +4,12 @@ // ------------------------------------------------------------ using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.ApplicationInsights; namespace MockDroneScheduler { public static class TracingExtensions { - private const string AppInsightsSectionName = "ApplicationInsightsLogger"; - public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this IServiceCollection services) { services.Configure( @@ -25,45 +20,5 @@ public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this return services; } - - public static ILoggingBuilder AddApplicationInsights( - this ILoggingBuilder loggingBuilder, - IConfiguration configuration) - { - var aiOptions = - configuration - .GetSection(AppInsightsSectionName) - ?.Get() ?? - new ApplicationInsightsLoggerExtendedOptions(); - - loggingBuilder.AddFilter - ( - "", - aiOptions.TelemetryLogLevel); - - loggingBuilder.AddApplicationInsights( - o => - { - o.IncludeScopes = - aiOptions.IncludeScopes; - o.TrackExceptionsAsExceptionTelemetry = - aiOptions.TrackExceptionsAsExceptionTelemetry; - }); - - return loggingBuilder; - } - - private class ApplicationInsightsLoggerExtendedOptions - : ApplicationInsightsLoggerOptions - { - public ApplicationInsightsLoggerExtendedOptions() - : base() - { - this.TelemetryLogLevel = LogLevel.Warning; - } - - public LogLevel TelemetryLogLevel - { get; set; } - } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json index 3e5f5a85..6c53351c 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json @@ -1,15 +1,15 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "None" + } + }, "LogLevel": { "Default": "Debug", "System": "Information", "Microsoft": "Information" } - }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "None", - "IncludeScopes": false, - "TrackExceptionsAsExceptionTelemetry": false } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json index 8e99c47d..8e39a665 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json @@ -1,13 +1,15 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "Error" + } + }, "LogLevel": { "Default": "Information" } }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "Error" - }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs index 07d7af27..312f3b50 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Program.cs @@ -45,7 +45,8 @@ private static IHostBuilder CreateHostBuilder(string[] args) }) .ConfigureLogging((context, builder) => { - builder.AddApplicationInsights(context.Configuration); + builder.AddConfiguration(context.Configuration.GetSection("Logging")); + builder.AddApplicationInsights(); var serilogBuilder = new LoggerConfiguration() .ReadFrom.Configuration(context.Configuration) diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs index f83551a7..fd85d38d 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/TracingExtensions.cs @@ -17,8 +17,6 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.ApplicationInsights; using Microsoft.Extensions.Options; namespace Fabrikam.Workflow.Service @@ -31,8 +29,6 @@ namespace Fabrikam.Workflow.Service /// internal static class TracingExtensions { - private const string AppInsightsSectionName = "ApplicationInsightsLogger"; - public static IServiceCollection AddApplicationInsightsTelemetry( this IServiceCollection services, IConfiguration configuration) @@ -104,45 +100,6 @@ public static IServiceCollection AddApplicationInsightsKubernetesEnricher (this return services; } - public static ILoggingBuilder AddApplicationInsights( - this ILoggingBuilder loggingBuilder, - IConfiguration configuration) - { - var aiOptions = - configuration - .GetSection(AppInsightsSectionName) - ?.Get() ?? - new ApplicationInsightsLoggerExtendedOptions(); - - loggingBuilder.AddFilter - ( - "", - aiOptions.TelemetryLogLevel); - - loggingBuilder.AddApplicationInsights( - o => - { - o.IncludeScopes = - aiOptions.IncludeScopes; - o.TrackExceptionsAsExceptionTelemetry = - aiOptions.TrackExceptionsAsExceptionTelemetry; - }); - - return loggingBuilder; - } - - private class ApplicationInsightsLoggerExtendedOptions - : ApplicationInsightsLoggerOptions - { - public ApplicationInsightsLoggerExtendedOptions() - : base() - { - this.TelemetryLogLevel = LogLevel.Warning; - } - public LogLevel TelemetryLogLevel - { get; set; } - } - private class TelemetryConfigurationOptionsSetup : IConfigureOptions { private const string CustomKeyVaultAppInsightsIKey = "ApplicationInsights-InstrumentationKey"; diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json index bc80d3c4..91dfd50f 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.Development.json @@ -1,16 +1,16 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "None" + } + }, "LogLevel": { "Default": "Information" } }, "ApplicationInsights": { "DeveloperMode": true - }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "Debug", - "IncludeScopes": false, - "TrackExceptionsAsExceptionTelemetry": true } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json index c51d2805..63622fef 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/appsettings.json @@ -1,6 +1,11 @@ { "Logging": { "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "Error" + } + }, "LogLevel": { "Default": "Information" } @@ -8,9 +13,6 @@ "ApplicationInsights": { "DeveloperMode": false }, - "ApplicationInsightsLogger": { - "TelemetryLogLevel": "Error" - }, "Serilog": { "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], From feddf682fb8063cdfa14548641635c506717a492 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Wed, 20 Mar 2019 16:48:40 +0000 Subject: [PATCH 102/246] Merged PR 698: Update diagram to match current design Update diagram to match current design --- architecture.png | Bin 28961 -> 8615 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/architecture.png b/architecture.png index 4d00150eff21664ba9eceb73236c0b1f189f3cf1..58b585095c85e34e1310aace136d8d72ede26b8e 100644 GIT binary patch literal 8615 zcmd6Nhg*|N6EAi}#bdD1R3sFsN|$P(OA(|91Q4Z{Kxj%6D;SWLPz?3ZK?J0Q&=o=r zO*$k2sX4Tubcn#+fZzGP`xo5H^Z4NIJ3BiwJ2SiU8;}pQRZlarGBPkQoK{y;(qmva z=*z%xfZ_N-Fr#G3YXN>8aMx44%aGr}J_81aZSQE^VPGf>Kl$782pFGmRWo*HV1Rhg z{~Z{yx$>NWfm2Rh>5l$$^98cvd#;wDreCKA8ovdy!GGMg3~G0KtQ_>U<*l3R4c8l* z9>;}sJs8>lLEWoAl61fAB{R3k@u!!LTx^a!;&p@aG91PXB)~ zc&s|EFjD1KRZDw3mpoBVrWr|vV_i*B19r~lC*qGV%xjaNHuaNoP!AnxMEE-}Bgci% z@={QAQ-=XvU0ugjPMQm_)lrl9?8tZtS@i59Y|u)^{Q9%-j&F6hvafBC-%K2}R>kCs z=DENp3rNPPfkOc=rcvnWN)aSr_nyL&uO4^inH1cEdlM}{dS^PLYgyAkHdy_EE&Mg( zl{9pIQ4J?_d9h8UL05N`w!xaNV9&guAuOH4*$d&bmt5hL^Dkvp-FKu{Be(&R;q9WSra5D>@U^f?e-8c+;@^ z9%pfPa9hTscjzJEo~_ojC0mWU9&)69%x=$GMfzXJlMECc!$L{M)~XlVTU081(I}c( zw-+jlQpqShr~D`i2iKvkMQD+_r3GA{mMvxbjW-TNJ0@)IJ~m#PiI|W&7BZDp~I92{$PZp1BO^mWOd!R%#Yt=2}R52?9`){Hjahy6@ZN6x+yEPCm<>E9_A;Em6_ia78iLA^_(NaLSjH>ElQ zH_g<67}MQPOWyP^a(M%D<^@uhPPOg!R-|W+E)ws)#T$T_{K^v7NhvCcR&@5h2r)VU zq_kD;FQ^}TU|3evSZd&V6>**(#|nnI(TlW+UxP90D&zcgj+&&=QQxHXl@%*O=binj z$s>1MJv}{reNC&>9dw}i2SKc86Q30R;~_&e^=G=hPo+HR)+igP)%M5lPfampJ2**^?u5AMM20^rH5qcd2SxZ+j~w)j)6N1uCjPkE{C+=E zP<^}2)7&^Nf`~e-{BmzAiCK00;LD^Tr~VlaEAmqU4hm@PBF-;>pT0hWhf?Y(2r$!2 zotbG_uA*8U+R3>c{_s-=>9n;1J z@byQtm9p%mQ&ye=jxf_B^z(-0_G!dm(JlCxSC@cKsJaIPh#&}BdVF+eJ$v|0*W4=H z#id-G(Eu3fdCsQa$czV0osme-Nz%sOnS$=YQFRJ*m+f*Wo%O1ud(Nvo`WRnd|AyjC znCgysWkz3{boF>X_$ep=%sT0*NKEiTh<27(e7h6NARE}5#>x~#hR4RIyIunRm2orNGqskY( zbWnqFL*b5*I2jX(PA5sW&2kGfGX(macqC3kjnsb);aOsvbTb*+aE4>+zoep_ADSlM zW#Fhzc=JsjxI=EMJ_yZ+K*~zhU>u>HVta1Tf(I$q>3nK|GAhXWm5f%_L$O`-f4GXv zcD7NLwX?VahDD!-TU~)51yPvdWhyr?#t=0+~%hc zn;|5nzJUB6{wX)W&W8P^MHSH!`FVB1r+KJBmFsIE8BZ1$6$bq&@9=#pJ8<4!WDPye7aP}XQiofTmr*mu7e9+L%NYy* z)oAu>Y?;S~B2x8LWY?FbVl7%ZAr+a&{DIC~9b(gS@WzQkp@}2IRgLZQg2pMCLkZL` zL8`mV$+M# z7_xT2q8)c8uI%#haw9rN3a42%ExnV8Ze%&UHWKhI5^WR~>hFcptX;u-h@Y zD_XLSJ-S!TDvq7ZQlTz(D={9Y7uD*2cL1L!E~YwU;tAx8?G(H7*9y&l(;I>AJzFY> zKt~A9d!+S#(JB%m+cex`O^wBAZKKX6l|?ceb!ozeMp!CzhXYRsBL&!G?+rLV08@j2 zJE?Wem%D zqSCnv!-t(GAx7~p#)G>7U&*}B8ZcefiiXxt`12Vl@gb&&BxJnrv}(h;X;z@T6>qE( zb}Iog;D3(en{j(4e{Q7fSq zQy##T`qoDST!I_DdEnv|C{F9ZxMh##@b7u|dQZ9LNHloF>W`dfBHh{$i;Wa2=atSs z1kS%)jpT;b2baN-P;NNN!O~w&07um2Ok}8gDq5M!0biIf8@}HyY8tGafYI|Z@BTi+ zO9e3kRp(JYA;w&>loi366I-2$xz*}K7=g+*oKgIJ)f*$|fDWZUf+4Bc@BR4gp5z^? z4m)t0b8@uiWNd^7{H~XjA>u;=jpV5FFFhKqrAJ5zb9tr2&I4+dZKsUZHEE|_lqnDD z4l8NctE#Sk_U1W$aI^gQ&=&)G*8aN?uaz`Cl*5&Gp@XyB?AOhi!Z0V(tf6O@@^ zpP*JCBQ06xIMo^SOlBtDGtf=+I62g+WI+5w@7m2GV{XU@(*3Pf83YwCWV3F~sYCNPH z?{#qa(9Yo;o$+ zc`xg?3kiem+869|($QA74cvXV#Sq-HFE+-GnVM=1s}Y`z^N6U%w+c zcECn3gBbt5$&=6FK$R9FVhXXle@Q$Q*Cfr~ZpUgcjdT~w-`xq0)5v@HsiDK$hWAFFnz5Z?d(S;rbd&r2^D?B{9&Hh=Nt zyKQ-NK+~fZx@Vw?Z$AmoZ1h5m4$e1$R16<)U7MIlBbOaB?uhHOl$3971X)`6q(!Cg znbCmV>Vy>-vlOd-9fy(4ds~&^ntQ$1RXCb_i}Kdkeh(2M=2oUr3X$@BT|K410O@!##`{~l_Xg=oy!=;ts@!{ovxgJ)muFb>6+MKy zYGw>B=%)Nfy!^{}9CjAtphe2f%@rCoFn1Y`ONr$${2t*I_}r?1GfpTjOXX1;cG%^o zYEafxw!uznSdItCh z#QEDfq(s?X&)=0`ApA}j0*-Jhw8^77NHLwLk9S!;qs0o=jG>@A_}uZ)!Wi# z&>yrXH2t3+PH@>isDw!Q&)-$u$-*8+60gb@vBRk-oD9la z#pbtmR`DZ2P5Ierp8PSu6Y0n$#Ego}R!EvuB$3~lVz8F%_@H;PDB(H|dq0~wvV6xa ztw5D{YFIElZFZ%{CM?Gc>3gD$OSrg=Px5j&+Ty2m7jgJI9|jpi$jD<2%#4$sq+6m=_EUB@i6#UV!&>@ z%Zud}-sIm?A_0YpNgv0r1c&#f?PIm)cx9iA5)C3zv!cC#mj zt3@jD^wBr}ApfF2gS(gK1`3=Pa*bMSn+Q411}oMzRI{6`IMg;JKwfm#ECr=9w8jokjuX8a8{1S<3IyoDVMSCGbIv1&qhBePmgU6cZjW)*-gMnXubx@K6)>ZnvpD z_@S_R&j+VoZs%6E*j`x{{`ijkZVm(TwD3fB#|?yO7+{cSa}D0l`WfLlIgOe0vNlz5 zCgDAc`?I+b+*JwljA?D^H7iE=Q^<>>dA#v-no-0Pop?ut2_E8|aSy>1oG=M`W5 zA?+HXG!l23G5MLbv10^IMuT2cfU?wgls&8i3cu+X>eG+oS%6@!y02mW`m6~X^NQ+^ zvyxc45*NiDImB7oF_uBLsl4^+0u;U_qJ8BUMOTx*J?*hcP_oLL0(7hfmqL3fvz^{f zkQul)E}6~m7-|DquWPK$jnDRC)aWU& zoh6QNo!1t=Ci`xXg?B*5kD{hb@rhF-jlY3T{u0cq1qdj0Dm3613A`^}?;|onZ_{`H zm^XBGL`I7jE*<)aia#swU!f<-3NB8IqfBx()9}1C6u~u4m!nu+#bVD{ITBVRjcQ~3a&&7TcWFYX2b&`xH5{bDRY$dKr?a+M9Q-(6U zZ<(W}v{^;%G5wUEC)A+%s7%PSfj|M4_t+5@B8K(Z?IuwUNQ9vl1wbTIGt(3;ai~ zgsNaaTy*3^xMO1D`A`=ypWDO`X7(S>3%e;8(!!hV-W}=WKZ+R zP7^AEJIXq*o4msJA@JTR5F@u(<2gQ7HJF$?j`^Or^`w&Ta&d&XxY&JoP@YI4oL?4% zhl8Uy!ac2rJatNi_gW}UH@{KAH1)}he2d$sg)BP&qBS8Xm?^gz?-37~Qfu~4NPOtv z39h)0b?T|&huy!imY&Lr^sCMjUdSqPmj`v@D&ARt%)Z+O6g6r8DQbTG3M9wc2_o)| zX`Vf1H?ug=kOOV)M{gg7LX29tx#7}Zb$DIXVcZh!VoSy725OAUQ@j`OtRn#jA;{dx zdGLnlDI`5iCxfDYwOtILhU@+{yFYFKyROKVj~&gD4Z^Q-BiTv`1;!b|Ic{6(@zM0q z?ekhg`c{BNrJxL?o$;x@JR0zG&t*C387NPAeD=Xd+v(+UinrhHx8naT@ja$lj3-(( zjQT4{+NHd7AS^d@59$b-mJ_=iUP%@n4ft~Af9IFvo{oGW=y?5K#5-~ThMo)}F0OCx zyb;>>y)`J@Ew*YIsatM3hhCTz-s!HUV)-N$P6Lx=PGe>~_(RNUcK>FigL-}aDE#S% zH<|E!oLVVzs@8;>^V)Hlp4>uODP0@lAa}BY98(1X;}3NhzTGPQppun{p?ki7h6F<2&u!^ zq9WVM2339u6j^w)4Fumc!)EX~`T<~t!#Um-3g`9!NVU+RRW3wy2@k?dd))j6&cGsC|3Pwey)qjuS4T&@swLD&T?-3& zzwPEbPn3FZZ+SY7Vz@5y7)kFzSE?UzOCedmUj$YBx_ejl$eqUlce8tXdeHuOqJ!LJ zf=eSw``y-CfUf;Tn9{o&yK8-03!r{#04e4e^Wtac0AIX|+M*kPcf6dSqn7wRywGp= zK-hZ#@VUje+eou(P?xrkNizcY5BuM&>&i!m1CPJ{dtmPL(L*)63?g>`dT7prKH)Oh za36q}UkX4UpmUkJxaiFCoCy#oSr!kGMn=rd%zg!$$z23ncS^>$%Oo;^1;CdXT;H$y zEYNoB#EYySUj+0_yhwp^!g|!>D*zI#YIZAMk#mtr92R}>3P={q`1a!2vGFI24#4Q( z?cIJ{Q8x!u5J~$8otiat55-%*avQLEPyo^iZ3h6SLKAubVSP{uFw=qp+P1=IfNKjd z=L6A1ZTZd#+8N;sKq()E7r+0Z?L3#mPGvevvXy^4>wtYEIQg9iao{%5ReIj5()oFP z{$_$8+sW*v$2XpXUY*=9kU{+R1$l6zNyhKt(gg(t&IeXEHhysdmRgLjf6UI#uB~Ce zNzJ47k@0r`Y3^K(^Qjl!2h*_;xah9ve@d;D_px*nt37M4`N&WGyC!iDC~lKg_t;Av z40dU&KI8>2I+{N4VXq%?+c&i7>FBz3CU$E04^1gOnI=I~g$Ca2lMpeBihr$=wg0pa z#MSt=YAkxzuk6!Hv*h-DA};g;JO61xwAd80Fa?3;3T`yuPZMJ0qttwBGWPeAKO}m8 z9R}tZg&+84m%G{Eup;6VT!h6C3>@A z$cYCb02;ILf&tI=pwa1UJgxGKmqlq)d*2i<%j*)ljk&!lV*G;rlw5M~y0F9!! zd;Ja@w+$b1tQsQ4ETemBp_Nk#L3inlkDZj-t-gZP>{669b(xtCkmh0TkAoY60e22y z`aF5{g6?NDV8vafr5%J0`{oK{yp!y)cXfv*RIC#}iUl2Ov6Sj-%G4x2s95Bd=QR>e zc+ov=ofu2+AYBuTB@7f;5Miz#t*Nh&Y!pQx+}KmW9FP2FEkIYf5n$tMx78S%d4y(@{%MFgRN9~ zzVnB9M(Nj-P@V_|S!oc$8BT7<5{gfv}vC#zht3mD9HzxcC!N<(x5t)s9xU)<^Bj4AR9>ER*82IdfK z3@nK;mlL%GWUnWcyE>KQnl7$86;zcZw|3C;*h|W2E_+N*U!oPNxR>T?&(o%+gq%6>hNd@DMHpIOe(zO{mw^7AFE zB-%?46{2gl^N3LXpsb-r-Uw!(H?@M66qu_z3_^V{5w|dI^J4~_5JOfp&eRZ>rvh5{ z+B}Gbxw6fH*E)v)k#wn1v<@@)&sBs3kdOglM4$bCKRC`juv~R^R{YW_x`%pz%R}xm l(dxD{C;>YacYp6TyV`mdjfG@_KdxXeEB1nU%AW}+6gMh?JDGJiDbSVNNQUcN?&C-qJ z(j8KQEb%$_s>sXd`ycpT*Y^kY^4|MCXU@!=nK_S{Io>Ktw+YTro`Jz&1hO)>RAI0a zH89xms#C|olk&kxAMnpHTh-e)VVSK|bKpNGp4?Ep0fS`+p51+X68!(PwTy->48|}E z{W&87touy%M^ZDiI(bdEb9ReQ83^xB|(gBTO!P*N7*{W&-#}n99 zrK3GfL%Ca|=fb(E4drH4rSB#e%isP&%4jHe#>f1&?A+S2zBp%{TV<(x_MWSF?TUjV zSKX^V!F`(j?LOUOF!$%-O;Z;?Z?3LBHX?y}okPN4g&+GC_KvrH?l*5XI6lgvM*@S9 z@9pntA|CNlMyW@;cZ#$jQS0mL`tb8Gb?gT+`XMy#C*edeDTU*((JKg8Z=?w90@rDn zyYK9=2l{I;l^O;ZjN?7tBYzZ*1oo3RH12%={|Emc;gYEqQS)2F=Wct5_?V_bYCRJ7 zEAV45>pNYDuU`ye%xpy8iV@*<**^WR@Zlt{eqnlujTUf!7RR;3desb ztEmD}(mdM}iKsK7)4sPO?H>MM{5o!EH=MI}t05!tG0$hsyGHb9^tfjm9J`kD={t&q zBHyC;z-oSfE=M$(q~EGi`KU45P+?+`g>yEE|AX(IEiQyDCaWt=pjg5F#hi$P zaPPWFcXV;Qf@nMm^5(L@12N*vj1S;5AM83dVCh^=uwMf2!1-J^dFZ5yyf z;@>Q>1y&BdH;e}F{1Fl1s>{LV=6$)Z>mmr4{htv@o2GL57G!bXxb3940fXiBo5%r9 z#0}L=jLTo!Cec)@8+6ag%IYr-?!9tlmnr(quOE42-&lS*b$MlF>fw=32QQReP8Aaq zGoU!~Fb#Sr6xV8B5>|V}Y_dPyJ}JAoz5VfQ*WU#WjD9~G65Q6NtR4T?khYr}>IB`3 zpFve2SdWbB5$1eDRa8_oDy9QR00)VO-Mq$0Yn(FDWWKM9xJ2D~uKnVn`?TQPN!gwy z`xbx6BTJapU?Yp4c^&+q*O-@^pYRBuDNSz{HIfhG z=gM^2hK#(9l1WQT8>hr%UUze1^&4B6B+*yOKU9^jtgKYiCTL}-8Ff$}8EwMF#RXNl zhu&>LpBRhmSlMc^;W|R05mF!FqG7`w?>$Ay)1=h{* zz80fI_1dHBSe%=aC0ux37Jo09;*4%uX-Uahp|!p&4CXmc^sOWGeF_XD68x|w!im`v z>I@%<%JHL+JL1-OJ8g=(d0sXf@$iJM?(dfmTeiGrII@g&nJe?h_5c?pnT~$dR$c8u z)k5wE+3%Y9$MkCG75Zw{|tjZ=D3q!mradV~peLDdeTUXupgT$1@m z^rvMMQywq6HaCaDk9%=M3iybT2AwRX`zD^P5=U0(zP05b=)5JeqI&!Lk->{KIU?ee zZWE%P&`it_U41o?M=J$(1T-d-71VUt_S)Y*h1kNX@& z!V~QU?a}7fjb?8ip_7j4i>A=!K zmcz7uYY!};enDAT*=V*?M@NVEH0*4I5c;aSG_ZBBkJKhI0e#0{e7AvxxCe)o0Xf0Tjp3U``C^|5lH?S%1IIhR|yK=TVid>h3!MaI-ot4lYLZjtn z8{JwA!ApM7OJq8_y2%V~2m56}#KbTdpEQ`k`gv}@f?Uq86`X)cltGcs&^Ob(H4$aK zwY|Mz{9`bc=TInAFo|)^>O+P&BM{tR7T#lEmapuhVq$W#Bg4b8_oD$d(e7ZPQ6o6g zJ2Yv=NnBjq=oHM|^DCGGC<>daj?Fn}*Od0?;%I1LoIlUwrs1};%95heUpk+as zj&W0>OsGM@;NCBu7ruZVcIssZgavHCfC#I$OCL@*Olg;E_ zq_FY{t9jhZ5DUzAcXzQmS5Uyu)6I;KYi9Gll6$KYgDKH?ANzi@6c`1@#-eTngJ%mn zk|U31?tbnqL2?b{l&-5nL@mj^cX^dhWYyt7A`^&O!61S}sR&zCx@4%pE?^HKeA{OL zjNDYfRTjo)shg3nRC2SjN;AI^3jz&>&5SHRq*7Rc@@HD|~g@ zQpNinfm{E54*6;Vob9Acan~G`YgyX#5v#)#q5Mzj#$A^zyd+{W&4sP0nZP3kCs+fJJw65T`?Ej^2c);Vw5J6BLUzR_*omUKi9;x-@{p{K$Sd#+r+po4mB#Qx{j)=mau|fG1GI-jF?P z$yXqq7))VGenbm2AnIWVkClg$7PS-PyJF-m;`SvZz7y8rc?KJRc2M%S#XT$fyF>E-sSJBt914C zz~ALPjH2VE;&NgH?0^m#-}|O)u2NjBug~po#!vY92E_^%3Wb8B#Y^^Sc=d@L4fqM9 zM<086;R$k=S5(X-hWP8-Jl$t&8_m*fTd&yf*}a4R!B%fg==Y=oVD(LppB4-b125G; z0D=-uPR{9@`X{-rzVyL=#aLAGuJ0UD9qdGCwO_S$b#-kNhUV-*JVjw6je~duortO) z;1l4st=nDntSn5c|IC<;cLjp*r=X=e!02&Rsf^Es&UTi_?alP+#hN9;+-E0LW5&DFWdr5V^S#`w^%_I!;ERa|YWCYjO-)>t6yQ&EP>wSo6bT->* z=lCf7n}gD#Ns$ZM^VBBQ$f!vCRvC(2x>ZAQ6lQ?&@%*J1oo`3+rN5kzd>G#Koovi~ zWJhpOMO$ozuBU|Y*jLDM5EcPXp>Ovyt7!2vyvAUhv8;K)o&4N0-L)_o+qTtKP5E=v zWyBPkb@(z*rcdbU2u}N-R2VnvJqFtZr#+Y!7fs+4aoOOK<@HNElb@w)lFF0FOYtw} z$DL3cDN$~FtjCR^e1}4La1zdQW2Blb)~rHl1DrT-kTJqx-2`ACSXTiTgtMWPlB>nh z5;PdS6@?m}A?AFjQMk{(hpt`l8J&s-F(B+fv2xwz2F`*{G zPDy}v%4F$uZBlz{v4JtQqom3Fr&-}=$8X5v%2Mq9ERAgLW?(;!_4N`>Kg-=59GL27 z0s{k$P7(N{s*8*g%@v~ks2RixqL#}&s2>#0p){$#U50(U1U5vw4G55I97h&5lUQ1a z-^eBw4wLvty!B=(`}m-AZZ)PO1QrdQ3%~Rma0h$XwymqBg;Cu6SZQTjW?*JQG!4#J zQbLy`i}_~i4cO?a#V@3TI0&4KU|)J|ih%32u@eIxg81g((vOORt%H#zdozsb5<*Pz z?de;dRgNVF^3igkVeoRey@h9DdXt_0B7G=RtwCkp2RPAixzfnU$nx^CW9q>zp)|_x zyT_-1D|EL9VZqM$AQUUNBHyAg7z}V%yAOaJXSmuzx~bpEy8YpjD8G#3RX?%Z8pA~v z4v}X5Ez(`C!V7`D!#`>woRsR(m63j;Nsmspp%_M_EO;JHyyrB#Ek59vZ>UbMr+N@P z`Tju9Lk)@)m+ai!IO}h%>Du-`PU|t{j+!kXP6c+!R@)-OBVzx(Cq4Dhyk9Zfgq$ENPpOpUqFZbI!#~;0r5v!h3jlQZ)90(7#_ZXr0BJR*~x6h zb%uW@$sb&rnVFfGNOUz>y0hns9wL!u*ULFg7ph{SFuxgf@;x_C9}}Znq&HCwF4{d5 z^P*>z<~>o}{P5EH%Jvh}mXzx)SDem<`JujdRgJM6_o(D`!HMx&ry6nCr;;~U?~tDN ziu72xX_J-ujCzpe3xrn~6QNiH{et7ow`9+5$ZL32C(z`FO*)Od4%i=L5V0%2Uw88M zObN~gZnD8tmcl%V_gfV(HK=~qN~H&~;>nCf|& zqL0ZIbf0b;|7MB#vB?b0B(K~{MI~L6 z6G%_A_6WPRV_Lf^Br<~VrsVP-TW}CbxQ=jgIvp^e6ybu-vP-46!mX>-W`tMgzNle9 zOH-IpmEHJ+S{qDeMypPgC^oQtY$Eliy_(!?PiE0F63pn6spL;CLd6qH)v`Opz4g>C zP2Q69A>>JUM~reiIOCmMT=*o}NnMyX7;I+Wijp@-J%?GQZd+usm?JCg;wv2CKF6ow zf~bg6_*UJEDjJpt>j<&8xK_mkdIE*XZ-xu4EzWg4KgoGtcWlhSZF|lzz|yWi}jWsmtVA@mXHjoe=dmq1@(dx2cdXmKuMhX zp=w8w#2eWMC3h60heL9$btIyuHiaMCB#CWA0@R*a)JYy1ZtEwyUE`)yv>j5|2Ye|gEu9KY#X!MVtU`#s0Pq&qM9Q9J@v(#!wRUWCld4E3< z6LCh;qtp%Qc|E-R;y>fkOX*5YOsj{uy1MRHi|tGz(9^{z9jAU8n1r<+>KWO8$&!U(6Tbu20xUl#Zy+esMv;nn-Q8l+~{hn3G-xdqH+;cqrrgK z_?2M=nuutCq_Nc0F4cjBM%en9?>=vO;Jda`C!Qrq2KrdFhx~K2KBwQ;)cEQU;MeK< zn!S%Pdv!v2&Ft|Rrl*hhU#m0Zn;bk$(mI!KZj|xb%=ZMVtmUk8obyoC{@7w9tZ1;6H}hhkz9e zT7d~EjI)c2iFlJ6IS4xyeZ-_R_L=TVDJ*CCj^JEmk3^_z{SbYMDHwe1>nZQO%iora z_TV43YTGC9Mr|gwo}!htdeBNeEy}Uyt|iBIHc&)I+puwYx@bj&Ej#W7($x=U4Iu{Y z7ZW>V2tATtg4m71l+Swx&?dU==IAK!{o@qN&+OO@x=t2-RS$U5+2u~zGtB!I)ZTfK zXF6pD*)QUIPYy4s(Q0a>k5`!`b8K=DxrsK2&Dj;r5<2LfOkAUh7^>XIo$kH!ncA)N z#@xxiR}*tFF|<+>wx3F^U9RbX#)`=LP%chZ!49&nD@haOt6fD;tx` zn=i$YR5HTJpDfs?LY*Hsvr%fY$<@?YC`#AXQBMQAY+3gqcPXl3}WP%1U1a|_ID9w8mp{6 z%>MrMMDqLE3S`=ASx?~Vh$br&s)AOoCN!w4Uool>kt=^4J zdwW?|+2fv}-A~x0TL1B5di1K!F$0Ko#g_=ZY`!g2RW8^R`6U$Al#nm(=?=Rzqo6+56Vb4on48<_HU8vonZV>ynZE-%wHyl88&{^tTIoLZtdceVw z!ZZ=Uv?(Iz`nzO$hY|`=6&!B7=SA&FzdR?fc5_)B_a^8#dooSY!e*cT)MN&u+uV?s z`sZ`2hG)s{2s#*U>5IJlb-;xdyw zzwoig^cjfv1GBs4cz2wcY9HTsjFW}SMt!H+8Bj*3?S0E(WNk=qjt!__mc#2ZDJ=db ziW|{+mDHl&(&eW-jbn2O->k!;h!;QWmZPK-f7{PHp+l__DB1}qvV=l|?$()^6sk{h zhU@@)?dAYL(T+wt795t2nUi=xU1)hQ5PL%Ln?jUflbXusPsZAhwOjgDZ~BnwwxxIr zE0(=8Nk=Xqdko1%HX=QQ{VZw9E(V5$&70;E$@Ne&0}7H7z3OTAY8%vxy?P63`Vb%O{cMC z=kk?`($WSG5DNJ*wY*OFuZir;<>(WCoZ7cMbrGCkoK zP%F_pK|!~`dGJ>ufs3#);MxvT76bL>*;h3HK%~eJg_a5dRMt2o+N#{~Gkl^(V&dYP zydj-Nc|!h^cOdd0OzNGfZ|zNP=?2 z3n5Puq=1c^83_b?uf`S@(#%U89UYBpOc-7){8mcqXM;;>fnv8IXpB%$ueZ%f4xfLP zH{|Ok7$4$SVKD$)z2ZX*{4qiI?I=+?S$*kUGROOVXA?(@>n5_P9PCSmji?cp1%u$z zH7`$BePU~Z--?lGoc+K!5CwR39wazK{Q;&=uouK+E&AtosQ|uB(f2Jo9}~x??`~~~CHvwlBq~D^H()Fgmd#8M5)P!N~us&n73LHUPr$3+5)uu06m^NX^ z!vj($uO;kzDbit7q|cETUmfRm2*Xw&tN{M{72%BLKO1IkC%Qu^#&sHkp|wJ&JWZ3q z>6GW5en?4cytL}u4~afHngKdZ?5W10JNC074BNgtOWm6nHlb(|ptw}KqPXYqsc8G-PR&*U#P{g`AiT5@bA!m$6@YwAr))d-q|^$JA=u4z}Bq7UIv0X z`yUvWkrX_bJ0G-Fj;R{%j8&ry^gQRE7Pb z`I&_UdE0`)rQY4VyD%6f6cohDSD-%hEl>+2ekG7y2#`)3ooe{b8?reh173h|me{Gi zJ?$$QU2_-WFk|@!X#g&3 zh(tNGbV{%mM}*M@FcXB63A_NY&j){q3zmx4=cc9@9R<0ESd{2b!zB11F`tf)k5B4H zl<6pz+X1*2Ro4Iqn;Ik}c|d(tCX^ToWz74~4?q3H(^4*<@v$!vO{s1%qM!t0Gy1hG&5xh30;` zwY5cJwT6zVz-=&L0J+1{kXUPgwW!f77%a0W-P_%@rUdx>JS0#m?wA&a5s<9c%f2RA z@crk{)RbV5;o(mSJ^_122w9wjw{K|`3n7JInE?sdz*{0>Vm8qCrXVbVO(K1|?h43N zMI|);GK;(0+t(6duy?r7w_+}$Di4>{bc=bS6gCVPfQP_}rWiT^2r^72jf=FE$ku{2 z06@DR2E;1h4>AONkkRjia0CWNWUc@QnJ=(H0GWpP-;EFWFZ0d(`hX3@f%`Jbl2<@c zf*|zWGbndSk~IEhs(I>Dx>_3e3iwd$S5Wrz+a-VLPMcxNf^qeeAQgz6s)eM+z;2)8 z3+X{!-~~bJKINJON|4Lnon5fF9)s#_9Qw#TlRwCg<%ajm$nJ`&4!xcVZ34b}ak~5I z=P;wDd7XW@Oh|8vVzE>;tO7_b>8o-k8iK7XQdGmB$_p2?&yu+N;|6+*Zyp54E$y4o zremGlbA1c!F05Uk#-QmOvO!8g?ZZ{fJKiMolnn#ECw4>T*E>uYz^buJQ@o{+PsQb} zRbyaCnN6ewKi@CYw*JY<3D`$@Wzj1KQ;>S_l5_Rj)Gz=;8kTw>25tVb4^T=$+kXWE z#sZE9yJmms!?8@uiR4=6d|!1s8qJ!5FhQR11}=Y1@%aZ-k=)0ooK22rE@GQ+z!g;2NeqjGh9mogS_h@-7_Z8%b9v3J9!1p zdUw#p8nDz8?1Qv&C1%!$2Vuf%B7r}_W^5uxvfoy?Le8ZTg;@4T>yfLldje(1SskB0 zN0tzSAU7U3C9xasuS-gVmF98W|ITeq8#++|;yX>ouCMso14g;QUTh9@0{#XTbG~)% zc|dh}qF{wSM?U~gI!Cq>scu>dvVwb=Adb+mr9^JP*s!%DhvXKxSc4$zhs>h(1hdNg zy^8MJA|fK+wT1a^|EV$142ZnNp_jv*H-KlTzmXV%Wp%4*UU5kYGOS5kYcnyj@WP)m zh(p1&i52#n7Y}EeH-jBXQA`yjADvpJvx314`n@0dlg~TU2sq~53MPl3?^_7S1kqMq z;Lh64jKe)FL-@nC=tYm6JJN#tC5tl`Bf!2H07+jE2vbN&w*M}*L4iuFVBg4e1?nF+ zP$K=FAbY*^#FK+gNd9pJdL}Z*``@Xjr?JYd%mV_Bn1oyqg`SqtNGYsix^{=)-$fNI zi4_i8f^@pMyU2vmta)xfn~;B!h!Bqt?qPMvA5sUGJ$8BofF&URsY9iK0-;g5iZvyQ z?6BhE&x@CqJwTnq^1ynjFV%N|L%9SayJYfl|Ae4_)Nja?wi`nV@jh;BZ%-7=3zGLz z)qiSY{;a!M@kb>%g*{ok7+7!ApaxtoNQ42+Qv7$_A9h?+5Xu$60||%u2ZA6oSY6=G zVRg!(dhCe_Q2I~OOsT9ib2b1ZGHmmdY!`s^q&u$s$+8~4yDG-qmO7*jl2pKl%-Y$w zUyPGp+vRV3e^}jeDDOrW>%50^eUq{4%>Ww%Zbu|V=8oHcM(l_9z4=`!UW>=>3VQ3A z-kULNVC*#m|CGi2F%~R?c$>oNAXhm|p6QAWTbOl+Lpgr zaz_CVf%h*D6;5K_jl2f&1Gi-IFPmK^m=bJ9@UKX;p3{bU2#S&>)%In|IRjr2@$X2@ z_qTvO+@zu*A|f*SNLBRMj#9S&x)S{y{9n3?<{?HKOxfIX=;Y7J{A<*g|Fa1emOR!r zP|?D1t+G!%LN(`17^WTsb;$??{L7TgFGPSB3tL&?ZY025iLH#Ut_F9>y!_95*+g=9 z^TDwqfkv~DQ#KTF(jha{KL5vCph!aFNd=nI2_6Wt=PnfM%gX^>GFNW?YbnUlc=Z$6 z>Fw*K++I(A#lFV8YF4QOXE(OXJ9}a&!MB|L)qLA& zv5!x@L*<%9r6z%~=W9AXzPTCJB{R+tY834nC0T=ILaIH`FHW%CpP#@TV{tmj9 z(#>>3SX&A>ly(C0NeP8BqPxcRx^viCWPU%?p3L^S+R>YjzQ-N}|D3$(F&bp=U~Lxu zhJlQ)=>sD=o&Tajb>8AW0ng(tb(J4<%zLj43ym*tce_m!Dm#CUkW}bx2ks)N$^!kt z_#ASKp|$m(v$f16+qwl5j6{(joUc>*+Zp8c&w4S{0_RG2JYrm0A-ckSOMWqs`X&8~ z(c&7bmu99R)_z_z|8E5=c42ri*J^5d$lz-DI(u|ufWY@4hYGr$G)rGcdCt4 z*ppPWYriloCgghd$LmgpWIa!>vs@nS|82O}(!ty@u$kI|toQ9WV$->5`uO|IH%b!pP{`u zZ>%1`*YcTyc#3xM^*JQpg<6B8oljqWHYc@?m+lVo@Mu$73B7GgQFZB1zv`*}8c)|| zma%Vne^oq>b$*1LBU_Up`&GZsx8pm*{;RL6!jiUFwbxw>x|dM9JIuvqwzi^<3Rqo|2e)MKSZ9_N1{wox?#8 zg_6G~_ts#E)gS6GsQ!Vo=D(1A&21YQ`b zaij3f2wPdxr6^&-2>C2#mx*tPmSjl>3(NSj^~8W>4;$VOR!>}jpGE)(+$q^r!lTAb zT`Q3#E&#Q=SN2GBF6LF!^q!DRMv`Nk*K!6hP_R$?Tv0pnmpQgpy!0^WfoVzQu}34f za|m~(kmV&q1m{yLnB>I1HPOS(J}M|kXWk3z1;7SC*FzUbtZj+@3o+o?5;p;-BPdP7=_xr~S0@ zlN}@cZpD=~tEZ0S${9D^^e5S3wUqia7Dv_ksDLTVy9RGbna)+UYW8*Oksjf= z&zQ>U;cE69nSqPn@u)`dHodpk-yZCyi0{enk78rtBEnYT-IDR$PoUhidRRDreR`$Y z%VD+KDr$p;9yMhbmrNv239824MUxK>;4_IWrgRZ)2V z5vMYd0a)=Q(Z|!Gn9@AodsB>hs;|xuH>0=TuD`MsEhA#Ew6n8n(3Walvf*cRNEy-= zLG(q%M8Mk$hpzF%LAAuxr-O~R`^a5&;ER(&cfFVvKG|J?SH}rs&R#i)xYL2KadVm6 z6O-9v5!{bS&i*AulkldWhUw}xz1E9ap;zWA7|-+~M;>Xjt(-IHQ~6P;Go-t}7BL#% zAw6?%>b0IK)90vftKu^aWkN&u?S&-}%FQxt)Tl+vpR95#ue_H85q*r|%qka?!J?>3yR8YZ)hr9%D{-m#uTDt1gm%7l@;E!6e$JC4Oa~0kOv5HmWR8g=4f0pKyJL!?0&;YWDQ_Rd3lM7}Vv=@5jmN@AgXm_h=_lbx4vUCr&0}r;v4`vSV_x+DG2prJH z6RI2f>TwTMJ5)6#>~X$}#1US%G#uj${Ju4`tY;&dvKtq#ol+G0@JTg*knZv5sn#gT z5bxh4+z-wg<$Pt`Vcc&$SHsGJam&Pi&Dyjp&5u|6VpyD+%-nkMtvJry^Sddc-_8O* z)^G*R*%Ndwrl9D<4>m!Arom^O7|D%J9f3Y*&z(;RC>_TXFTb6p)Gw>h*~;H7V=jBJ zvnAE~gQ|Gpr7nNH;Z`L;B)=27`#8q)(*)*Pd0X9AfH#c9Y^i4pmGbCLtZe9eF# zxJp>$hKlExyL(wCd@nHP3?-U;%vSbKC3tPO-n~CB8f6wwHCl*aXvf;;IqB zo3uCwPI?L@zq$W^dm-wp@#8X&ke@dic7A|#Rij*cE+EI!!7VwNx} z!R`Bom+mY}ut_Gak{@b~fa3SDgYOD_ravoKYe7@XNqg-gFzl*$Z|;L}Bh#B-xTs{2 z9PFEFS0%NtpO`tRpAvpH{K_(t!e62au16&_ZdLFWLT z;I&n`>d4k7N3{F)VBKwJuY6adAy?mFN99ZNi%5|aI{IIs?nhAI_w&k^aF_EB`M~!f zuS$OEeg(7xHvtM^$$~;QTn?6&6x(p=QorVUm@DWc9e!z4!vH!DAjoNY7JHKX3{@OU z2?_uM9^`v-|7&)16R6-aKP31sMAeMNl}@5S#}0LHiuBG6X@gxb3*K0CNlA$r?;pNO z9YRcU%O0TWG$%4?0zhTK0L4&7jtlhNAH{W#={#tLI;V znt-_*G0@9we_w9#Lw%nAbdaf|N20FcN*k&Y8WQ>%r1@G@Wa=2{4YeAMQOyumCLe`8 zlqD3#&nOMxsi%dx-h;e+0_YAv6-ioa962+f&J!aGU*6rS1WhST4nlE5M6x!^#9G8h z&bDhOY8W!V>Op9i&3Qq!CYPUg0!9QJPI3G_IBIF?+lac+;8%HHx+tPNBY3(HG@QvB zlAXyK(Jg|*pAwKkc+O5hFHS+oOn|M|=wYRH&eTH+BuPkRXc#QEb>_~s53JX;WX%)0 zxu@qZbF~QJjv?O2YK0GrF?$(%=At=-`>Zj~C_gVS8ztOB;Q-75ogTbSFr zHj_hi0;GR==b+EM?L$KmEJTIRz?5=#Z-{{_6`G`oNbdan4Qy>B-qU5`(7)JPYfof1 ze-6Cb`rx{{UPxqm>rc!m$eoJCg3QS5Wx>*kAv$J{!=t#?f?m$(L_oGG{#3?ev`^}5 z|5BI*DEH$g@5ck+Z2GMb8&Gtepg)P40)+V;{m!QE70QgqyNIYhZusY|s_a-xEiZmL z_)`;bB;LHbu!)$e$O;3Jy7pHG2?9{x69Z0{G!Gi;u8AN~r9+&#!c9r(9NSqMvU=2A z*2jvx9=(dJkko*@C$gJdwB5kT2U3gaY3SY(^-pZW%-fHyR z*zr03g3RhyynV`s$1EP`k&r*D#9~5TOc2V<;$`b~nf#EkMMiZDnE}A$0J{rzq(%%w za_w_9L5EYSf7RmmMjywF8uNybHU`IP69CP=^?$qospjzss1`sdnCbw?cp7wtPul&P;ZVBQ=8Zf;Zv$ZYJLO4S|+sJJy3&*5cU80vlT5>V13TCWx@05Rj^r~ zlKp3ZXSAU`jg6SS{iu@-MUq4R!3)%f@c?c^(+NuHc&c0j$2 z(X2?Ua1Hg5z5WZ2t`rl9QkdjO+JkC@g#xKA1Fn$&)$+7E3axJq<>P0jRY_I1F6vMz z^i97X1WJ1^mk4CE8_PBjWoKPo0u+c^sXHx?j;_ZIA=#!sZkWhil`W!oGHPjOv|bLf zKML}wAp%=&0K`8BE88d3ASi5Ue+}NsKMJ9MNEDi|h^X;A%NC=o(`p2o!B!ZH6K;pQC z7|~1iQUgkM{%;{>0(Q1x#q~qJcQt%7tJAMoQF}k9m>jyh4jgeRFw@n-dt|+k(r_p) zzXapK4rN}nQ8fFTisaIOd&&JW9y^{Pz_C0r{;1h;^zf@4NO z#g>4w8>%cOfi>u7b&({c;⁡@BiZbT)DEs@yl?{%zQ2it;Q1?&cSx|uh}WI#SrTf z^EM;3+v6{+tqaBt%B4w~cu9eHG?zSVzA$YKF4^yzIfx3r14R-Ij9!T?MQ{}A4k(o|;hYNiV}qysA4f}8i@ zU#mddBF|~}>NlwHv}X#0FWcyhmRQ(n>`~!t&(;<2Y05tWLq0&T1<>g8Wq}>|+5|I% zAiH@3xL6=5BE0_cMOAi-s#D66~uf0$ozpXFz`DnLp~< zfXz*IlL-59Y*4EF;{5QIGKS)()FXb8w@jXW!-MjUl39Iei(P&snX$qrVD7#2 zzu3DunpK(0@W?K@UV&!*X4E{Z?Ru_$C)<}2Wa^Oj(4^o+diWhxkYzBD*&9)E8fj5s zG~D=mM$96lS?FD775bU&dDP0`H16_T^_9=HOKvg#CrPrbogN<^34 zP|#6f)rsWgy6UxQ^nTRTG>-^$oHf+R^xZ!YO~bC`$7wxPH0Bi5cAml_NQ8i)yBxnP~YIB1fB&WBWfm;t>vvppx(UtZgWc8bTv}Z&DU%c zgVD69QaGoj>!R}pv7ZIfya&W~JEcXTwq7t;`>7fR zccV8tpU2}eBN)XM_DaX07Rpb@ute`cL|;Y(rj7S}YI+FDdgLAryME3Q#g04x&Ro0_ z8#L;*!rGilIf|~IJI$eH$B#5vs`5ZVnz8$=Xr_-tv5v>}LD2tZww((cHpu!CgLeL9 z)s?6BOS?A5liGO)fPPYZI**-m7;tc&W0OW5^p+cE)=kM2EIhdYIm=HJ*xz0L{N6KX z=a@CY2l@~8PGcn?jwMS7ludL$Tz;1G9ii`7u4I9RjRvCr^-^i)E5Q)ARk)6Rd!34KYKBeu)?L?D zEai=0Wv279&$UQ_q*%PUTreMBMNyanV+Qo+B!CV0o^AZ1c@O|vo*KEI{GdUtEx?p@ zpkadq_nFww$C|RzjmkQ|A|y``Zi6BRBEW90jlJ;F8oTp|9RjB_lv`JCVL1^7rlj;x zhg&sl?SXW{&!12mS2r`39AAJ9@YaDU2l`g0Z)%{?cPG39svDqujh$5ksZ-HDFc?&q#*3S!&|3*Y6#NDI693j~t<>)sGRWOYM_37=(kM!gHiceL z_f;lx4)nGZMXAmZnhcTm#ILNb-k}wHRJmSS=R!^HjwJyXJ^k=}=Q@a!t1Xkq$=&ES_)6CC{ z`!*^)47AeTM#j(NO}MXtoclrEO#u)u{0Tci@d+=Gje8DyrGUO{e1>q&z9d83;cs7M z`$d4Wtgm-zNf4M9Xo`Mtc5>p9YyqV$OCZ4cV^qWcq= z0DAxuK(%_0Io>Hbqx<;juyVU%A$o*Jn3g-08X}&lJ_3oV!OxS&v{E0&t3W=0tG1vo zJb+U;PD!&3gU+$_`V%gJ>rW~Rih9w;yq)c)t6kmE|NGOJ{2plZ0*y(^$lX5tynAN> ze+f_iZoF^5=p184Z+`vK-cA=3Aw2vvH$!-s+yjEQvb8wl6kS!d1N!QlRi5*09a$C3 z+9~9|P5PWu{8_IUf}@MU4x0vu>PTr*+1Mi zi3P=ox=sV_E5dnQ!1f*CHQ>d-EpV*0X}_whHBr8}p|JVC(gsVCoQvoJIkekiA{41V zj|drbbtchmu4h({wc{5h<^N}`nLVacppVW46nXGWh9KgFe?t|Jb;K`k$?E{MqarSG zN-0-U9+wst$NkUmgu2m|;3&_-+?JSoa788UKUCX=s3KbZpz9zelcHn)=UXuLNru>s zL1R#CL1EL=(so4pu&{@0WHuLBEHUslp~9sj%Kv9UoDdSmhk6lC9L2<1|StJQC$*2-#bH? z?O6Nwk`OE}T)?FQR3UPzYP-Wcd{3T2o44eyxRvam#OpCwLDE~&MbMG!$ep}fB>4qZ zE2|qMfA}+`)B|#zK|41pL02cwt2pi%!eAVPg%VxA#y+3Uk?uaX!W6 z)kaOy^C4t7X>!Kru<#IY;(Kx}B%LA>`JPzP#HT*3)K}tCJ)VX{AeJ64s{Fat1==h5 z;LsaVeOFLs>MBQwkVC@(#_d2^H@i8+ti{6TPk{exTjj@qa~bIeP}XI1ielB)Yj+sE z7q?H>*Fkjg??oyS+6~#M<=s$U9Pt{+@!#gWLpEn8loMHWg9dR~f1gb}80toTPyFkG z5@_M`FKhMAg|=tumY#8O2L)}sZjo%Zqi55i+i@NbGlGuauAgBxfst&!zv_x1s)h7` zoX;;g(Rh85+E`CE1m%I`abI&~xdgv;_>MbSk`;l-nC(~pE-$VZ=W@`AkeOoi3i<1x zCQwH*<#l%iWbld%Rl%h@6ai*;Bh}xCfe&0SLCw}!O&X?a%c02#00+BQLEW36JEvVW z1r$i~7X+G^D)?;jI(o0I_1snn2fG3L38(~1>uJ=I7-WxV0yQmo>5yj~%W{K~ByUlv zlT1;2_@*BEIo7)hH3pCe0w2OmAvb$0?G^FCn|G7nv~dP+77Qv<@9+!4Hwjb(=25vyLg zgGn`>)0WOoK1=p#J%vv~L~I()ilTpL+P*kQiBUdvu?@jEr60%sOTE2`APv=qHRdYZ;*?!E@ox}(CmI& z-pUJZt|(foD9w32FWtqi$WST!s0ba9Wuz`l+>n;q>J#{-a>n>Bccchkw-^>a$$oI+ z_C$rNp>k#5UA`HM%sEH-M(w)pN6x%YnAvF@iU;n^hZS>bGpC$uVcFsrZ;yj6efhOL zzMq8lA8$kaOWG7uc?Bc-rw7?}lkBz_mGJmw+^UoB(;S{e`LXLRmp7}TF#Q?qMR$y@ zJH20hZP+kQ1+ z3kX9^*h;sE?q!-^>ZAQ@v3x)o>J_hCMB^=n$pfg0=~izE~5+3lw12<;(ef9iSD>8@rk8VB9!0iQm` zC=u-SS&+D|fcs14yqF+ZhPW{SFSG8I{tiaNjo{(xCj<9iSN}o&Nh%Y%MkzBa391f0 z#n_J1mxyi~Nar*bJ^lM4D1UxXCf2S-Xzess46f(;eV-tf)C<`AF`7D-tOa-_1w8+N zl2|WKir4TIV6WVgkT0&faU_bf-hNpPd`vTOHVC1KeX{=sfWV~tBtq5u6ZR$AI+bFF zl}f)xgp765W6KIOxM~HTj_3~n1G=UgjdDS}YLkSO;oASkbnla}SM*Fia)6ou4gUGD z6v!jDc3mXE!T0c&uTnNAzHe_kqIq{7-mO6MM_keqpjw-H&rc`r6m_jkCy+>a`Vi!D z_XRaSlLD}?;3Qsg#69DZ;rXK-`MZ`Bi`>H~di09FEMXSwWl>p$k?>|)r!x#!v1aM-O^qC&9u-uRyeQAo zBy~gX1{}uY?!EwhAx1vS2knd1NXzB3 zhpPFTbJGT#K}<3sYoMSNU+@s2KmL0eZ|6Jfag;tP|;HZ!N0?v zn~eOnF5>h71bh@Whv?ynUyL*N0a=^2(4drN{5@dcUm1%2|gFww=ziCeVly|yi8x z&YazopNH2sT?W4LmsqZYUQTeMRD2DD_rhQj0MPi&*cSjK+A9W)YI@)~xGJX*q*(E} z`U$}~FuoqdTgo57HBw(e``T|J%z%D6v6KvRp#NWWR~`=a`u3;O;)pm=oFkNNB3t$_ zQrVJ0NFrNfuM8n;lqDH^Gs)ObvZS(vjxE`yGM2na2@nbu!lIN{%;CV@g+o8z2c*)W@%P*^zpPgnmi@bJJ zp$b|4!9Hc+!-V3u;Y4ELYeeMywYtB0W=@72^ZAT0w9e>@e8%!b4_S@mGd*PruZON$ z5G?*3T4F>8GExjqJTs?D=0?#hCk?vX)v{RRco(b?Uw#X0qMTSsN_%J2^ld_EFqPz& z_3#zt1;u8Y`CoNaJgCIPFYU3?x65{ord}iD41-H3#2NfJeluXA}x4gN-{ldT|B-WOu9&??IFd+zMD3ai@0np*Yh(y#EDv%G!69{V_i z;6ROz94Ad>KoKioR0H&IH)4#U^>+u^JrH#g&h{FX>htayVxveWS&G|+!711$$+mvJ z;X{v_KiMYrXJ!x#y($KQ%1|H_p!gMRc;_+(cinzy*>CY@R&o&CXyFVeO!`LyUweOv zEK7LZ0`?NICzDc}#@7~*@zh=tXF>7M0@24tC>6B>34{hJZz`K*;93*28AMFQm}1=LQJcs#F5>|+`H7VJ3GfMwKR#q0eLK1g@$%)4YnYNEE zGFrQ2)_LWo=PnO^eQU7e9d%&c+=Fi2#tn zr2@7vYad&rV8SEM&~AHwRT8G4b2#N}C80>D>6=SYf(P2_T9VGAPldtP@433GM(Gpc zUOA+2W{!c1CrC4Vz>~E;qXs9FruEp&RFeuN1uzM2)^KiQax`yVmCd2I?@4Z7Y-S?J z?2YuRj>SeVDQZieToxDm;pD_>(vW*Is8SgL3}SiOOcVvk_dxmRKsGl@{^UW{#|>rj z`ScFA`Zc0ZxaPqa0}<&zV|lth=GZ=trOI4MeI+kcQRCGAG{sgi?V(EwJJtPqnrF}4 z4uVCX{XS5%bFzoI!7Sw>QA+{U^l-9Xl* zC_X0Rrgp@lbCD{)SD;O`kroEeepKADJS6Fya2ZSvLub0h+`}OD$le#`SlDJ<4@y+z z6&r>R-)9I&*a1cMSr2W>*zoL>T9_ zK-u$r>)g+*)WEW$&Vn3_k zDRE1!?l81*udh8?wbVV6Pa+2Q0j7V*FH9^fY~SX#Et;g$%SaR3UWJS0aIAgrCgwl7u^3ueZ*4X36g~F=y!BRHX`T_pYqDr! z$EvELNPpmF{+QmT60g5N$O9mLV@HUUe%tZNKEJZ2MnQ!`DTn1fDHI*jXn)v0(`f8c z*_(;dTP5_df%qL}D_7Mb@Rx7d9~9q*o5n~iZH%Mpo8A znMd0bEI3!IoSIPbL5fdpTSF~9ldRpo2&(p5`|+Sk1;R20%JGiIvF8XBqRf-{OW=8% zxgL{Hnyg6+YE(HcK|hyl#UI0ka}k1n6zXPu9j8Askm)Zq7jI+tQtbT|=}pN#G+7rr zC1G5vFNrc8Zx}DQb)G=6j3S=Dtq}~=%I3o68`et5@pLy9g=6&VFEFmCGx{f<0Kk4n z{aC6QFWLafBhVDNUwLN{D+sY;WLb%Vjc{EK0p5I%z_0PX2?x#{_1Cnod_$+RP8ExQ z00+fIMl)bZ9Td*5tv+OT{_p}->RRkJmFYjtg7prj@-2FBbFl9qeZE+G_0)QYa>(#l zTe6VNSCd%)pGgd7{Xw^H%{zf-QE&5PdQ$Oi>dQR>N^5-0S4iGI7wYa#yJJOO_@!A> z#|36MN9Q7iR9mF0)>DPy9y$*4Ue4x|r`m3cbaR`L+YvQ)1k}Pf zN=z<0Sj@`4tXOJH2n6*T8`?DlkWIiJW15#MMasnY3P`8K)Pz5#xSLK%s6C9)-APrE zvmTAl-lA$0D|@aaq%U^b6GRj%u+hXQ8}mu`_M0Nn&X;3lhFUURsdG^TzAjGX+2>^X z{Fk_og{R&pRbCX>l@{u{H~HHd@b{P(^w3$3XWC0NnlM|rGMvP6Ssn9b`ozR;n@blp zFohCV^*W>F*+%=G5$+dm2SSYoH3g>TlZIevr`il{L{qzReG{SfWn0C_xb9R_zdeQG zW%m8;2AWou529@M+3752Mk^ys%l{gEPDiIaC!!T(=N>F>?(p8TRD(O2V<3j{9?za| z+bcQ24yS=Vq=2o=_ww@@*v)TwJ!3bn!b=e@;`GooS&=v1-WaL6u$5SKG+3|t3z|9-J9pBFe5lGxCtB5T7xmB!ettADMw1WMq*M%N?A;MZnvEt@*=9M&Hdgr zr8#qsdKVa>#D3iEt>y)6cs9L~gL(Wbp%m+^VAsy%p=PYAp{3{kTNSd;k}qjYg829b zU{Sgqh%pUeEr-K#x3sIT&Bg8gYgAIIRPSof?YU0#ez^~w>T3Q`zOHLeb&>tiI!N78 z^Mc{g#@%X`J=`LJX+tl(iaLT^h1Y|Nh}lw=30o7iDQo^R3rxB6BujV#g4D?a>s(Ix zq@}Uz>n=-L!w#mL9Vl|Lx@e2wJ10Hu)qawsjGx{u&eb7ul8v3fYVtRlouk9I#c%7M z9L^YojM8wLW@nNDI}ru{E^6hCFjSeLB2k$T?k8|vyeEeB%T&wy9ucrl%RxkTY442| zZI^|uXqi9^E#dm={nd&dxdID#v4(&6$%O|p-;T&yonI|6NUFKt=e5M*{Y0zsGpCNh zyY^4t);_hq&+g2wID|hq9~DlwDMs3!N3*z5!FEG?7j;~Bm76e>s^%2vurXVPPng3q zR-wfLGV5GflZ%{+N^tUuU!>Df*BrgeJy)uTbhJj#{p##tgvN(1TxD*Nw71?Xj?uQ| z{9C*a)4na+eS36SrWkp4cGUu&D+DSq=T?uYWp_--fm3t4X5z`$^98s5$?PyYpB^$@ z9sjIdj#KsOBJD=gVD~PWje*niiVYM+;Uk_dvd{SW2=RM{eN}d%c|tB<^t(U)Ml$%# zT@@6*sjNG0G5%9$XL!}xvN7Urjo49z07;qxndtSVXriK7IWn`0-!)fc z8s8r#-slw#iJ9JV$=C4}zx!T(CBdru!z|u4x6WNVW2H%;6sGgxs&@U=>BKxzd?to& zdKVdMmajdk7U~&VQY%xV>$elA zQe9GYV+R$(YrpaU1LU8s%lTIO3!B{h3(%rT&azZUBv8`3A;7SgyTfH|vx$~Sf_)02 z0=o~Fuk5JpftIO^zl(qzVMLE7>kV)tjrw< zQt-9Q=&av*4&G7N+_AbE85=?GnBj5#n&~>hm5>VPFzVgr>ZK>rS}Dh5ye~e}g&Z5B z6W$GmyFPJC+YFfi$|Gnk4B)X%W6&Sj$io~PBCjdeO2mT1eDJdJZyO1HWaG~Nr%fJ_ zC@GpO$LA0#yjQ{o4(feh3E-hBOGQjg95|VLr@{U+BI7mL`5wv1!E2qrx6}S&osq>h zO^`}_7TQ+eK56`5DJQY?Co9Vx#awfcU$zjI1uDk?_m*}+Z`{%u_H?i& zh|`4GYc%oM@(1&e#i2eQf4X5EOg;X@_DfZ*tL^+J;l}|B@xAX-t6XLB(KxG)CwQsq zPXx~t_2?Zt!U&~D;3xeGok?=)*rFOw%o!2E+WX-c zf7iA7y@<;{s?i)}y6>=WaFu9lo`bJ8VOFDd2waVa;oRWh|I|)&-f)G%Rs_>fr+xn|2Mk$@|rRp}c zH*y}PqW9>yE#Z5`9Hw6ey_y|&MN-M167Pz_FF+hT48zLUi(@@rrDNxb1I(hjPP~a} z1dl5q?sri(Uu>%$b=)6G6+|^GPaW(AY5bek%!fOe_0rM73X6|`-l<`L`3gXIqWLP; z)o^>7!FN?K`~!-cwE^I8g}(G-K#}nXVQCl?bLp7!Ic5xmaqI!--bMDv6CyPzC^ZSF z=rxEqgXS80#urx>>oIB>@kl|Gy!~^b1@ZxJY|Ao9TE+N~=gu{;KiY&dKwPh00((19 zob_Cy7AU5Jf;_v3BS1jp$O9Py3~(s@ zuYMov0y!0e?0IqO0ivg&@0WnVPM$MnsS8;XhKDvm8y$84Jhrz0Hw^T-4aLPEn*w>4 zH?vOP2{Z%Y_)| zk{3ZI`><*XP7D0qHc`ukU(FjG4Ft~{*Y~0xK0->lzUS6!U}0 zLo>$qlpMWT?}*K%J^35X-Q@IMB%2Q@g|EFrwHe)2G&V($zhTChZSP6i(-*arN=z*U zg(;V;$P|YwumKV{Zf}RuaU+-uLn@VVhdWG(d8TAXbtCVN*z_lYB2TT+$ke>{5H4;{ zh>{3UHXXitR-DIsVkgpCrY^G#Wg29oIr4Bkt9!+7b`NsHCFMFp>73NRw_aEBzkymObU^>~qYmI}xqS=5GU!_Fd19 zOSFx-zGxI%Kuo{O&TnTt0dfO8&`JBYSrF<1^qN16P4&*etRb!~@M8pVH~nAuJx8uv Zzdp}ZIyTu*cNm(HzK)T0(Fyyo{{gqDFa-br From e5362f0ea2c1a71e84a0a5b4c4b067a54b7c9e43 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Wed, 20 Mar 2019 19:31:13 +0000 Subject: [PATCH 103/246] Merged PR 700: add conditionals to avoid issues with ingestion when tls is not used add conditionals to avoid issues with ingestion when tls is not used Related work items: #9067 --- charts/ingestion/templates/ingestion-ingress.yaml | 6 ++++-- .../ingestion/templates/ingestion-secret-ingress-tls.yaml | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index a22d79a1..0f50d81c 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -12,14 +12,16 @@ kind: Ingress metadata: name: ingestion-ingress spec: + {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.hosts }} {{- if .tls }} - hosts: - {{ .name }} secretName: {{ .tlsSecretName }} - {{ end }} - {{ end }} + {{- end }} + {{- end }} + {{- end }} rules: {{- range .Values.ingress.hosts }} - host: {{ .name }} diff --git a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml index e95393b6..f8e52cb9 100644 --- a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml +++ b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml @@ -1,3 +1,4 @@ +{{- if .Values.ingress.tls}} {{- range .Values.ingress.tls.secrets }} kind: Secret apiVersion: v1 @@ -7,4 +8,6 @@ type: kubernetes.io/tls data: tls.crt: {{ .certificate | b64enc }} tls.key: {{ .key | b64enc }} +--- +{{ end }} {{ end }} From cea0d70f34162eb412ba18878bf5e6f521136544 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 22 Mar 2019 19:21:09 +0000 Subject: [PATCH 104/246] Merged PR 697: solve CVE-2018-10237 by removing dependency to guava 11.0 solve CVE-2018-10237 by removing dependency to guava 11.0 for more information, please refer to https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-10237 --- src/shipping/ingestion/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 602df5ab..22afe55f 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -38,12 +38,6 @@ test - - com.google.common.html.types - types - 1.0.5 - - org.mockito mockito-core From c8061f35996e5c90e623bceb8c201d98bd3d7889 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 1 Apr 2019 20:27:35 +0000 Subject: [PATCH 105/246] Merged PR: fix thread context issue that cause mixup operation ids under heavy load under very heavy load whenComplete won't resume in the very same thread, causing a thread context issue that ends up mixing up operation ids. solved: #9108 Related work items: #9108 --- .../ingestion/util/ServiceBusTracingImpl.java | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java index a5849b2a..f52f1cf0 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ServiceBusTracingImpl.java @@ -36,7 +36,7 @@ public class ServiceBusTracingImpl implements ServiceBusTracing { private final TelemetryClient telemetryClient; - @Autowired + @Autowired public ServiceBusTracingImpl(TelemetryClient telemetryClient) { this.telemetryClient = telemetryClient; @@ -44,21 +44,24 @@ public ServiceBusTracingImpl(TelemetryClient telemetryClient) @Override public CompletableFuture trackAndCorrelateServiceBusDependency( - String endpoint, + String endpoint, String queueName, IMessage message, Function> func) { CompletableFuture result; - + String target = String.format( - Locale.US, - TARGET_REMOTE_DEPENDENDENCY_FORMAT, + Locale.US, + TARGET_REMOTE_DEPENDENDENCY_FORMAT, endpoint, queueName); propagateCorrelationProperties(message); - + + final RemoteDependencyTelemetry remoteDependency = createRemoteDependencyTelemetry( + target); + final long start = System.nanoTime(); result = func.apply(message); result.whenComplete((r,t) -> { @@ -68,19 +71,18 @@ public CompletableFuture trackAndCorrelateServiceBusDependency( if (t == null) { successful = true; } - - telemetryClient.trackDependency( - createRemoteDependencyTelemetry( - target, - new Duration(intervalMs), - successful)); + + remoteDependency.setDuration(new Duration(intervalMs)); + remoteDependency.setSuccess(successful); + + telemetryClient.trackDependency(remoteDependency); if(successful == false){ RuntimeException runtimeEx = new RuntimeException(t); - telemetryClient.trackException(runtimeEx); + telemetryClient.trackException(runtimeEx); } }); - + return result; } @@ -89,13 +91,19 @@ private static void propagateCorrelationProperties(IMessage message) { .getRequestTelemetryContext() .getHttpRequestTelemetry() .getId(); - + // propagate Service Bus required properties // https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-end-to-end-tracing // https://docs.microsoft.com/en-us/azure/azure-monitor/app/correlation#telemetry-correlation-in-the-java-sdk message.getProperties().put(DIAGNOSTIC_ID_PROPERTY_NAME, parentId); } + private static RemoteDependencyTelemetry createRemoteDependencyTelemetry( + String target){ + + return createRemoteDependencyTelemetry(target, new Duration(0L), true); + } + private static RemoteDependencyTelemetry createRemoteDependencyTelemetry( String target, Duration duration, @@ -103,11 +111,11 @@ private static RemoteDependencyTelemetry createRemoteDependencyTelemetry( String dependencyId = TelemetryCorrelationUtils .generateChildDependencyId(); - RemoteDependencyTelemetry dependencyTelemetry = + RemoteDependencyTelemetry dependencyTelemetry = new RemoteDependencyTelemetry( - SERVICE_BUS_REMOTE_DEPENDENDENCY_NAME, + SERVICE_BUS_REMOTE_DEPENDENDENCY_NAME, "", - duration, + duration, successful); dependencyTelemetry.setId(dependencyId); @@ -135,8 +143,8 @@ private static String extractRootId(String parentId) { if (rootEnd < 0) { rootEnd = parentId.length(); } - + int rootStart = parentId.charAt(0) == '|' ? 1 : 0; return parentId.substring(rootStart, rootEnd); } -} \ No newline at end of file +} From 8c5d884ff225be9eb410f71723f6d8f17034cb04 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Apr 2019 15:06:05 +0000 Subject: [PATCH 106/246] Merged PR 699: now workflow support package upsert op now workflow support package upsert op - rename CreatePackageAsync -> UpsertPackageAsync - bug fix BackendServiceCallFailedException when 204 - get workflow ready for perf harness defenition -> package id seems to be fixed Related work items: #9083 --- .../PackageServiceCallerIntegrationTests.cs | 103 ++++++++++++++++-- .../RequestProcessorTests.cs | 14 +-- .../RequestProcessing/RequestProcessor.cs | 4 +- .../Services/IPackageServiceCaller.cs | 2 +- .../Services/PackageServiceCaller.cs | 28 ++++- 5 files changed, 133 insertions(+), 18 deletions(-) diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs index e6deb305..539dc038 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs @@ -104,7 +104,7 @@ public async Task WhenCreatingPackage_ThenInvokesDroneSchedulerAPI() }; var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; - await _caller.CreatePackageAsync(packageInfo); + await _caller.UpsertPackageAsync(packageInfo); Assert.NotNull(actualPackageId); Assert.Equal($"/api/packages/{packageInfo.PackageId}", actualPackageId); @@ -115,6 +115,56 @@ public async Task WhenCreatingPackage_ThenInvokesDroneSchedulerAPI() Assert.Equal(packageInfo.Weight, actualPackage.Weight); } + [Fact] + public async Task WhenUpdatingPackage_ThenInvokesDroneSchedulerAPI() + { + // Arrange + string actualPackageId = null; + PackageGen actualPackage = null; + Stream body = null; + _handleHttpRequest = ctx => + { + if (ctx.Request.Host.Host == PackageHost && + ctx.Request.Method.Equals("PUT")) + { + ctx.Response.StatusCode = StatusCodes.Status204NoContent; + body = ctx.Request.Body; + } + else if (ctx.Request.Host.Host == PackageHost && + ctx.Request.Method.Equals("GET")) + { + actualPackageId = ctx.Request.Path; + actualPackage = + new JsonSerializer() + .Deserialize( + new JsonTextReader( + new StreamReader( + body, + Encoding.UTF8))); + } + else + { + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + } + + return Task.CompletedTask; + }; + + var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; + + // Act + await _caller.UpsertPackageAsync(packageInfo); + + // Assert + Assert.NotNull(actualPackageId); + Assert.Equal($"/api/packages/{packageInfo.PackageId}", actualPackageId); + + Assert.NotNull(actualPackage); + Assert.Equal((int)packageInfo.Size, (int)actualPackage.Size); + Assert.Equal(packageInfo.Tag, actualPackage.Tag); + Assert.Equal(packageInfo.Weight, actualPackage.Weight); + } + [Fact] public async Task WhenPackageAPIReturnsOK_ThenReturnsGeneratedPackage() { @@ -136,7 +186,7 @@ await ctx.WriteResultAsync( }; var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; - var actualPackage = await _caller.CreatePackageAsync(packageInfo); + var actualPackage = await _caller.UpsertPackageAsync(packageInfo); Assert.NotNull(actualPackage); Assert.Equal((int)packageInfo.Size, (int)actualPackage.Size); @@ -144,6 +194,46 @@ await ctx.WriteResultAsync( Assert.Equal(packageInfo.Weight, actualPackage.Weight); } + [Fact] + public async Task WhenPackageAPIReturnsNoContent_ThenReturnsUpdatedPackage() + { + // Arrange + _handleHttpRequest = async ctx => + { + if (ctx.Request.Host.Host == PackageHost && + ctx.Request.Method.Equals("PUT")) + { + ctx.Response.StatusCode = StatusCodes.Status204NoContent; + } + else if (ctx.Request.Host.Host == PackageHost && + ctx.Request.Method.Equals("GET")) + { + await ctx.WriteResultAsync( + new ObjectResult( + new PackageGen { Id = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }) + { + StatusCode = StatusCodes.Status200OK + }); + + } + else + { + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + } + }; + + var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; + + // Act + var actualPackage = await _caller.UpsertPackageAsync(packageInfo); + + // Assert + Assert.NotNull(actualPackage); + Assert.Equal((int)packageInfo.Size, (int)actualPackage.Size); + Assert.Equal(packageInfo.Tag, actualPackage.Tag); + Assert.Equal(packageInfo.Weight, actualPackage.Weight); + } + [Fact] public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() { @@ -163,7 +253,7 @@ public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() var packageInfo = new PackageInfo { PackageId = "somePackageId", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }; - await Assert.ThrowsAsync(() => _caller.CreatePackageAsync(packageInfo)); + await Assert.ThrowsAsync(() => _caller.UpsertPackageAsync(packageInfo)); } [Fact] @@ -189,7 +279,7 @@ await ctx.WriteResultAsync( }; var result = - await _caller.CreatePackageAsync(new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + await _caller.UpsertPackageAsync(new PackageInfo { PackageId = "package", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); Assert.Equal(3, receivedRequests); } @@ -228,7 +318,7 @@ await ctx.WriteResultAsync( try { var result = - await _caller.CreatePackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + await _caller.UpsertPackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); Interlocked.Increment(ref successfulRequests); } catch @@ -266,7 +356,7 @@ public async Task WhenRequestsFailAboveTheThreshold_ThenCircuitBreakerBreaks() { await Task.Delay(TimeSpan.FromSeconds(i / 10)); var result = - await _caller.CreatePackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); + await _caller.UpsertPackageAsync(new PackageInfo { PackageId = $"package{i}", Size = ContainerSize.Medium, Tag = "sometag", Weight = 100d }); Interlocked.Increment(ref successfulRequests); } catch @@ -282,4 +372,3 @@ public async Task WhenRequestsFailAboveTheThreshold_ThenCircuitBreakerBreaks() } } } - diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs index 99b6b2eb..bf690381 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorTests.cs @@ -45,7 +45,7 @@ public RequestProcessorTests() public async Task WhenInvokingPackageServiceThrows_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ThrowsAsync(new Exception()).Verifiable(); var delivery = @@ -66,7 +66,7 @@ public async Task WhenInvokingPackageServiceThrows_ProcessingFails() public async Task WhenInvokingPackageServiceFails_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(default(PackageGen)).Verifiable(); var delivery = @@ -87,7 +87,7 @@ public async Task WhenInvokingPackageServiceFails_ProcessingFails() public async Task WhenInvokingDroneSchedulerThrows_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); _droneSchedulerServiceCallerMock .Setup(c => c.GetDroneIdAsync(It.IsAny())) @@ -111,7 +111,7 @@ public async Task WhenInvokingDroneSchedulerThrows_ProcessingFails() public async Task WhenInvokingDroneSchedulerFails_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); _droneSchedulerServiceCallerMock .Setup(c => c.GetDroneIdAsync(It.IsAny())) @@ -135,7 +135,7 @@ public async Task WhenInvokingDroneSchedulerFails_ProcessingFails() public async Task WhenInvokingDeliverySchedulerThrows_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); _droneSchedulerServiceCallerMock .Setup(c => c.GetDroneIdAsync(It.IsAny())) @@ -162,7 +162,7 @@ public async Task WhenInvokingDeliverySchedulerThrows_ProcessingFails() public async Task WhenInvokingDeliverySchedulerFails_ProcessingFails() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); _droneSchedulerServiceCallerMock .Setup(c => c.GetDroneIdAsync(It.IsAny())) @@ -189,7 +189,7 @@ public async Task WhenInvokingDeliverySchedulerFails_ProcessingFails() public async Task WhenProcessingAValidDelivery_ProcessingSucceeds() { _packageServiceCallerMock - .Setup(c => c.CreatePackageAsync(It.IsAny())) + .Setup(c => c.UpsertPackageAsync(It.IsAny())) .ReturnsAsync(new PackageGen { Id = "someid" }).Verifiable(); _droneSchedulerServiceCallerMock .Setup(c => c.GetDroneIdAsync(It.IsAny())) diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs index e72163d0..926d3090 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/RequestProcessing/RequestProcessor.cs @@ -37,7 +37,7 @@ public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IR try { - var packageGen = await _packageServiceCaller.CreatePackageAsync(deliveryRequest.PackageInfo).ConfigureAwait(false); + var packageGen = await _packageServiceCaller.UpsertPackageAsync(deliveryRequest.PackageInfo).ConfigureAwait(false); if (packageGen != null) { _logger.LogInformation("Generated package {packageId} for delivery {deliveryId}", packageGen.Id, deliveryRequest.DeliveryId); @@ -68,4 +68,4 @@ public async Task ProcessDeliveryRequestAsync(Delivery deliveryRequest, IR return false; } } -} \ No newline at end of file +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs index 0c394c93..4fa9eeaa 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/IPackageServiceCaller.cs @@ -10,6 +10,6 @@ namespace Fabrikam.Workflow.Service.Services { public interface IPackageServiceCaller { - Task CreatePackageAsync(PackageInfo packageInfo); + Task UpsertPackageAsync(PackageInfo packageInfo); } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs index 8d8dcba0..78a6fbb8 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/PackageServiceCaller.cs @@ -20,7 +20,7 @@ public PackageServiceCaller(HttpClient httpClient) _httpClient = httpClient; } - public async Task CreatePackageAsync(PackageInfo packageInfo) + public async Task UpsertPackageAsync(PackageInfo packageInfo) { try { @@ -29,6 +29,32 @@ public async Task CreatePackageAsync(PackageInfo packageInfo) { return await response.Content.ReadAsAsync(); } + else if (response.StatusCode == HttpStatusCode.NoContent) + { + return await this.GetPackageAsync(packageInfo.PackageId); + } + + throw new BackendServiceCallFailedException(response.ReasonPhrase); + } + catch (BackendServiceCallFailedException) + { + throw; + } + catch (Exception e) + { + throw new BackendServiceCallFailedException(e.Message, e); + } + } + + private async Task GetPackageAsync(string packageId) + { + try + { + var response = await _httpClient.GetAsync($"{packageId}"); + if (response.StatusCode == HttpStatusCode.OK) + { + return await response.Content.ReadAsAsync(); + } throw new BackendServiceCallFailedException(response.ReasonPhrase); } From f5bb5a3ade203044391e4baac7c50d406a35ea23 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Apr 2019 16:31:29 +0000 Subject: [PATCH 107/246] Merged PR 709: changes required that enable to support up to 2k req/s Related work items: #9090 --- azuredeploy.json | 9 +++++++-- charts/delivery/values.yaml | 2 +- charts/dronescheduler/values.yaml | 2 +- charts/package/templates/package-deploy.yaml | 2 +- charts/package/values.yaml | 4 +++- charts/workflow/values.yaml | 2 +- deployment.md | 1 + .../Fabrikam.Workflow.Service/WorkflowServiceOptions.cs | 4 ++-- 8 files changed, 17 insertions(+), 9 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 50364ebb..4cf34d19 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -359,7 +359,12 @@ "apiVersion": "2017-04-01", "dependsOn": [ "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" - ] + ], + "properties": { + "lockDuration": "PT5M", + "maxSizeInMegabytes": "1024", + "enablePartitioning": "true" + } }, { "name": "[variables('ingestionServiceAccessKey')]", @@ -747,4 +752,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index 494a260b..cdc8a314 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -1,5 +1,5 @@ # Default values for dronedelivery. -replicaCount: 1 +replicaCount: 3 identity: clientid: resourceid: diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index ea9d1dc4..10595902 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -1,5 +1,5 @@ # Default values for dronescheduler. -replicaCount: 1 +replicaCount: 3 identity: clientid: resourceid: diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index def4a8ae..bc194dfa 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -48,7 +48,7 @@ spec: name: package-secrets key: mongodb-pwd - name: COLLECTION_NAME - value: packages + value: {{ default "packages" .Values.cosmosDb.collectionName }} - name: APPINSIGHTS_INSTRUMENTATIONKEY valueFrom: secretKeyRef: diff --git a/charts/package/values.yaml b/charts/package/values.yaml index a997bfc1..2ea62d0e 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -1,5 +1,5 @@ # Default values for package service. -replicaCount: 1 +replicaCount: 3 dockerregistry: image: repository: @@ -8,3 +8,5 @@ image: reason: unknown log: level: error +cosmosDb: + collectionName: diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 7b2d9225..8e5b1e3f 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -1,5 +1,5 @@ # Default values for workflow. -replicaCount: 1 +replicaCount: 3 dockerregistry: identity: clientid: diff --git a/deployment.md b/deployment.md index 67cdacc9..5449aa73 100644 --- a/deployment.md +++ b/deployment.md @@ -244,6 +244,7 @@ helm install $HELM_CHARTS/package/ \ --set image.repository=package \ --set secrets.appinsights.ikey=$AI_IKEY \ --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ + --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ --set dockerregistry=$ACR_SERVER \ --namespace backend \ --name package-v0.1.0 diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs index 8eb21164..4902a2a5 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/WorkflowServiceOptions.cs @@ -9,8 +9,8 @@ internal class WorkflowServiceOptions { public WorkflowServiceOptions() { - MaxConcurrency = 10; - PrefetchCount = 10; + MaxConcurrency = 20; + PrefetchCount = 3000; } public string QueueEndpoint { get; set; } From 371124b80da0abd924b6cc2e8c95e1f449583a47 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Apr 2019 16:57:58 +0000 Subject: [PATCH 108/246] Merged PR 716: fix dangling comma fix dangling comma Related work items: #9095 --- src/shipping/package/app/controllers/package-controllers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shipping/package/app/controllers/package-controllers.ts b/src/shipping/package/app/controllers/package-controllers.ts index 47540636..bad68cb5 100644 --- a/src/shipping/package/app/controllers/package-controllers.ts +++ b/src/shipping/package/app/controllers/package-controllers.ts @@ -118,7 +118,7 @@ export class PackageControllers { case MongoErrors.TooManyRequests: logger.error('Too many requests', ctx.request.body) - ctx.response.status, 429; + ctx.response.status = 429; break; default: @@ -126,4 +126,4 @@ export class PackageControllers { } } } -} \ No newline at end of file +} From 7a05869368201df23e82b42379ce54a8521590fb Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Wed, 3 Apr 2019 18:09:03 +0000 Subject: [PATCH 109/246] Merged PR 718: ensure azure resource names are valid ensure azure resource names are valid Related work items: #9138 --- azuredeploy.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 4cf34d19..c8dbf3f1 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -181,12 +181,12 @@ "appInsightsName": "[concat('ai', uniqueString(resourceGroup().id))]", "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", - "deliveryCosmosDbName": "[concat(uniqueString(resourceGroup().id), '-delivery-service-cosmosdb')]", - "deliveryRedisName": "[concat(uniqueString(resourceGroup().id), '-delivery-service-redis')]", + "deliveryCosmosDbName": "[concat('delivery-service-cosmosdb-', uniqueString(resourceGroup().id))]", + "deliveryRedisName": "[concat('delivery-service-redis-', uniqueString(resourceGroup().id))]", "deliveryKeyVaultName": "[concat(resourceGroup().name, '-d-kv')]", - "packageMongoDbName": "[concat(uniqueString(resourceGroup().id),'-package-service-cosmosdb')]", - "ingestionSBNamespace": "[concat(uniqueString(resourceGroup().id), 'ingsbns')]", - "ingestionSBName": "[concat(uniqueString(resourceGroup().id), 'ingsb')]", + "packageMongoDbName": "[concat('package-service-cosmosdb-', uniqueString(resourceGroup().id))]", + "ingestionSBNamespace": "[concat('ingsbns-', uniqueString(resourceGroup().id))]", + "ingestionSBName": "[concat('ingsb-', uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(resourceGroup().name, '-ds-kv')]", "workflowKeyVaultName": "[concat(resourceGroup().name, '-wf-kv')]", From 9220d7837a26d503a568f4a59cfc8348d58a1339 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 29 Mar 2019 18:04:17 -0300 Subject: [PATCH 110/246] adding test results and metrics to README.md --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 8a08259f..060b8315 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,31 @@ The Drone Delivery application is a sample application that consists of several ![](./architecture.png) +## Test results and metrics +The Drone Delivery application has been tested up to 2000 messages/sec: + + +| Column0 | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | +|------------------------------------------|----------|---------------|---------------|-------------------------|-------------------------|----------|-----------|-----------|-----------| +| Nginx | 1 | N/A | N/A | serve: 1595 reqs/sec | serve: 1923 reqs/sec | N/A | N/A | N/A | N/A | +| Ingestion | 10 | 474 | 488 | ingest: 1275 msgs/sec | ingest: 1710 msgs/sec | 251 | 50.1 | 1560 | 2540 | +| Workflow (receive messages) | 35 | 1445 | 79 | egress: 1275 msgs/sec | egress: 1710 msgs/sec | 81.5 | 0 | 25.7 | 121 | +| Workflow (call backend services + mark message as complete) | 35 | 1445 | 79 | complete: 1100 msgs/sec | complete: 1322 msgs/sec | 561.8 | 447 | 1350 | 2540 | +| Package | 50 | 213 | 78 | N/A | N/A | 67.5 | 53.9 | 165 | 306 | +| Delivery | 50 | 328 | 334 | N/A | N/A | 93.8 | 82.4 | 200 | 304 | +| Dronescheduler | 50 | 402 | 301 | N/A | N/A | 85.9 | 72.6 | 203 | 308 | + + + +*sources: +1. Serve: Visual Studio Load Test Throughout Request/Sec +2. Ingest: Azure Service Bus metrics Incoming Messages/Sec +3. Egress: Azure Service Bus metrics Outgoing Messages/Sec +4. Complete: AI Service Bus Complete dependencies +5. Avg/50th/95th/99th: AI dependencies +6. CPU/Mem: Azure Monitor for Containers + + ## Deployment To deploy the solution, follow the steps listed [here](./deployment.md). From 75ac67b4b0d8b069f0f5d5ac4df79ef1a703c1ae Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 3 Apr 2019 14:50:59 -0300 Subject: [PATCH 111/246] remove column0 header --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 060b8315..3e050135 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The Drone Delivery application is a sample application that consists of several The Drone Delivery application has been tested up to 2000 messages/sec: -| Column0 | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | +| | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | |------------------------------------------|----------|---------------|---------------|-------------------------|-------------------------|----------|-----------|-----------|-----------| | Nginx | 1 | N/A | N/A | serve: 1595 reqs/sec | serve: 1923 reqs/sec | N/A | N/A | N/A | N/A | | Ingestion | 10 | 474 | 488 | ingest: 1275 msgs/sec | ingest: 1710 msgs/sec | 251 | 50.1 | 1560 | 2540 | From f7aec5e2fd3cd6683ceac61505074ff31997bc65 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 3 Apr 2019 14:56:14 -0300 Subject: [PATCH 112/246] format super script --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e050135..b75d75e2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The Drone Delivery application is a sample application that consists of several The Drone Delivery application has been tested up to 2000 messages/sec: -| | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | +| | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | |------------------------------------------|----------|---------------|---------------|-------------------------|-------------------------|----------|-----------|-----------|-----------| | Nginx | 1 | N/A | N/A | serve: 1595 reqs/sec | serve: 1923 reqs/sec | N/A | N/A | N/A | N/A | | Ingestion | 10 | 474 | 488 | ingest: 1275 msgs/sec | ingest: 1710 msgs/sec | 251 | 50.1 | 1560 | 2540 | @@ -50,7 +50,7 @@ The Drone Delivery application has been tested up to 2000 messages/sec: 2. Ingest: Azure Service Bus metrics Incoming Messages/Sec 3. Egress: Azure Service Bus metrics Outgoing Messages/Sec 4. Complete: AI Service Bus Complete dependencies -5. Avg/50th/95th/99th: AI dependencies +5. Avg/50th/95th/99th: AI dependencies 6. CPU/Mem: Azure Monitor for Containers From e9250adf23860020224f73e967d7c2c65f76ff3e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 5 Apr 2019 13:09:03 +0000 Subject: [PATCH 113/246] Merged PR 721: fix maven plugin goal issue fix maven plugin goal issue solved: #9021 Related work items: #9021 --- src/shipping/ingestion/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 22afe55f..250887e4 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -24,6 +24,7 @@ 1.16.10 2.7.5 2.1.2.RELEASE + com.fabrikam.dronedelivery.ingestion.IngestionApplication @@ -137,4 +138,4 @@ - \ No newline at end of file + From ec1e47b7f86ba59ce9f36031a0ed996f2d164886 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 5 Apr 2019 13:15:09 +0000 Subject: [PATCH 114/246] Merged PR 724: get integration tests in green get integration tests in green --- .../configuration/TestAppConfig.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java index a7135bc5..ef1bc4db 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java @@ -1,8 +1,19 @@ package com.fabrikam.dronedelivery.ingestion.configuration; +import com.fabrikam.dronedelivery.ingestion.util.ClientPool; +import com.fabrikam.dronedelivery.ingestion.util.ClientPoolImpl; +import com.fabrikam.dronedelivery.ingestion.util.InstrumentedQueueClient; + import com.microsoft.applicationinsights.TelemetryClient; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; import org.mockito.Mockito; +import static org.mockito.Mockito.*; + +import java.net.URISyntaxException; +import java.util.concurrent.CompletableFuture; + import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -15,4 +26,21 @@ public class TestAppConfig { public TelemetryClient getTelemetryClient() { return Mockito.mock(TelemetryClient.class); } + + @Bean + @Primary + public ClientPool getClientPool() throws InterruptedException, ServiceBusException, URISyntaxException { + ClientPool clientPoolMock = mock(ClientPoolImpl.class); + InstrumentedQueueClient instrumentedQueueClient = + mock(InstrumentedQueueClient.class); + CompletableFuture futureMock = + (CompletableFuture) mock(CompletableFuture.class); + + when(instrumentedQueueClient.sendAsync(any(IMessage.class))) + .thenReturn(futureMock); + when(clientPoolMock.getConnection()) + .thenReturn(instrumentedQueueClient); + + return clientPoolMock; + } } From f8d76f61b14542c5fc11325a47fd1adc5dc55b6a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 5 Apr 2019 13:48:23 +0000 Subject: [PATCH 115/246] Merged PR 710: add Load Testing add Load Testing add two scenarios that helps to clearly visualize whether the number of indicated replicas in every service is enough to handle a really small load of deliveries and another more heavy load. - new Visual Studio Load test project - add run settings for 8 mins execution - first scenario delivers 200 requests/sec - second scenario adds more load starting after 4 minutes of execution delivering up to 2.5 k request/s - expose context param to configure FQDN - add instructions Related work items: #9090 --- deployment.md | 7 + .../Fabrikam.Ingress.Ingestion.LoadTests.sln | 30 ++ .../ContextParamLoadTestPlugin.cs | 23 + .../DeliveryRequestLoadTest.loadtest | 473 ++++++++++++++++++ .../DeliveryRequestWebTest.cs | 106 ++++ ...abrikam.Ingress.Ingestion.LoadTests.csproj | 90 ++++ .../Properties/AssemblyInfo.cs | 39 ++ ...CreateDynamicDeliveryRequests.testsettings | 29 ++ src/shipping/ingress/readme.md | 26 + 9 files changed, 823 insertions(+) create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs create mode 100644 src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings create mode 100644 src/shipping/ingress/readme.md diff --git a/deployment.md b/deployment.md index 5449aa73..34660e08 100644 --- a/deployment.md +++ b/deployment.md @@ -432,8 +432,15 @@ open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/sched > We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. + ## Optional steps +### Load Test the application + +To run load testing against the solution, follow the steps listed [here](./src/shipping/ingress/readme.md). + +### Fluentd and Elastic Search + Follow these steps to add logging and monitoring capabilities to the solution. Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln new file mode 100644 index 00000000..149c43a9 --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln @@ -0,0 +1,30 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28714.193 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Ingress.Ingestion.LoadTests", "Fabrikam.Ingress.Ingestion.LoadTests\Fabrikam.Ingress.Ingestion.LoadTests.csproj", "{4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{AD4DC75F-D0D4-4ACA-B852-508F78588C4B}" + ProjectSection(SolutionItems) = preProject + FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings = FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {BE782D95-E9EF-4B0E-9C6E-99A3288DB6EA} + EndGlobalSection +EndGlobal diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs new file mode 100644 index 00000000..6899bf20 --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.VisualStudio.TestTools.LoadTesting; + +namespace Fabrikam.Ingress.Ingestion.LoadTests +{ + public class ContextParameterLoadTestPlugin : ILoadTestPlugin + { + public void Initialize(LoadTest loadTest) + { + loadTest.TestStarting += (s, e) => + { + foreach (string key in loadTest.Context.Keys) + { + e.TestContextProperties.Add(key, loadTest.Context[key]); + } + }; + } + } +} diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest new file mode 100644 index 00000000..ad50786e --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest @@ -0,0 +1,473 @@ + + + + + + + + + + + + + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs new file mode 100644 index 00000000..4b247250 --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs @@ -0,0 +1,106 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using Microsoft.VisualStudio.TestTools.WebTesting; + +namespace Fabrikam.Ingress.Ingestion.LoadTests +{ + public class DeliveryRequestWebTest : WebTest + { + private const string ContextParamIngestUrl = "INGEST_URL"; + + private const string DeliveryId = "42"; + private const string PackageId = "442"; + private const string PackageLocationDropOff = "0,37.4315730000000,-78.65689399999997"; + private const string PackageLocationPickup = "0,36.778261,-119.41793200000001"; + + public DeliveryRequestWebTest() + { + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + ServicePointManager.ServerCertificateValidationCallback = RemoteCertificateValidationCallBack; + } + + public static bool RemoteCertificateValidationCallBack( + object sender, + X509Certificate certificate, + X509Chain chain, + SslPolicyErrors sslPolicyErrors) => true; + + public override IEnumerator GetRequestEnumerator() + { + Uri deliveryRequestUri = this.CreateDeliveryRequestUri(); + + var deliveryRequest = new WebTestRequest(deliveryRequestUri) + { + Method = "POST", + Body = CreateRandomHttpBodyString() + }; + + yield return deliveryRequest; + } + + private Uri CreateDeliveryRequestUri() + { + if (!(this.Context[ContextParamIngestUrl].ToString() is var ingestUrl + && !string.IsNullOrEmpty(ingestUrl))) + { + throw new ArgumentNullException($"{ContextParamIngestUrl} load test context param value can not be null"); + } + + if (!Uri.TryCreate( + ingestUrl, + UriKind.Absolute, + out Uri ingestUri)) + { + throw new ArgumentException($"{ingestUrl} is not a valid absolute URI"); + } + + if (!Uri.TryCreate( + ingestUri, + "/api/deliveryrequests", + out Uri deliveryRequestUri)) + { + throw new ArgumentException($"{ingestUrl}/api/deliveryrequests is not a valid URI"); + } + + return deliveryRequestUri; + } + + private static StringHttpBody CreateRandomHttpBodyString() + { + Guid tag = Guid.NewGuid(); + + var httpBodyRequestWithRandomTag = + new StringHttpBody + { + ContentType = "application/json", + InsertByteOrderMark = false, + BodyString = $@"{{ + ""confirmationRequired"": ""FingerPrint"", + ""deadline"": ""DeadlyQueueOfZombiatedDemons"", + ""deliveryId"": ""{DeliveryId}"", + ""dropOffLocation"": ""{PackageLocationDropOff}"", + ""expedited"": true, + ""ownerId"": ""1"", + ""packageInfo"": {{ + ""packageId"": ""{PackageId}"", + ""size"": ""Small"", + ""tag"": ""{tag}"", + ""weight"": 14 + }}, + ""pickupLocation"": ""{PackageLocationPickup}"", + ""pickupTime"": ""2017-11-08T23:26:30.118Z"" +}}" + }; + + return httpBodyRequestWithRandomTag; + } + } +} \ No newline at end of file diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj new file mode 100644 index 00000000..355997b6 --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj @@ -0,0 +1,90 @@ + + + + Debug + AnyCPU + + + 2.0 + {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0} + Library + Properties + Fabrikam.Ingress.Ingestion.LoadTests + Fabrikam.Ingress.Ingestion.LoadTests + v4.7.2 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + WebTest + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + False + + + + + + + False + + + + + + + + + + Designer + + + + + + + False + + + False + + + False + + + False + + + + + + + + \ No newline at end of file diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..7ac70ccd --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs @@ -0,0 +1,39 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("DeliverySchedulerLoadTest")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("DeliverySchedulerLoadTest")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("4bd986a7-3f52-46fa-aa35-cd5d8ac532e0")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +[assembly: AssemblyVersion("0.1.0.0")] +[assembly: AssemblyFileVersion("0.1.0.0")] diff --git a/src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings b/src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings new file mode 100644 index 00000000..ba006600 --- /dev/null +++ b/src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings @@ -0,0 +1,29 @@ + + + These are default test settings for a local test run. + + + + + + + + + + + +
+
+
+
+ + + + + + + + + + + \ No newline at end of file diff --git a/src/shipping/ingress/readme.md b/src/shipping/ingress/readme.md new file mode 100644 index 00000000..9d0c154d --- /dev/null +++ b/src/shipping/ingress/readme.md @@ -0,0 +1,26 @@ +## Delivery Request Load Testing + +### Prerequisites +a. Microsoft Visual Studio 2017 or later Enterprise Edition +b. Web performance and load testing tools must be installed. For more information, please take a look at [Install the Load testing component](https://docs.microsoft.com/en-us/visualstudio/test/quickstart-create-a-load-test-project?view=vs-2017#install-the-load-testing-component) +c. Azure DevOps subscription +d. ```EXTERNAL_INGEST_FQDN``` value from [deployment instructions](../../../deployment.md) + +### Run load testing in Azure DevOps + +1. open Microsoft Visual Studio 2017 or later +2. navigate to Team Explorer and connect to your Azure DevOps subscription +3. double click on DeliveryRequestLoadTest.loadtest + > if the xml is open instead of the load test designer. + Right click on .loadtest file from the Solution Explorer, and then click ```Open With``` and select Load Test Editor +4. double click ```INGEST_URL``` Context Parameter +5. from Properties panel, set its value using the environment variable ```EXTERNAL_INGEST_FQDN``` as shown below: + > https://```$EXTERNAL_INGEST_FQDN``` + + > Important: the url should look like, https://```.```.cloudapp.azure.com +6. click Run Load Test + > if the ```EXTERNAL_INGEST_FQDN``` was not configured properly it is noticed from Test Results ```Graph``` because Total Requests metric is 0. + Please check from ```Details``` looking for errors and ensure the check FQDN is well formed. + +> Note: this load test executes two separate scenarios, the first one will use a single user to delivery at about 200 request/sec. +While the second scenario is designed to load 40 users to deliver up to 2600 request/sec. Depending on how much ingestion and backend services instances are configured, the load they will ingest/egress respectively. From 863bc0dbef88dba59c7fe26d2b3e618d70d0d998 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 5 Apr 2019 18:40:55 +0000 Subject: [PATCH 116/246] Merged PR 728: convert from anonoymous type to json string convert from anonoymous type to json string --- .../DeliveryRequestWebTest.cs | 37 ++++++++++--------- ...abrikam.Ingress.Ingestion.LoadTests.csproj | 4 ++ .../packages.config | 4 ++ 3 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs index 4b247250..ce10d0c2 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs @@ -9,6 +9,7 @@ using System.Net.Security; using System.Security.Cryptography.X509Certificates; using Microsoft.VisualStudio.TestTools.WebTesting; +using Newtonsoft.Json; namespace Fabrikam.Ingress.Ingestion.LoadTests { @@ -75,29 +76,31 @@ private Uri CreateDeliveryRequestUri() private static StringHttpBody CreateRandomHttpBodyString() { - Guid tag = Guid.NewGuid(); + Guid randomTag = Guid.NewGuid(); var httpBodyRequestWithRandomTag = new StringHttpBody { ContentType = "application/json", InsertByteOrderMark = false, - BodyString = $@"{{ - ""confirmationRequired"": ""FingerPrint"", - ""deadline"": ""DeadlyQueueOfZombiatedDemons"", - ""deliveryId"": ""{DeliveryId}"", - ""dropOffLocation"": ""{PackageLocationDropOff}"", - ""expedited"": true, - ""ownerId"": ""1"", - ""packageInfo"": {{ - ""packageId"": ""{PackageId}"", - ""size"": ""Small"", - ""tag"": ""{tag}"", - ""weight"": 14 - }}, - ""pickupLocation"": ""{PackageLocationPickup}"", - ""pickupTime"": ""2017-11-08T23:26:30.118Z"" -}}" + BodyString = JsonConvert.SerializeObject(new + { + confirmationRequired = "FingerPrint", + deadline = "DeadlyQueueOfZombiatedDemons", + deliveryId = DeliveryId, + dropOffLocation = PackageLocationDropOff, + expedited = true, + ownerId = "1", + packageInfo = new + { + packageId = PackageId, + size = "Small", + tag = randomTag, + weight = "14" + }, + pickupLocation = PackageLocationPickup, + pickupTime = "2019-04-05T11:00:00.000Z" + }) }; return httpBodyRequestWithRandomTag; diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj index 355997b6..a7e24517 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj @@ -43,6 +43,9 @@ False + + ..\packages\Newtonsoft.Json.12.0.1\lib\net45\Newtonsoft.Json.dll + @@ -59,6 +62,7 @@ Designer + diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config new file mode 100644 index 00000000..97c22dc5 --- /dev/null +++ b/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file From a7a3dee5bae44659b083fd695233eb1b518793d8 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 5 Apr 2019 17:12:18 -0300 Subject: [PATCH 117/246] clean up cicd 1. remove wip instructions 2. remove cd yaml pipeline since actual yaml support is going to be released soon and for the momment we plan to export-import json release pipelines 3. remove vars --- deploymentCICD.md | 115 ------------------ src/shipping/delivery/azure-pipelines.yml | 5 - .../dronescheduler/azure-pipelines.yml | 5 - src/shipping/package/azure-pipelines-cd.yml | 64 ---------- src/shipping/package/azure-pipelines.yml | 5 - 5 files changed, 194 deletions(-) delete mode 100644 deploymentCICD.md delete mode 100644 src/shipping/package/azure-pipelines-cd.yml diff --git a/deploymentCICD.md b/deploymentCICD.md deleted file mode 100644 index 248e5212..00000000 --- a/deploymentCICD.md +++ /dev/null @@ -1,115 +0,0 @@ -# Deploying the Reference Implementation - -## Prerequisites -TODO: use azure shell -- Azure suscription -- [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) -- [Helm 2.11.1](#) -- Azure Pipelines CLI: See the [install steps](https://docs.microsoft.com/en-us/cli/vsts/install?view=vsts-cli-latest) for instructions on installing the VSTS CLI on Windows, Linux, or Mac. -1. [create Azure DevOps account](https://azure.microsoft.com/en-us/services/devops) -2. create a new Azure DevOps organization and a [personal access token](https://docs.microsoft.com/vsts/accounts/use-personal-access-tokens-to-authenticate) -2. [add Azure subscription as service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=vsts#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal) -3. [optionally, assign service connection application to role, so it is allowed to create new azure resources](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal#assign-application-to-role) - -Create a new Azure Repo - -```bash -# login in Azure DevOps -vsts login --token # use the token created in prerequisite step th - -# configure as default the new organization create in prerequisite step th -vsts configure --defaults instance=https://dev.azure.com// - -# create repo -vsts code repo create -``` - -Clone or download this repo locally - -```bash -git clone https://github.com/mspnp/.git && \ -cd && \ -git remote add # this remote url corresponds to the prerequisite step 4th -``` -Export the following environment variables - -``` -export SERVICECONNECTION= # use the name configured in the 2nd prerequisite step -export LOCATION= -export UNIQUE_APP_NAME_PREFIX=[YOUR_UNIQUE_APPLICATION_NAME_HERE] - -export RESOURCE_GROUP="${UNIQUE_APP_NAME_PREFIX}-rg" && \ -export ACR_NAME="${UNIQUE_APP_NAME_PREFIX}-acr" && \ -export AKS_NAME="${UNIQUE_APP_NAME_PREFIX}-cluster" -``` - -The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. - -## Create the Kubernetes cluster - -Provision a Kubernetes cluster in AKS -TODO: use arm and deploy aks and other infrastructure needed -```bash -# Log in to Azure -az login - -# Create a resource group for AKS -az group create --name $RESOURCE_GROUP --location $LOCATION - -# Create the AKS cluster -az aks create --resource-group $RESOURCE_GROUP --name $AKS_NAME --node-count 4 --enable-addons monitoring --generate-ssh-keys - -# Install kubectl -sudo az aks install-cli - -# Get the Kubernetes cluster credentials -az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$AKS_NAME - -# Create the ACR instance -az acr create --name $ACR_NAME --resource-group $RESOURCE_GROUP --sku Basic - -# Add a repository to Helm client -az acr helm repo add -n $ACR_NAME - -# Install Helm -helm init --upgrade -``` - -## Setup CICD Pipeline using Azure Pipelines - -Replace azure pipeline place holders - -``` -sed -i "s#AzureSubscription: #AzureSubscription: $SERVICECONNECTION#g" azure-pipelines.yml && \ -sed -i "s#Location: #Location: $LOCATION#g" azure-pipelines.yml && \ -sed -i "s#ResourceGroup: #ResourceGroup: $RESOURCEGROUP#g" azure-pipelines.yml && \ -sed -i "s#AzureContainerRegistry: #AzureContainerRegistry: $ACR_NAME#g" azure-pipelines.yml && \ -sed -i "s#AzureKubernetesService: #AzureKubernetesService: $AKS_NAME#g" azure-pipelines.yml -``` - -Push changes to azure repos or github - -```bash -git push master -``` - -Follow instructions below to configure your first Azure Pipeline - -[Get your first build with Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started-yaml?view=vsts#get-your-first-build) - -> Note: this first build will attemp to execute the azurepipeline.yml against master - -Trigger the CICD pipeline by pushing to staging - -``` -git checkout -b staging && \ -git push staging -``` - -> Note: also feature branches are going through the CI pipeline. - -Follow CICD from Azure Pipelines - -``` -open https://dev.azure.com///_build -``` diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml index 5c2d2e93..f222a652 100644 --- a/src/shipping/delivery/azure-pipelines.yml +++ b/src/shipping/delivery/azure-pipelines.yml @@ -1,10 +1,5 @@ variables: - ResourceGroup: 'pnpdh-rg' - AzureKubernetesService: 'pnpdh-cluster' BuildConfiguration: "Release" - AzureSubscription: arm_aks-ra - AzureContainerRegistry: pnpdhacr.azurecr.io - AzureContainerRegistryHelmRepo: pnpdhacr name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) diff --git a/src/shipping/dronescheduler/azure-pipelines.yml b/src/shipping/dronescheduler/azure-pipelines.yml index d51d9417..2e1e59d2 100644 --- a/src/shipping/dronescheduler/azure-pipelines.yml +++ b/src/shipping/dronescheduler/azure-pipelines.yml @@ -1,10 +1,5 @@ variables: - ResourceGroup: 'pnpdh-rg' - AzureKubernetesService: 'pnpdh-cluster' BuildConfiguration: "Release" - AzureSubscription: arm_aks-ra - AzureContainerRegistry: pnpdhacr.azurecr.io - AzureContainerRegistryHelmRepo: pnpdhacr name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) diff --git a/src/shipping/package/azure-pipelines-cd.yml b/src/shipping/package/azure-pipelines-cd.yml deleted file mode 100644 index 701b4d08..00000000 --- a/src/shipping/package/azure-pipelines-cd.yml +++ /dev/null @@ -1,64 +0,0 @@ -steps: - -- script: echo '##vso[task.setvariable variable=repositoryName]package' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- task: HelmInstaller@0 - displayName: 'Install Helm 2.9.1' - inputs: - kubectlVersion: 1.10.3 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - namespace: prod - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: AzureCLI@1 - displayName: 'Add repository to Helm client' - inputs: - azureSubscription: 'arm_service_connection' - - scriptLocation: inlineScript - - inlineScript: 'az acr helm repo add --name $(AzureContainerRegistryHelmRepo)' - -- task: HelmDeploy@0 - displayName: 'helm upgrade ' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - namespace: backend - - command: 'upgrade ' - - chartType: Name - - chartName: $(AzureContainerRegistryHelmRepo)/$(repositoryName) - - arguments: '--version $(Build.SourceBranchName)' - - releaseName: $(repositoryName)-$(Build.SourceBranchName) - - overrideValues: 'image.tag=$(Build.SourceBranchName),image.repository=$(repositoryName),dockerregistry=$(AzureContainerRegistry)' - - waitForExecution: false diff --git a/src/shipping/package/azure-pipelines.yml b/src/shipping/package/azure-pipelines.yml index 1aca0789..2e0a26bb 100644 --- a/src/shipping/package/azure-pipelines.yml +++ b/src/shipping/package/azure-pipelines.yml @@ -1,10 +1,5 @@ variables: - ResourceGroup: 'pnpdh-rg' - AzureKubernetesService: 'pnpdh-cluster' BuildConfiguration: "Release" - AzureSubscription: arm_aks-ra - AzureContainerRegistry: pnpdhacr.azurecr.io - AzureContainerRegistryHelmRepo: pnpdhacr name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) From 10c3d27b6c3335018c0638f18246cc18abd0dc14 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Sun, 7 Apr 2019 03:44:10 -0700 Subject: [PATCH 118/246] Minor fixes to deployment instructions (#124) --- deployment.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deployment.md b/deployment.md index 34660e08..48d24bce 100644 --- a/deployment.md +++ b/deployment.md @@ -31,7 +31,6 @@ Set environment variables. ```bash export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] -export SSH_PRIVATE_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PRIVAYE_KEY_FILE_HERE] export LOCATION=[YOUR_LOCATION_HERE] @@ -66,7 +65,7 @@ Deployment ```bash # Deploy the managed identities # These are deployed first in a separate template to avoid propagation delays with AAD -az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file azuredeploy-identities.json +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file ${PROJECT_ROOT}/azuredeploy-identities.json export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) @@ -82,7 +81,7 @@ until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Wai # Deploy all other resources # The version of kubernetes must be supported in the target region export KUBERNETES_VERSION='1.12.6' -az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file azuredeploy.json \ +az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file ${PROJECT_ROOT}/azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ servicePrincipalId=${SP_OBJECT_ID} \ @@ -203,6 +202,7 @@ helm install $HELM_CHARTS/delivery/ \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ + --set reason="Initial deployment" \ --namespace backend \ --name delivery-v0.1.0 @@ -246,6 +246,7 @@ helm install $HELM_CHARTS/package/ \ --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ --set dockerregistry=$ACR_SERVER \ + --set reason="Initial deployment" \ --namespace backend \ --name package-v0.1.0 @@ -296,6 +297,7 @@ helm install $HELM_CHARTS/workflow/ \ --set keyvault.resourcegroup=$RESOURCE_GROUP \ --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ --set keyvault.tenantid=$TENANT_ID \ + --set reason="Initial deployment" \ --namespace backend \ --name workflow-v0.1.0 @@ -363,6 +365,7 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ + --set reason="Initial deployment" \ --namespace backend \ --name ingestion-v0.1.0 @@ -413,6 +416,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ + --set reason="Initial deployment" \ --namespace backend \ --name dronescheduler-v0.1.0 From 26e5bde085d0982c2340f1a2509cb74b0855d257 Mon Sep 17 00:00:00 2001 From: Mike Wasson Date: Mon, 15 Apr 2019 12:38:37 -0700 Subject: [PATCH 119/246] Update deployment steps for drone scheduler (#125) The deployment steps try to get the Resource ID and Client ID from the "azuredeploy" deployment but these are actually returned in the "azuredeploy-identities" deployment --- deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment.md b/deployment.md index 48d24bce..b859d810 100644 --- a/deployment.md +++ b/deployment.md @@ -391,8 +391,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) ``` Build and publish the container image From 583c67dea58ab6fcbbe9a3134f6a279848b18685 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 8 Apr 2019 17:42:30 +0000 Subject: [PATCH 120/246] Merged PR 732: add missing mock for environment add missing mock for environment --- .../ingestion/util/ClientPoolImpl.java | 12 ++++++------ .../ingestion/util/Environment.java | 10 ++++++++++ .../ingestion/util/EnvironmentImpl.java | 15 +++++++++++++++ .../configuration/TestAppConfig.java | 19 +++++++++++++++++-- 4 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/Environment.java create mode 100644 src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/EnvironmentImpl.java diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java index 0304160d..af3a61bb 100644 --- a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/ClientPoolImpl.java @@ -18,7 +18,7 @@ public class ClientPoolImpl implements ClientPool { private static final String SCHEME = "https"; - + private final InstrumentedQueueClient[] queueClients; private final String[] queueNames; private final ApplicationProperties appProperties; @@ -28,14 +28,14 @@ public class ClientPoolImpl implements ClientPool { private final ServiceBusTracing tracing; @Autowired - public ClientPoolImpl(ApplicationProperties appProps, ServiceBusTracing tracing) { + public ClientPoolImpl(ApplicationProperties appProps, ServiceBusTracing tracing, Environment environment) { this.appProperties = appProps; this.tracing = tracing; - this.queueNames = System.getenv(appProperties.getEnvQueueName()).split(","); - nameSpace = System.getenv(appProperties.getEnvNameSpace()); - sasKeyName = System.getenv(appProperties.getEnvsasKeyName()); - sasKey = System.getenv(appProperties.getEnvsasKey()); + this.queueNames = environment.getenv(appProperties.getEnvQueueName()).split(","); + nameSpace = environment.getenv(appProperties.getEnvNameSpace()); + sasKeyName = environment.getenv(appProperties.getEnvsasKeyName()); + sasKey = environment.getenv(appProperties.getEnvsasKey()); this.queueClients = new InstrumentedQueueClient[this.appProperties.getMessageAmqpClientPoolSize()]; } diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/Environment.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/Environment.java new file mode 100644 index 00000000..3b167ee6 --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/Environment.java @@ -0,0 +1,10 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +public interface Environment { + public String getenv(String name); +} diff --git a/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/EnvironmentImpl.java b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/EnvironmentImpl.java new file mode 100644 index 00000000..4475df5f --- /dev/null +++ b/src/shipping/ingestion/src/main/java/com/fabrikam/dronedelivery/ingestion/util/EnvironmentImpl.java @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +package com.fabrikam.dronedelivery.ingestion.util; + +import org.springframework.stereotype.Component; + +@Component +public class EnvironmentImpl implements Environment { + public String getenv(String name) { + return System.getenv(name); + } +} diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java index ef1bc4db..df27e6e6 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/configuration/TestAppConfig.java @@ -3,17 +3,18 @@ import com.fabrikam.dronedelivery.ingestion.util.ClientPool; import com.fabrikam.dronedelivery.ingestion.util.ClientPoolImpl; import com.fabrikam.dronedelivery.ingestion.util.InstrumentedQueueClient; +import com.fabrikam.dronedelivery.ingestion.util.Environment; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.azure.servicebus.IMessage; import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import org.mockito.Mockito; import static org.mockito.Mockito.*; import java.net.URISyntaxException; import java.util.concurrent.CompletableFuture; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -21,10 +22,24 @@ @Configuration public class TestAppConfig { + @Autowired + private ApplicationProperties appProperites; + @Bean @Primary public TelemetryClient getTelemetryClient() { - return Mockito.mock(TelemetryClient.class); + return mock(TelemetryClient.class); + } + + @Bean + @Primary + public Environment getEnvironment() { + Environment envMock = mock(Environment.class); + + when(envMock.getenv(appProperites.getEnvQueueName())) + .thenReturn("test-queue"); + + return envMock; } @Bean From 5514fd612c487b4de0dd2efcdecceef72c2cd26a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 8 Apr 2019 17:46:12 +0000 Subject: [PATCH 121/246] Merged PR 731: reduce docker image size up to 100MiB reduce docker image size up to 100MiB --- src/shipping/ingestion/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shipping/ingestion/Dockerfile b/src/shipping/ingestion/Dockerfile index f65cc83f..2d24edbf 100644 --- a/src/shipping/ingestion/Dockerfile +++ b/src/shipping/ingestion/Dockerfile @@ -19,6 +19,6 @@ WORKDIR /app ADD https://github.com/Microsoft/ApplicationInsights-Java/releases/download/2.3.0/applicationinsights-agent-2.3.0.jar ./appinsights/applicationinsights-agent-2.3.0.jar ADD /etc/AI-Agent.xml ./appinsights/AI-Agent.xml -COPY --from=build /usr/src/app/target ./ +COPY --from=build /usr/src/app/target/ingestion-0.1.0.jar ./ ENTRYPOINT ["java","-Djava.security.egdfile=file:/dev/./urandom","-javaagent:/app/appinsights/applicationinsights-agent-2.3.0.jar","-jar","ingestion-0.1.0.jar"] From 8dbfd775b67ddef7a379293b3daac97ea60f278c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 10 Apr 2019 15:56:32 +0000 Subject: [PATCH 122/246] Merged PR 735: remove hardcoded acr name remove hardcoded acr name https://github.com/mspnp/microservices-reference-implementation/issues/123 solved: #9145, #123 Related work items: #9145 --- src/shipping/delivery/azure-pipelines-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shipping/delivery/azure-pipelines-ci.yml b/src/shipping/delivery/azure-pipelines-ci.yml index 58db4e9d..98973d1e 100644 --- a/src/shipping/delivery/azure-pipelines-ci.yml +++ b/src/shipping/delivery/azure-pipelines-ci.yml @@ -145,7 +145,7 @@ steps: inlineScript: | force='--force' - packageExists="$(az acr helm list -n pnpdhacr --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" export force export packageExists @@ -154,4 +154,4 @@ steps: force='' fi - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name pnpdhacr $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; From 26b9aa1a9873f3ba6c906dee3b7d1f63c7ce88b3 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 10 Apr 2019 16:08:11 +0000 Subject: [PATCH 123/246] Merged PR 725: improve app properties testing improve app properties testing --- .../ingestion/ApplicationPropertiesTest.java | 50 ++++++------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java index 5a2bb3da..e0736cca 100644 --- a/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java +++ b/src/shipping/ingestion/src/test/java/com/fabrikam/dronedelivery/ingestion/ApplicationPropertiesTest.java @@ -1,48 +1,26 @@ package com.fabrikam.dronedelivery.ingestion; import static org.junit.Assert.*; -import static org.mockito.Mockito.times; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import com.fabrikam.dronedelivery.ingestion.configuration.ApplicationProperties; public class ApplicationPropertiesTest { - - public class Environment { - public String getenv(String envName) { - return System.getenv(envName); - } - } - - - @Configuration - public static class Config { - @Bean - public static PropertySourcesPlaceholderConfigurer propertyConfigurer() { - return new PropertySourcesPlaceholderConfigurer(); - } - } - - - @Before - public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); - } - @Test - public void canGetPropertiesfromEnvVariables() { - Environment systemMock = Mockito.mock(Environment.class); - Mockito.when(systemMock.getenv("ENV_QUEUE_NAME")).thenReturn("variableMock"); + public void canSetPropertyValuesUsedByClientPoolImpl_thenGetPropertyValues() { + // Arrange ApplicationProperties appProps = new ApplicationProperties(); - assertEquals(systemMock.getenv(appProps.getEnvQueueName()),"variableMock"); - Mockito.verify(systemMock, times(1)) - .getenv(appProps.getEnvQueueName()); - } + // Act + appProps.setEnvQueueName("ENV_QUEUE_NAME_MODIFIED"); + appProps.setEnvNameSpace("ENV_QUEUE_NS_MODIFIED"); + appProps.setEnvsasKeyName("ENV_KEY_NAME_MODIFIED"); + appProps.setEnvsasKey("ENV_KEY_VALUE_MODIFIED"); + + // Assert + assertEquals("ENV_QUEUE_NAME_MODIFIED", appProps.getEnvQueueName()); + assertEquals("ENV_QUEUE_NS_MODIFIED", appProps.getEnvNameSpace()); + assertEquals("ENV_KEY_NAME_MODIFIED", appProps.getEnvsasKeyName()); + assertEquals("ENV_KEY_VALUE_MODIFIED", appProps.getEnvsasKey()); + } } From 352f5ddf20314950ea6dcde86d02d7c8bd344f08 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 10 Apr 2019 17:06:57 +0000 Subject: [PATCH 124/246] Merged PR 734: version secret name --- charts/ingestion/templates/ingestion-deploy.yaml | 8 ++++---- charts/ingestion/templates/ingestion-ingress.yaml | 3 ++- .../templates/ingestion-secret-ingress-queue.yaml | 2 +- .../ingestion/templates/ingestion-secret-ingress-tls.yaml | 3 ++- charts/package/templates/package-deploy.yaml | 4 ++-- charts/package/templates/package-secrets.yaml | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 38f7ef93..4460abc5 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -57,7 +57,7 @@ spec: - name: QUEUE_NAMESPACE valueFrom: secretKeyRef: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets key: queue_namespace - name: QUEUE_NAME valueFrom: @@ -67,17 +67,17 @@ spec: - name: QUEUE_KEYNAME valueFrom: secretKeyRef: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets key: queue_keyname - name: QUEUE_KEYVALUE valueFrom: secretKeyRef: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets key: queue_keyvalue - name: APPINSIGHTS_INSTRUMENTATIONKEY valueFrom: secretKeyRef: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets key: appinsights-ikey - name: APPINSIGHTS_LOGGERLEVEL value: {{ default "error" .Values.telemetry.level | quote }} diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index 0f50d81c..c1391449 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -13,12 +13,13 @@ metadata: name: ingestion-ingress spec: {{- if .Values.ingress.tls }} + {{- $relname := .Release.Name }} tls: {{- range .Values.ingress.hosts }} {{- if .tls }} - hosts: - {{ .name }} - secretName: {{ .tlsSecretName }} + secretName: {{ $relname }}-{{ .tlsSecretName }} {{- end }} {{- end }} {{- end }} diff --git a/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml b/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml index d7e6bbd9..ae109dd1 100644 --- a/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml +++ b/charts/ingestion/templates/ingestion-secret-ingress-queue.yaml @@ -1,7 +1,7 @@ kind: Secret apiVersion: v1 metadata: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets type: Opaque data: appinsights-ikey: {{ .Values.secrets.appinsights.ikey | b64enc }} diff --git a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml index f8e52cb9..3feb02a6 100644 --- a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml +++ b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml @@ -1,9 +1,10 @@ {{- if .Values.ingress.tls}} +{{- $relname := .Release.Name }} {{- range .Values.ingress.tls.secrets }} kind: Secret apiVersion: v1 metadata: - name: {{ .name }} + name: {{ $relname }}-{{ .name }} type: kubernetes.io/tls data: tls.crt: {{ .certificate | b64enc }} diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index bc194dfa..ca6e46c2 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -45,14 +45,14 @@ spec: - name: CONNECTION_STRING valueFrom: secretKeyRef: - name: package-secrets + name: {{ .Release.Name }}-secrets key: mongodb-pwd - name: COLLECTION_NAME value: {{ default "packages" .Values.cosmosDb.collectionName }} - name: APPINSIGHTS_INSTRUMENTATIONKEY valueFrom: secretKeyRef: - name: package-secrets + name: {{ .Release.Name }}-secrets key: appinsights-ikey - name: LOG_LEVEL value: {{ .Values.log.level }} diff --git a/charts/package/templates/package-secrets.yaml b/charts/package/templates/package-secrets.yaml index 8c44cb70..7e431dd4 100644 --- a/charts/package/templates/package-secrets.yaml +++ b/charts/package/templates/package-secrets.yaml @@ -9,7 +9,7 @@ kind: Secret apiVersion: v1 metadata: - name: package-secrets + name: {{ .Release.Name }}-secrets type: Opaque data: appinsights-ikey: {{ .Values.secrets.appinsights.ikey | b64enc }} From 7099d4f559708c6220753720606affe54a933e1f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 10 Apr 2019 17:39:14 +0000 Subject: [PATCH 125/246] Merged PR 733: improve multistage in ingestion for ci improve multistage in ingestion for ci - get it ready to extract junit results - download dependencies only instead of package for maven image. Save time building ingestion using --target to build specific stages (e.g. maven), so it can be pushed to a remote repository. Subsecuent builds, use --cache-from to pull them and prevent your build from executing this stage again. - remove files that are not in use. related: #9143 Related work items: #9143 --- src/shipping/ingestion/Dockerfile | 24 ++- src/shipping/ingestion/mvn-entrypoint.sh | 41 ---- src/shipping/ingestion/mvnw | 233 --------------------- src/shipping/ingestion/mvnw.cmd | 145 ------------- src/shipping/ingestion/pom.xml | 1 - src/shipping/ingestion/settings-docker.xml | 6 - 6 files changed, 18 insertions(+), 432 deletions(-) delete mode 100644 src/shipping/ingestion/mvn-entrypoint.sh delete mode 100644 src/shipping/ingestion/mvnw delete mode 100644 src/shipping/ingestion/mvnw.cmd delete mode 100644 src/shipping/ingestion/settings-docker.xml diff --git a/src/shipping/ingestion/Dockerfile b/src/shipping/ingestion/Dockerfile index 2d24edbf..1ea40cd3 100644 --- a/src/shipping/ingestion/Dockerfile +++ b/src/shipping/ingestion/Dockerfile @@ -2,14 +2,26 @@ FROM openjdk:8-jre-alpine as base WORKDIR /usr/src/app EXPOSE 80 -FROM maven:3.5-jdk-8-slim as build +FROM maven:3.6.0-jdk-8-slim as maven WORKDIR /usr/src/app COPY pom.xml ./ -COPY settings-docker.xml /usr/share/maven/ref/ -RUN /usr/local/bin/mvn-entrypoint.sh \ - mvn package -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true --fail-never -COPY . . +RUN mvn clean dependency:go-offline + +FROM maven as build +WORKDIR /usr/src/app + +COPY src ./src +RUN mvn compile + +FROM build as testrunner +WORKDIR /usr/src/app + +ENTRYPOINT ["mvn", "verify"] + +FROM build as package +WORKDIR /usr/src/app + RUN mvn package -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true FROM base as final @@ -19,6 +31,6 @@ WORKDIR /app ADD https://github.com/Microsoft/ApplicationInsights-Java/releases/download/2.3.0/applicationinsights-agent-2.3.0.jar ./appinsights/applicationinsights-agent-2.3.0.jar ADD /etc/AI-Agent.xml ./appinsights/AI-Agent.xml -COPY --from=build /usr/src/app/target/ingestion-0.1.0.jar ./ +COPY --from=package /usr/src/app/target/ingestion-0.1.0.jar ./ ENTRYPOINT ["java","-Djava.security.egdfile=file:/dev/./urandom","-javaagent:/app/appinsights/applicationinsights-agent-2.3.0.jar","-jar","ingestion-0.1.0.jar"] diff --git a/src/shipping/ingestion/mvn-entrypoint.sh b/src/shipping/ingestion/mvn-entrypoint.sh deleted file mode 100644 index f8ed7fda..00000000 --- a/src/shipping/ingestion/mvn-entrypoint.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/bash -eu - -set -o pipefail - -# Copy files from /usr/share/maven/ref into ${MAVEN_CONFIG} -# So the initial ~/.m2 is set with expected content. -# Don't override, as this is just a reference setup -copy_reference_file() { - local root="${1}" - local f="${2%/}" - local logfile="${3}" - local rel="${f/${root}/}" # path relative to /usr/share/maven/ref/ - echo "$f" >> "$logfile" - echo " $f -> $rel" >> "$logfile" - if [[ ! -e ${MAVEN_CONFIG}/${rel} || $f = *.override ]] - then - echo "copy $rel to ${MAVEN_CONFIG}" >> "$logfile" - mkdir -p "${MAVEN_CONFIG}/$(dirname "${rel}")" - cp -r "${f}" "${MAVEN_CONFIG}/${rel}"; - fi; -} - -copy_reference_files() { - local log="$MAVEN_CONFIG/copy_reference_file.log" - - if (touch "${log}" > /dev/null 2>&1) - then - echo "--- Copying files at $(date)" >> "$log" - find /usr/share/maven/ref/ -type f -exec bash -eu -c 'copy_reference_file /usr/share/maven/ref/ "$1" "$2"' _ {} "$log" \; - else - echo "Can not write to ${log}. Wrong volume permissions? Carrying on ..." - fi -} - -export -f copy_reference_file -copy_reference_files -unset MAVEN_CONFIG - -cd $SOURCEPATH - -exec "$@" diff --git a/src/shipping/ingestion/mvnw b/src/shipping/ingestion/mvnw deleted file mode 100644 index a1ba1bf5..00000000 --- a/src/shipping/ingestion/mvnw +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/sh -# ---------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -# Maven2 Start Up Batch script -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir -# -# Optional ENV vars -# ----------------- -# M2_HOME - location of maven2's installed home dir -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files -# ---------------------------------------------------------------------------- - -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false -case "`uname`" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # - # Look for the Apple JDKs first to preserve the existing behaviour, and then look - # for the new JDKs provided by Oracle. - # - if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then - # - # Apple JDKs - # - export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home - fi - - if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then - # - # Apple JDKs - # - export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home - fi - - if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then - # - # Oracle JDKs - # - export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home - fi - - if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then - # - # Apple JDKs - # - export JAVA_HOME=`/usr/libexec/java_home` - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -if [ -z "$M2_HOME" ] ; then - ## resolve links - $0 may be a link to maven's home - PRG="$0" - - # need this for relative symlinks - while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG="`dirname "$PRG"`/$link" - fi - done - - saveddir=`pwd` - - M2_HOME=`dirname "$PRG"`/.. - - # make it fully qualified - M2_HOME=`cd "$M2_HOME" && pwd` - - cd "$saveddir" - # echo Using m2 at $M2_HOME -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --unix "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# For Migwn, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$M2_HOME" ] && - M2_HOME="`(cd "$M2_HOME"; pwd)`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" - # TODO classpath? -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - if $darwin ; then - javaHome="`dirname \"$javaExecutable\"`" - javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" - else - javaExecutable="`readlink -f \"$javaExecutable\"`" - fi - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --path --windows "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --windows "$CLASSPATH"` -fi - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - local basedir=$(pwd) - local wdir=$(pwd) - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - wdir=$(cd "$wdir/.."; pwd) - done - echo "${basedir}" -} - -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - echo "$(tr -s '\n' ' ' < "$1")" - fi -} - -export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -exec "$JAVACMD" \ - $MAVEN_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} "$@" diff --git a/src/shipping/ingestion/mvnw.cmd b/src/shipping/ingestion/mvnw.cmd deleted file mode 100644 index 2b934e89..00000000 --- a/src/shipping/ingestion/mvnw.cmd +++ /dev/null @@ -1,145 +0,0 @@ -@REM ---------------------------------------------------------------------------- -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ---------------------------------------------------------------------------- - -@REM ---------------------------------------------------------------------------- -@REM Maven2 Start Up Batch script -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir -@REM -@REM Optional ENV vars -@REM M2_HOME - location of maven2's installed home dir -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files -@REM ---------------------------------------------------------------------------- - -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" -if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -set MAVEN_CMD_LINE_ARGS=%* - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" - -set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" -if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%" == "on" pause - -if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% - -exit /B %ERROR_CODE% \ No newline at end of file diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 250887e4..397ff779 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -24,7 +24,6 @@ 1.16.10 2.7.5 2.1.2.RELEASE - com.fabrikam.dronedelivery.ingestion.IngestionApplication diff --git a/src/shipping/ingestion/settings-docker.xml b/src/shipping/ingestion/settings-docker.xml deleted file mode 100644 index 586c587c..00000000 --- a/src/shipping/ingestion/settings-docker.xml +++ /dev/null @@ -1,6 +0,0 @@ - - /usr/share/maven/ref/repository - From e9116370c7a18e72a9f846bf173d4db614e8d095 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 11 Apr 2019 14:42:06 +0000 Subject: [PATCH 126/246] Merged PR 738: single svc config file per service solved: #9159 Related work items: #9159 --- .../delivery-service-experimental.yaml | 33 -------------- .../templates/delivery-service-sxs.yaml | 28 ------------ .../delivery/templates/delivery-service.yaml | 44 ++++++++++++++----- .../dronescheduler-service-experimental.yaml | 33 -------------- .../templates/dronescheduler-service-sxs.yaml | 28 ------------ .../templates/dronescheduler-service.yaml | 44 ++++++++++++++----- .../ingestion-service-experimental.yaml | 33 -------------- .../templates/ingestion-service-sxs.yaml | 28 ------------ .../templates/ingestion-service.yaml | 44 ++++++++++++++----- .../package-service-experimental.yaml | 33 -------------- .../templates/package-service-sxs.yaml | 29 ------------ charts/package/templates/package-service.yaml | 44 ++++++++++++++----- 12 files changed, 136 insertions(+), 285 deletions(-) delete mode 100644 charts/delivery/templates/delivery-service-experimental.yaml delete mode 100644 charts/delivery/templates/delivery-service-sxs.yaml delete mode 100644 charts/dronescheduler/templates/dronescheduler-service-experimental.yaml delete mode 100644 charts/dronescheduler/templates/dronescheduler-service-sxs.yaml delete mode 100644 charts/ingestion/templates/ingestion-service-experimental.yaml delete mode 100644 charts/ingestion/templates/ingestion-service-sxs.yaml delete mode 100644 charts/package/templates/package-service-experimental.yaml delete mode 100644 charts/package/templates/package-service-sxs.yaml diff --git a/charts/delivery/templates/delivery-service-experimental.yaml b/charts/delivery/templates/delivery-service-experimental.yaml deleted file mode 100644 index 52600d65..00000000 --- a/charts/delivery/templates/delivery-service-experimental.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Delivery service experimental -################################################################################################### - -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "delivery-v0.1.0") }} -apiVersion: v1 -kind: Service -metadata: - name: delivery-experimental - labels: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "delivery.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -{{ end }} diff --git a/charts/delivery/templates/delivery-service-sxs.yaml b/charts/delivery/templates/delivery-service-sxs.yaml deleted file mode 100644 index 3d78c796..00000000 --- a/charts/delivery/templates/delivery-service-sxs.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Delivery service sxs -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: {{ include "delivery.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "delivery.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 55e546b4..11d87d93 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -6,28 +6,52 @@ ################################################################################################### # Delivery service ################################################################################################### - +{{- $appname := include "delivery.name" . -}} +{{- $chart := include "delivery.chart" . -}} +{{- $instancename := .Release.Name }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "delivery.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ if (eq .Release.Name "delivery-v0.1.0") }} +{{- range tuple "delivery" "delivery-experimental" }} +{{- $svcname := . }} +--- # the following object is meant ot be created first time only. # its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "delivery-v0.1.0") }} apiVersion: v1 kind: Service metadata: - name: delivery + name: {{ $svcname }} labels: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "delivery.chart" . }} + helm.sh/chart: {{ $chart }} spec: ports: - name: http port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: {{ template "delivery.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ end }} {{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml b/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml deleted file mode 100644 index 9ba17ad6..00000000 --- a/charts/dronescheduler/templates/dronescheduler-service-experimental.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Dronescheduler service experimental -################################################################################################### - -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} -apiVersion: v1 -kind: Service -metadata: - name: dronescheduler-experimental - labels: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "dronescheduler.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -{{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml b/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml deleted file mode 100644 index 59d80f37..00000000 --- a/charts/dronescheduler/templates/dronescheduler-service-sxs.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Dronescheduler service -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: {{ include "dronescheduler.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "dronescheduler.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 8080 - selector: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index fd2bdd01..a25d1990 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -6,28 +6,52 @@ ################################################################################################### # Dronescheduler service ################################################################################################### - +{{- $appname := include "dronescheduler.name" . -}} +{{- $chart := include "dronescheduler.chart" . -}} +{{- $instancename := .Release.Name }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dronescheduler.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} +{{- range tuple "dronescheduler" "dronescheduler-experimental" }} +{{- $svcname := . }} +--- # the following object is meant ot be created first time only. # its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} apiVersion: v1 kind: Service metadata: - name: dronescheduler + name: {{ $svcname }} labels: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "dronescheduler.chart" . }} + helm.sh/chart: {{ $chart }} spec: ports: - name: http port: 80 targetPort: 8080 selector: - app.kubernetes.io/name: {{ template "dronescheduler.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ end }} {{ end }} diff --git a/charts/ingestion/templates/ingestion-service-experimental.yaml b/charts/ingestion/templates/ingestion-service-experimental.yaml deleted file mode 100644 index 4c24b666..00000000 --- a/charts/ingestion/templates/ingestion-service-experimental.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ingestion service experimental -################################################################################################### - -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "ingestion-v0.1.0") }} -apiVersion: v1 -kind: Service -metadata: - name: ingestion-experimental - labels: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "ingestion.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -{{ end }} diff --git a/charts/ingestion/templates/ingestion-service-sxs.yaml b/charts/ingestion/templates/ingestion-service-sxs.yaml deleted file mode 100644 index 36d2ae0b..00000000 --- a/charts/ingestion/templates/ingestion-service-sxs.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ingestion service sxs -################################################################################################### -apiVersion: v1 -kind: Service -metadata: - name: {{ include "ingestion.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "ingestion.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index 4527ac4d..68df3612 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -6,28 +6,52 @@ ################################################################################################### # ingestion service ################################################################################################### - +{{- $appname := include "ingestion.name" . -}} +{{- $chart := include "ingestion.chart" . -}} +{{- $instancename := .Release.Name }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ingestion.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ if (eq .Release.Name "ingestion-v0.1.0") }} +{{- range tuple "ingestion" "ingestion-experimental" }} +{{- $svcname := . }} +--- # the following object is meant ot be created first time only. # its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "ingestion-v0.1.0") }} apiVersion: v1 kind: Service metadata: - name: ingestion + name: {{ $svcname }} labels: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "ingestion.chart" . }} + helm.sh/chart: {{ $chart }} spec: ports: - name: http port: 80 targetPort: 80 selector: - app.kubernetes.io/name: {{ template "ingestion.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ end }} {{ end }} diff --git a/charts/package/templates/package-service-experimental.yaml b/charts/package/templates/package-service-experimental.yaml deleted file mode 100644 index ab528da5..00000000 --- a/charts/package/templates/package-service-experimental.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Package service experimental -################################################################################################### - -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "package-v0.1.0") }} -apiVersion: v1 -kind: Service -metadata: - name: package-experimental - labels: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "package.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -{{ end }} diff --git a/charts/package/templates/package-service-sxs.yaml b/charts/package/templates/package-service-sxs.yaml deleted file mode 100644 index 4b532e7a..00000000 --- a/charts/package/templates/package-service-sxs.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# Package service sxs -################################################################################################### - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "package.fullname" . | replace "." "" }} - labels: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "package.chart" . }} -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index f31c9b27..b6195a8f 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -6,28 +6,52 @@ ################################################################################################### # package service ################################################################################################### - +{{- $appname := include "package.name" . -}} +{{- $chart := include "package.chart" . -}} +{{- $instancename := .Release.Name }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "package.fullname" . | replace "." "" }} + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ if (eq .Release.Name "package-v0.1.0") }} +{{- range tuple "package" "package-experimental" }} +{{- $svcname := . }} +--- # the following object is meant ot be created first time only. # its configuration will be later managed by CI/CD -{{ if (eq .Release.Name "package-v0.1.0") }} apiVersion: v1 kind: Service metadata: - name: package + name: {{ $svcname }} labels: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: azurepipelines - app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery - helm.sh/chart: {{ include "package.chart" . }} + helm.sh/chart: {{ $chart }} spec: ports: - name: http port: 80 targetPort: 80 selector: - app.kubernetes.io/name: {{ template "package.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} +{{ end }} {{ end }} From 9559cd0d493ec16d533082f4f6792a7e1a56bbaf Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 11 Apr 2019 15:58:53 +0000 Subject: [PATCH 127/246] Merged PR 737: version ids name attempt to create identity only if present solved: #9163 Related work items: #9163 --- charts/delivery/templates/delivery-deploy.yaml | 7 ++++--- charts/delivery/templates/delivery-identity.yaml | 9 +++++---- .../dronescheduler/templates/dronescheduler-deploy.yaml | 7 ++++--- .../templates/dronescheduler-identity.yaml | 9 +++++---- charts/workflow/templates/workflow-deploy.yaml | 7 ++++--- charts/workflow/templates/workflow-identity.yaml | 9 +++++---- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 702d80ae..89e45bab 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Delivery ################################################################################################### +{{- $fullname := include "delivery.fullname" . | replace "." "" }} apiVersion: apps/v1beta2 kind: Deployment metadata: - name: {{ include "delivery.fullname" . | replace "." "" }} + name: {{ $fullname }} labels: app.kubernetes.io/name: {{ include "delivery.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -18,7 +19,7 @@ metadata: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "delivery.chart" . }} - aadpodidbinding: delivery + aadpodidbinding: {{ $fullname }} annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: @@ -37,7 +38,7 @@ spec: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "delivery.chart" . }} - aadpodidbinding: delivery + aadpodidbinding: {{ $fullname }} spec: securityContext: fsGroup: 1 diff --git a/charts/delivery/templates/delivery-identity.yaml b/charts/delivery/templates/delivery-identity.yaml index f9141fa2..f300535d 100644 --- a/charts/delivery/templates/delivery-identity.yaml +++ b/charts/delivery/templates/delivery-identity.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Delivery service identity ################################################################################################### +{{- $fullname := include "delivery.fullname" . | replace "." "" }} apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: - name: delivery-identity + name: {{ $fullname }}-identity annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "0" @@ -21,10 +22,10 @@ spec: apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: - name: delivery-identity-binding + name: {{ $fullname }}-identity-binding annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "1" spec: - AzureIdentity: delivery-identity - Selector: delivery + AzureIdentity: {{ $fullname }}-identity + Selector: {{ $fullname }} diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index 72a30a66..0bdf28cc 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Dronescheduler ################################################################################################### +{{- $fullname := include "dronescheduler.fullname" . | replace "." "" }} apiVersion: apps/v1beta2 kind: Deployment metadata: - name: {{ include "dronescheduler.fullname" . | replace "." "" }} + name: {{ $fullname }} labels: app.kubernetes.io/name: {{ include "dronescheduler.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -18,7 +19,7 @@ metadata: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "dronescheduler.chart" . }} - aadpodidbinding: dronescheduler + aadpodidbinding: {{ $fullname }} annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: @@ -37,7 +38,7 @@ spec: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "dronescheduler.chart" . }} - aadpodidbinding: dronescheduler + aadpodidbinding: {{ $fullname }} spec: securityContext: fsGroup: 1 diff --git a/charts/dronescheduler/templates/dronescheduler-identity.yaml b/charts/dronescheduler/templates/dronescheduler-identity.yaml index ab7fc6e6..a5bea852 100644 --- a/charts/dronescheduler/templates/dronescheduler-identity.yaml +++ b/charts/dronescheduler/templates/dronescheduler-identity.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Dronescheduler service identity ################################################################################################### +{{- $fullname := include "dronescheduler.fullname" . | replace "." "" }} apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: - name: dronescheduler-identity + name: {{ $fullname }}-identity annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "0" @@ -21,10 +22,10 @@ spec: apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: - name: dronescheduler-identity-binding + name: {{ $fullname }}-identity-binding annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "1" spec: - AzureIdentity: dronescheduler-identity - Selector: dronescheduler + AzureIdentity: {{ $fullname }}-identity + Selector: {{ $fullname }} diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 86095234..7c41f03c 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Workflow ################################################################################################### +{{- $fullname := include "workflow.fullname" . | replace "." "" }} apiVersion: apps/v1beta2 kind: Deployment metadata: - name: {{ include "workflow.fullname" . | replace "." "" }} + name: {{ $fullname }} labels: app.kubernetes.io/name: {{ include "workflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -18,7 +19,7 @@ metadata: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "workflow.chart" . }} - aadpodidbinding: workflow + aadpodidbinding: {{ $fullname }} annotations: kubernetes.io/change-cause: {{ .Values.reason }} spec: @@ -37,7 +38,7 @@ spec: app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "workflow.chart" . }} - aadpodidbinding: workflow + aadpodidbinding: {{ $fullname }} spec: securityContext: fsGroup: 1 diff --git a/charts/workflow/templates/workflow-identity.yaml b/charts/workflow/templates/workflow-identity.yaml index c9b4836c..f2c3c5f9 100644 --- a/charts/workflow/templates/workflow-identity.yaml +++ b/charts/workflow/templates/workflow-identity.yaml @@ -6,10 +6,11 @@ ################################################################################################### # Workflow service identity ################################################################################################### +{{- $fullname := include "workflow.fullname" . | replace "." "" }} apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentity metadata: - name: workflow-identity + name: {{ $fullname }}-identity annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "0" @@ -21,10 +22,10 @@ spec: apiVersion: "aadpodidentity.k8s.io/v1" kind: AzureIdentityBinding metadata: - name: workflow-identity-binding + name: {{ $fullname }}-identity-binding annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "1" spec: - AzureIdentity: workflow-identity - Selector: workflow + AzureIdentity: {{ $fullname }}-identity + Selector: {{ $fullname }} From 2ddf2b8a773ab95ed1b7c30ca6fdc39504caf287 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 12 Apr 2019 12:01:36 +0000 Subject: [PATCH 128/246] Merged PR 740: add comment for known limitation add comment for known limitation Related work items: #8500 --- src/shipping/delivery/azure-pipelines-validation.yml | 8 ++++---- .../dronescheduler/azure-pipelines-validation.yml | 9 +++------ src/shipping/package/azure-pipelines-validation.yml | 6 +++--- src/shipping/workflow/azure-pipelines-validation.yml | 8 ++++---- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/shipping/delivery/azure-pipelines-validation.yml b/src/shipping/delivery/azure-pipelines-validation.yml index c44ba31f..dac4fc24 100644 --- a/src/shipping/delivery/azure-pipelines-validation.yml +++ b/src/shipping/delivery/azure-pipelines-validation.yml @@ -1,6 +1,6 @@ variables: BuildConfiguration: "Release" - + name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy @@ -33,10 +33,10 @@ trigger: - topic/experimental/* - paths: - include: + paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html + include: - - /src/shipping/delivery/ + - /src/shipping/delivery/ resources: - repo: self diff --git a/src/shipping/dronescheduler/azure-pipelines-validation.yml b/src/shipping/dronescheduler/azure-pipelines-validation.yml index 0929ff0b..c018c02f 100644 --- a/src/shipping/dronescheduler/azure-pipelines-validation.yml +++ b/src/shipping/dronescheduler/azure-pipelines-validation.yml @@ -20,12 +20,9 @@ pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch P trigger: batch: true - paths: - include: - - /src/shipping/dronescheduler/ - exclude: - - / - - /src/ + paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html + include: + - /src/shipping/dronescheduler/ branches: include: diff --git a/src/shipping/package/azure-pipelines-validation.yml b/src/shipping/package/azure-pipelines-validation.yml index bf7d8fa0..d511923a 100644 --- a/src/shipping/package/azure-pipelines-validation.yml +++ b/src/shipping/package/azure-pipelines-validation.yml @@ -30,10 +30,10 @@ trigger: - topic/experimental/* - paths: - include: + paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html + include: - - /src/shipping/package/ + - /src/shipping/package/ resources: - repo: self diff --git a/src/shipping/workflow/azure-pipelines-validation.yml b/src/shipping/workflow/azure-pipelines-validation.yml index 42456246..21854f43 100644 --- a/src/shipping/workflow/azure-pipelines-validation.yml +++ b/src/shipping/workflow/azure-pipelines-validation.yml @@ -1,6 +1,6 @@ variables: BuildConfiguration: "Release" - + name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy @@ -33,10 +33,10 @@ trigger: - topic/experimental/* - paths: - include: + paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html + include: - - /src/shipping/workflow/ + - /src/shipping/workflow/ resources: - repo: self From 2b5776c0c9e7e31b969dec7ed03f5699736e5dad Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 12 Apr 2019 17:50:59 +0000 Subject: [PATCH 129/246] Merged PR 742: identity & binding: re-create when upgrading re-create when upgrading now attempts to create the CRD on kubectl-apply but deliting it first case scenario: identities have changed (e.g. resource or client ids) without bumping service version. related: #9163 Related work items: #9163 --- charts/delivery/templates/delivery-identity.yaml | 6 ++++-- .../dronescheduler/templates/dronescheduler-identity.yaml | 6 ++++-- charts/workflow/templates/workflow-identity.yaml | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/charts/delivery/templates/delivery-identity.yaml b/charts/delivery/templates/delivery-identity.yaml index f300535d..02f099aa 100644 --- a/charts/delivery/templates/delivery-identity.yaml +++ b/charts/delivery/templates/delivery-identity.yaml @@ -12,7 +12,8 @@ kind: AzureIdentity metadata: name: {{ $fullname }}-identity annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "0" spec: type: 0 @@ -24,7 +25,8 @@ kind: AzureIdentityBinding metadata: name: {{ $fullname }}-identity-binding annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "1" spec: AzureIdentity: {{ $fullname }}-identity diff --git a/charts/dronescheduler/templates/dronescheduler-identity.yaml b/charts/dronescheduler/templates/dronescheduler-identity.yaml index a5bea852..7b3ca299 100644 --- a/charts/dronescheduler/templates/dronescheduler-identity.yaml +++ b/charts/dronescheduler/templates/dronescheduler-identity.yaml @@ -12,7 +12,8 @@ kind: AzureIdentity metadata: name: {{ $fullname }}-identity annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "0" spec: type: 0 @@ -24,7 +25,8 @@ kind: AzureIdentityBinding metadata: name: {{ $fullname }}-identity-binding annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "1" spec: AzureIdentity: {{ $fullname }}-identity diff --git a/charts/workflow/templates/workflow-identity.yaml b/charts/workflow/templates/workflow-identity.yaml index f2c3c5f9..a74a526e 100644 --- a/charts/workflow/templates/workflow-identity.yaml +++ b/charts/workflow/templates/workflow-identity.yaml @@ -12,7 +12,8 @@ kind: AzureIdentity metadata: name: {{ $fullname }}-identity annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "0" spec: type: 0 @@ -24,7 +25,8 @@ kind: AzureIdentityBinding metadata: name: {{ $fullname }}-identity-binding annotations: - "helm.sh/hook": "pre-install" + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": hook-failed,before-hook-creation "helm.sh/hook-weight": "1" spec: AzureIdentity: {{ $fullname }}-identity From 2a1388d71a79ef50ed29f673a8435c12bd0f50ce Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 12 Apr 2019 17:58:46 +0000 Subject: [PATCH 130/246] Merged PR 741: add ingestion ci pipeline YAML files add ingestion ci pipeline YAML files Ingestion service was the missing service in our CI pipelines. These 3 new files will enable the possibility of creating 2 new pipelines. Both pipelines are using as the very same template file for CI. But passing some arguments (full-ci & build-image) is possible to enable and execute some set of special tasks as shown below: - validation: - what? build/test - when? trigger on top of new refs feature/topic/PR/trunk branches - release: - what? build/test/push artifacts - when? trigger on top of new refs release branches - why? separate pipeline makes clear separation helping to reduce noise when dealing with CD. ``` (full-ci: true) azure-pipelines.yml --------------------------------------- |-------> azure-pipelines-ci.yml azure-pipelines-validation.yml ---------------------------- (full-ci: false && build-img: ) ``` solve: #9157 --- src/shipping/ingestion/azure-pipelines-ci.yml | 157 ++++++++++++++++++ .../ingestion/azure-pipelines-validation.yml | 49 ++++++ src/shipping/ingestion/azure-pipelines.yml | 31 ++++ 3 files changed, 237 insertions(+) create mode 100644 src/shipping/ingestion/azure-pipelines-ci.yml create mode 100644 src/shipping/ingestion/azure-pipelines-validation.yml create mode 100644 src/shipping/ingestion/azure-pipelines.yml diff --git a/src/shipping/ingestion/azure-pipelines-ci.yml b/src/shipping/ingestion/azure-pipelines-ci.yml new file mode 100644 index 00000000..88c483c3 --- /dev/null +++ b/src/shipping/ingestion/azure-pipelines-ci.yml @@ -0,0 +1,157 @@ +steps: +- script: echo '##vso[task.setvariable variable=repositoryName]ingestion' + name: setvarAzureContainerRegistry +- script: echo $(repositoryName) + name: echovarRepositoryName + +- script: echo '##vso[task.setvariable variable=chartPath]charts/ingestion' + name: setvarChartPath +- script: echo $(chartPath) + name: echovarChartPath + +- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/ingestion/Dockerfile' + name: setvarDockerFileName +- script: echo $(dockerFileName) + name: echovarDockerFileName + +- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' + name: setvarReleaseVersion +- script: echo $(releaseVersion) + name: echovarReleaseVersion + +- script: echo '##vso[task.setvariable variable=aksNamespace]backend' + name: setvarAKSNamespace +- script: echo $(aksNamespace) + name: echovarAKSNamespace + +- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' + name: setvarImageName +- script: echo $(imageName) + name: echovarImageName + +- task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + +- task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/usr/src/app/target/surefire-reports/' + + imageName: '$(imageName)-test' + + runInBackground: false + +- task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/TEST*.xml' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + +- task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + +- task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(AzureSubscription) + + azureContainerRegistry: $(AzureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + +- task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(AzureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(AzureKubernetesService) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + +- task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + +- task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(AzureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + force='--force' + packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" + + export force + export packageExists + + if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then + force='' + fi + + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; diff --git a/src/shipping/ingestion/azure-pipelines-validation.yml b/src/shipping/ingestion/azure-pipelines-validation.yml new file mode 100644 index 00000000..c3627972 --- /dev/null +++ b/src/shipping/ingestion/azure-pipelines-validation.yml @@ -0,0 +1,49 @@ +variables: + BuildConfiguration: "Release" + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + + - /src/shipping/ingestion/ + branches: + include: + + - master + + - release/ingestion/v* + + +trigger: + batch: true + paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html + include: + + - /src/shipping/ingestion/ + branches: + include: + + - master + + - feature/* + + - topic/* + +resources: +- repo: self + +jobs: + +# CI +- job: ingestioncivalidationjob + displayName: "ingestion CI Validation (build & test)" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: False + buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/heads/master')) ] + steps: + - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/ingestion/azure-pipelines.yml b/src/shipping/ingestion/azure-pipelines.yml new file mode 100644 index 00000000..b5a510d6 --- /dev/null +++ b/src/shipping/ingestion/azure-pipelines.yml @@ -0,0 +1,31 @@ +variables: + BuildConfiguration: "Release" + +name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) + +trigger: + branches: + include: + + # for new release to production: release flow strategy + - release/ingestion/v* + + - refs/relelase/ingestion/v* + +resources: +- repo: self + +jobs: + +# CI +- job: ingestionjobci + displayName: "ingestion CI" + pool: + vmImage: 'Ubuntu 16.04' + timeoutInMinutes: 90 + variables: + fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/ingestion/v') ] + steps: + - script: echo $(fullCI) + name: echovar + - template: ./azure-pipelines-ci.yml From 4ccd89f5a36fd6897778bbbd336a6201a0ce120e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 15 Apr 2019 14:53:28 +0000 Subject: [PATCH 131/246] Merged PR 744: rename missing versioned secret name rename missing versioned secret name --- charts/ingestion/templates/ingestion-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 4460abc5..40014e5b 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -62,7 +62,7 @@ spec: - name: QUEUE_NAME valueFrom: secretKeyRef: - name: ingestion-secrets + name: {{ .Release.Name }}-secrets key: queue_name - name: QUEUE_KEYNAME valueFrom: From 191f23b9f51d807ae516151cf3f84a5632670783 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 16 Apr 2019 16:40:38 +0000 Subject: [PATCH 132/246] Merged PR 747: include other required namespaces for CD include other required namespaces for CD --- k8s/k8s-rbac-ai.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/k8s/k8s-rbac-ai.yaml b/k8s/k8s-rbac-ai.yaml index 2fdeaec9..e71c2336 100644 --- a/k8s/k8s-rbac-ai.yaml +++ b/k8s/k8s-rbac-ai.yaml @@ -16,6 +16,15 @@ subjects: - kind: ServiceAccount name: default namespace: backend +- kind: ServiceAccount + name: default + namespace: backend-dev +- kind: ServiceAccount + name: default + namespace: backend-qa +- kind: ServiceAccount + name: default + namespace: backend-staging roleRef: kind: ClusterRole name: appinsights-k8s-property-reader From 3fe3a973d3f2de89a4aed4bf011e29e5707f308d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 17 Apr 2019 22:37:08 +0000 Subject: [PATCH 133/246] Merged PR 750: remove workaround for helm push --force arc helm now supports creating packages when they don't exist using --force. - remove the workaround for --force arg Related work items: #8534 --- src/shipping/delivery/azure-pipelines-ci.yml | 12 +----------- src/shipping/dronescheduler/azure-pipelines-ci.yml | 12 +----------- src/shipping/ingestion/azure-pipelines-ci.yml | 12 +----------- src/shipping/package/azure-pipelines-ci.yml | 12 +----------- src/shipping/workflow/azure-pipelines-ci.yml | 12 +----------- 5 files changed, 5 insertions(+), 55 deletions(-) diff --git a/src/shipping/delivery/azure-pipelines-ci.yml b/src/shipping/delivery/azure-pipelines-ci.yml index 98973d1e..76b6c494 100644 --- a/src/shipping/delivery/azure-pipelines-ci.yml +++ b/src/shipping/delivery/azure-pipelines-ci.yml @@ -144,14 +144,4 @@ steps: scriptLocation: inlineScript inlineScript: | - force='--force' - packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" - - export force - export packageExists - - if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then - force='' - fi - - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/dronescheduler/azure-pipelines-ci.yml b/src/shipping/dronescheduler/azure-pipelines-ci.yml index f61068cd..3b0e8d57 100644 --- a/src/shipping/dronescheduler/azure-pipelines-ci.yml +++ b/src/shipping/dronescheduler/azure-pipelines-ci.yml @@ -112,14 +112,4 @@ steps: scriptLocation: inlineScript inlineScript: | - force='--force' - packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" - - export force - export packageExists - - if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then - force='' - fi - - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/ingestion/azure-pipelines-ci.yml b/src/shipping/ingestion/azure-pipelines-ci.yml index 88c483c3..a8e98c62 100644 --- a/src/shipping/ingestion/azure-pipelines-ci.yml +++ b/src/shipping/ingestion/azure-pipelines-ci.yml @@ -144,14 +144,4 @@ steps: scriptLocation: inlineScript inlineScript: | - force='--force' - packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" - - export force - export packageExists - - if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then - force='' - fi - - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/package/azure-pipelines-ci.yml b/src/shipping/package/azure-pipelines-ci.yml index bb7597bc..7c063986 100644 --- a/src/shipping/package/azure-pipelines-ci.yml +++ b/src/shipping/package/azure-pipelines-ci.yml @@ -105,14 +105,4 @@ steps: scriptLocation: inlineScript inlineScript: | - force='--force' - packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" - - export force - export packageExists - - if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then - force='' - fi - - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/workflow/azure-pipelines-ci.yml b/src/shipping/workflow/azure-pipelines-ci.yml index 1c9e4e97..d1ffca8b 100644 --- a/src/shipping/workflow/azure-pipelines-ci.yml +++ b/src/shipping/workflow/azure-pipelines-ci.yml @@ -144,14 +144,4 @@ steps: scriptLocation: inlineScript inlineScript: | - force='--force' - packageExists="$(az acr helm list -n $(AzureContainerRegistryHelmRepo) --query "$(repositoryName)[?version=='$(Build.SourceBranchName)'].version")" - - export force - export packageExists - - if [ "$packageExists" = "[]" ] || [ "$packageExists" = "" ]; then - force='' - fi - - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) $force; + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; From 6088fd4d4ec19a5d4730994e684e435d08fc9a65 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 19 Apr 2019 14:46:28 +0000 Subject: [PATCH 134/246] Merged PR 743: version ingress object # Version ingress object ## Intro now the ingress supports multiple versioned endpoints that can be consumed by its clients. Please take a look below different case scenarios and their outcome: ## Reference ```tf:``` it means gate approvers confirmed a released version is ready to be productive ## Automated case scenarios ### Ingestion v0.1.0 ```t0```: - / --> svc/ingestion --> Faulted - /v0.1.0 --> svc/ingestion-v010 --> pod/ingestion-v010 ```tf```: - / --> svc/ingestion --> pod/ingestion-v010 - /v0.1.0 --> svc/ingestion-v010 --> pod/ingestion-v010 ### Ingestion 0.2.0 ```t0```: - / --> svc/ingestion --> pod/ingestion-v010 - /v0.1.0 --> svc/ingestion-v010 --> pod/ingestion-v010 - /v0.2.0 --> svc/ingestion-v020 --> pod/ingestion-v020 ```tf``` - / --> svc/ingestion --> pod/ingestion-v020 - /v0.1.0 --> svc/ingestion-v010 --> pod/ingestion-v010 - /v0.2.0 --> svc/ingestion-v020 --> pod/ingestion-v020 ## The manual case scenario ### Ingestion v0.1.0 ```t0``` == ```tf```: - / --> svc/ingestion --> pod/ingestion-v010 - /v0.1.0 --> svc/ingestion-v010 --> pod/ingestion-v010 --- .../templates/ingestion-ingress.yaml | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index c1391449..e28a80e6 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -4,16 +4,20 @@ # ------------------------------------------------------------ ################################################################################################### -# ingestion ingress +# ingress ################################################################################################### - +{{- $svcversion := .Chart.AppVersion | replace "." "" }} +{{- $appversion := .Chart.AppVersion }} +{{- $defaultversionedpath := printf "/%s" $appversion }} +{{- $relname := .Release.Name }} apiVersion: extensions/v1beta1 kind: Ingress metadata: - name: ingestion-ingress + name: {{ $relname }}-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / spec: {{- if .Values.ingress.tls }} - {{- $relname := .Release.Name }} tls: {{- range .Values.ingress.hosts }} {{- if .tls }} @@ -28,8 +32,18 @@ spec: - host: {{ .name }} http: paths: + {{ if .path }} + - path: {{ printf "%s/%s" .path $appversion | quote }} + {{ else }} + - path: {{ $defaultversionedpath | quote }} + {{ end }} + backend: + serviceName: "{{ .serviceName }}-{{ $svcversion }}" + servicePort: http + {{ if (eq $appversion "v0.1.0") }} - path: {{ default "/" .path }} backend: serviceName: "{{ .serviceName }}" servicePort: http + {{ end }} {{ end }} From 94ae7ed0e97ca16db3597f450e9ed2fde88778e4 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 25 Apr 2019 16:35:37 -0300 Subject: [PATCH 135/246] Address CVE-2019-1000613 --- src/shipping/ingestion/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/shipping/ingestion/pom.xml b/src/shipping/ingestion/pom.xml index 397ff779..559169c2 100644 --- a/src/shipping/ingestion/pom.xml +++ b/src/shipping/ingestion/pom.xml @@ -90,6 +90,12 @@ applicationinsights-logging-logback 2.3.0 + + + org.bouncycastle + bcprov-jdk15on + 1.61 + From 4c7abef7026d11a0871551085e34bf0d6fa169e4 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 15:36:28 -0300 Subject: [PATCH 136/246] [delivery] set requests/limits for delivery --- charts/delivery/templates/delivery-deploy.yaml | 8 ++++---- charts/delivery/values.yaml | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 89e45bab..0efcaa43 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -62,8 +62,8 @@ spec: containerPort: 8080 resources: requests: - cpu: 1 - memory: 2G + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} limits: - cpu: 1 - memory: 3G + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index cdc8a314..11a0eda1 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -16,3 +16,10 @@ keyvault: telemetry: level: "Error" reason: unknown +resources: + requests: + cpu: 350m + memory: 350Mi + limits: + cpu: 500m + memory: 500Mi From e93b5548f70b5b997bf1348606402135f3f2ca2e Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 15:36:55 -0300 Subject: [PATCH 137/246] [dronescheduler] set requests/limits for delivery --- .../templates/dronescheduler-deploy.yaml | 12 ++++++------ charts/dronescheduler/values.yaml | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index 0bdf28cc..9040f6df 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -57,9 +57,9 @@ spec: - name: service containerPort: 8080 resources: - requests: - cpu: 1 - memory: 1G - limits: - cpu: 1 - memory: 1G + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 10595902..44b1d731 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -13,3 +13,10 @@ keyvault: reason: unknown telemetry: level: "Error" +resources: + requests: + cpu: 450m + memory: 350Mi + limits: + cpu: 600m + memory: 500Mi From a9a98865e0729e51d6cba95d34faf0ba0946de9e Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 15:37:08 -0300 Subject: [PATCH 138/246] [ingestion] set requests/limits for delivery --- charts/ingestion/templates/ingestion-deploy.yaml | 7 +++++++ charts/ingestion/values.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 40014e5b..f14e04ba 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -83,3 +83,10 @@ spec: value: {{ default "error" .Values.telemetry.level | quote }} - name: CONTAINER_NAME value: *ingestion-container_name + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 5e49237e..62c23ccc 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -8,3 +8,10 @@ image: reason: unknown telemetry: level: "error" +resources: + requests: + cpu: 500m + memory: 600Mi + limits: + cpu: 650m + memory: 800Mi From 0eae781c01439739cda2217935df57358e396bf2 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 15:37:20 -0300 Subject: [PATCH 139/246] [package] set requests/limits for delivery --- charts/package/templates/package-deploy.yaml | 7 +++++++ charts/package/values.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index ca6e46c2..a6bb9045 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -61,3 +61,10 @@ spec: ports: - name: service containerPort: 80 + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 2ea62d0e..1de9dab4 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -10,3 +10,10 @@ log: level: error cosmosDb: collectionName: +resources: + requests: + cpu: 300m + memory: 100Mi + limits: + cpu: 410m + memory: 140Mi From 5d4d02772600ff61216a55f901be56e8d7600f08 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 15:37:49 -0300 Subject: [PATCH 140/246] [workflow] set requests/limits for delivery --- charts/workflow/templates/workflow-deploy.yaml | 7 +++++++ charts/workflow/values.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 7c41f03c..f42ed266 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -90,3 +90,10 @@ spec: resourcegroup: {{ .Values.keyvault.resourcegroup }} subscriptionid: {{ .Values.keyvault.subscriptionid }} tenantid: {{ .Values.keyvault.tenantid }} + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 8e5b1e3f..9ab718f3 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -28,3 +28,10 @@ keyvault: tenantid: telemetry: level: "Error" +resources: + requests: + cpu: 1700m + memory: 100Mi + limits: + cpu: 2300m + memory: 140Mi From f454856ded84ddba514955fee931460d341d886b Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 9 May 2019 16:25:45 -0300 Subject: [PATCH 141/246] upgrade to mongo 3.2.1 --- src/shipping/package/app/initializer.ts | 2 +- src/shipping/package/app/models/repository.ts | 6 +++--- src/shipping/package/package.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/shipping/package/app/initializer.ts b/src/shipping/package/app/initializer.ts index bd9e6c4c..fc1f741f 100644 --- a/src/shipping/package/app/initializer.ts +++ b/src/shipping/package/app/initializer.ts @@ -23,7 +23,7 @@ export class PackageServiceInitializer private static async initMongoDb(connection: string, collectionName: string) { try { - var db = (await MongoClient.connect(connection)); + var db = (await MongoClient.connect(connection)).db(); await db.command({ shardCollection: db.databaseName + '.' + collectionName, key: { tag: "hashed" } }); } catch (ex) { diff --git a/src/shipping/package/app/models/repository.ts b/src/shipping/package/app/models/repository.ts index f190ac89..9379f9b4 100644 --- a/src/shipping/package/app/models/repository.ts +++ b/src/shipping/package/app/models/repository.ts @@ -5,7 +5,7 @@ import { Package } from "./package" import { Settings } from '../util/settings'; -import * as Logger from '../util/logging'; +import * as Logger from '../util/logging'; import { MongoErrors } from '../util/mongo-err'; var MongoClient = require('mongodb').MongoClient; @@ -22,7 +22,7 @@ export class Repository static async initialize(connection: string) { - Repository.db = (await MongoClient.connect(connection)); + Repository.db = (await MongoClient.connect(connection)).db(); } private collection() { @@ -44,7 +44,7 @@ export class Repository } } } - + async updatePackage(p: Package) { await this.collection().update({_id:p._id}, p); } diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 1e0e573f..27e1ebea 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -23,7 +23,7 @@ "author": "Microsoft Patterns and Practices", "license": "MIT", "dependencies": { - "applicationinsights": "^1.1.0", + "applicationinsights": "^1.2.0", "fleek-context": "^1.0.6", "fleek-router": "^2.1.0", "fleek-validator": "^2.0.1", @@ -34,7 +34,7 @@ "koa-convert": "^1.2.0", "koa-cors": "0.0.16", "koa-route": "^3.2.0", - "mongodb": "2.2.28", + "mongodb": "3.2.1", "winston": "^2.4.0" }, "devDependencies": { From d03c092097b8590d2ced667c36e965d53dea2fc1 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 8 May 2019 16:41:05 -0300 Subject: [PATCH 142/246] upgrade to wiston 3.2.1 - remove deprecated winston version in favor a new one - address breaking changes since new major - remove not needed dev dep @types/winston 2.4.4 - replace handmade wrapper by using convenient winston native level-based methods --- src/shipping/package/app/util/logging.ts | 66 +++++++++++------------- src/shipping/package/package.json | 3 +- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/src/shipping/package/app/util/logging.ts b/src/shipping/package/app/util/logging.ts index c01a7cd2..177a1613 100644 --- a/src/shipping/package/app/util/logging.ts +++ b/src/shipping/package/app/util/logging.ts @@ -3,7 +3,27 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -import * as winston from 'winston'; +import { createLogger, format, transports } from 'winston'; +const { colorize, combine, timestamp, errors, printf, splat } = format; +const defaultFormat = combine( + colorize(), + timestamp(), + splat(), + errors(), + printf( + ({ + timestamp, + level, + message, + ...rest + }) => { + let restString = JSON.stringify(rest, undefined, 2); + restString = restString === '{}' ? '' : restString; + + return `[${timestamp}] ${level} - ${message} ${restString}`; + }, + ), + ); export interface ILogger { log(level: string, msg: string, meta?: any) @@ -12,45 +32,21 @@ export interface ILogger { warn(msg: string, meta?: any) error(msg: string, meta?: any) } + export function logger(level: string) { - winston.configure({ + const logger = createLogger({ level: level, transports: [ - new (winston.transports.Console)({ - level: level, - colorize: true, - timestamp: true - })] - }); + new transports.Console({ level: level}) + ], + format: defaultFormat + }); + + logger.info('winston logger created'); + return async function(ctx: any, next: any) { - ctx.state.logger = new WinstonLogger(); + ctx.state.logger = logger; await next(); } } -class WinstonLogger { - constructor() {} - log(level: string, msg: string, payload?: any) { - var meta : any = {}; - if (payload) { - winston.log(level, msg, payload); - } - else - { - winston.log(level, msg); - } - } - - info(msg: string, payload?: any) { - this.log('info', msg, payload); - } - debug(msg: string, payload?: any) { - this.log('debug', msg, payload); - } - warn(msg: string, payload?: any) { - this.log('warn', msg, payload); - } - error(msg: string, payload?: any) { - this.log('error', msg, payload); - } -} diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 27e1ebea..21f9060c 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -35,14 +35,13 @@ "koa-cors": "0.0.16", "koa-route": "^3.2.0", "mongodb": "3.2.1", - "winston": "^2.4.0" + "winston": "^3.2.1" }, "devDependencies": { "@types/koa": "^2.0.48", "@types/koa-bodyparser": "^5.0.1", "@types/mocha": "^5.2.6", "@types/node": "^11.10.5", - "@types/winston": "^2.4.4", "del": "^4.0.0", "gulp": "^4.0.0", "gulp-cli": "^2.0.1", From 19d8deb0269b27413c684236e779e2bccdc0476a Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 10 May 2019 14:36:30 -0300 Subject: [PATCH 143/246] log telemetry messages with same severity it's been logged in the app - removing colorize because https://github.com/microsoft/node-diagnostic-channel/issues/57 solved: #9094 --- src/shipping/package/app/util/logging.ts | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/shipping/package/app/util/logging.ts b/src/shipping/package/app/util/logging.ts index 177a1613..cf25b63b 100644 --- a/src/shipping/package/app/util/logging.ts +++ b/src/shipping/package/app/util/logging.ts @@ -4,26 +4,26 @@ // ------------------------------------------------------------ import { createLogger, format, transports } from 'winston'; -const { colorize, combine, timestamp, errors, printf, splat } = format; + +const { combine, timestamp, errors, printf, splat } = format; const defaultFormat = combine( - colorize(), - timestamp(), - splat(), - errors(), - printf( - ({ + timestamp(), + splat(), + errors(), + printf( + ({ timestamp, level, message, ...rest - }) => { + }) => { let restString = JSON.stringify(rest, undefined, 2); restString = restString === '{}' ? '' : restString; return `[${timestamp}] ${level} - ${message} ${restString}`; - }, - ), - ); + }, + ), +); export interface ILogger { log(level: string, msg: string, meta?: any) @@ -34,7 +34,6 @@ export interface ILogger { } export function logger(level: string) { - const logger = createLogger({ level: level, transports: [ From af5b34e9026036087137e0c00a97ad09451acb6f Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 16 Apr 2019 14:36:43 -0300 Subject: [PATCH 144/246] [delivery] add CI instructions --- deploymentCICD.md | 589 ++++++++++++++++++ src/shipping/delivery/azure-pipelines-ci.yml | 147 ----- .../delivery/azure-pipelines-validation.yml | 56 -- src/shipping/delivery/azure-pipelines.yml | 151 ++++- 4 files changed, 730 insertions(+), 213 deletions(-) create mode 100644 deploymentCICD.md delete mode 100644 src/shipping/delivery/azure-pipelines-ci.yml delete mode 100644 src/shipping/delivery/azure-pipelines-validation.yml diff --git a/deploymentCICD.md b/deploymentCICD.md new file mode 100644 index 00000000..74ee16c4 --- /dev/null +++ b/deploymentCICD.md @@ -0,0 +1,589 @@ +# Deploying the Reference Implementation + +## Prerequisites + +- Azure subscription +- [Azure CLI 2.0.49 or later](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Azure DevOps account](https://azure.microsoft.com/services/devops) +- [Docker](https://docs.docker.com/) +- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) +- [JQ](https://stedolan.github.io/jq/download/) + +> Note: in linux systems, it is possible to run the docker command without prefacing +> with sudo. For more information, please refer to [the Post-installation steps +> for linux](https://docs.docker.com/install/linux/linux-postinstall/) + +Clone or download this repo locally. + +```bash +git clone https://github.com/mspnp/microservices-reference-implementation.git +``` + +The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. + +## Generate a SSH rsa public/private key pair + +the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). +> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. + +## Azure Resources Provisioning + +Set environment variables. + +```bash +export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] + +export LOCATION=[YOUR_LOCATION_HERE] + +export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] + +export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export SUBSCRIPTION_NAME=$(az account show --query name --output tsv) +export TENANT_ID=$(az account show --query tenantId --output tsv) + +export PROJECT_ROOT=./microservices-reference-implementation +export K8S=$PROJECT_ROOT/k8s +export HELM_CHARTS=$PROJECT_ROOT/charts +``` + +Infrastructure Prerequisites + +```bash +# Log in to Azure +az login + +# Create a resource group and service principal for AKS +az group create --name $RESOURCE_GROUP --location $LOCATION && \ +export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ +export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ +export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ +export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) +``` + +Deployment + +> Note: this deployment might take up to 20 minutes + +```bash +# Deploy the managed identities +# These are deployed first in a separate template to avoid propagation delays with AAD +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file ${PROJECT_ROOT}/azuredeploy-identities.json +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) + +# Wait for AAD propagation +until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done + +# Deploy all other resources +# The version of kubernetes must be supported in the target region +export KUBERNETES_VERSION='1.12.6' +az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file ${PROJECT_ROOT}/azuredeploy.json \ +--parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + servicePrincipalId=${SP_OBJECT_ID} \ + kubernetesVersion=${KUBERNETES_VERSION} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ + deliveryIdName=${DELIVERY_ID_NAME} \ + deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ + droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ + droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ + workflowIdName=${WORKFLOW_ID_NAME} \ + workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} +``` + +Get outputs from Azure Deploy +```bash +# Shared +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) +``` + +Enable Azure Monitoring for the AKS cluster +```bash +az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +``` + +Download kubectl and create a k8s namespace +```bash +# Install kubectl +sudo az aks install-cli + +# Get the Kubernetes cluster credentials +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME + +# Create namespaces +kubectl create namespace backend && \ +kubectl create namespace frontend +``` + +Setup Helm in the container + +```bash +kubectl apply -f $K8S/tiller-rbac.yaml +helm init --service-account tiller +``` + +Integrate Application Insights instance + +```bash +# Acquire Instrumentation Key +export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) +export AI_IKEY=$(az resource show \ + -g $RESOURCE_GROUP \ + -n $AI_NAME \ + --resource-type "Microsoft.Insights/components" \ + --query properties.InstrumentationKey \ + -o tsv) + +# add RBAC for AppInsights +kubectl apply -f $K8S/k8s-rbac-ai.yaml +``` + +## Setup AAD pod identity and key vault flexvol infrastructure + +Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol + +Note: the tested nmi version was 1.4. It enables namespaced pod identity. + +```bash +# setup AAD pod identity +kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml + +kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml +``` + +## Setup Azure DevOps + +``` +# add extensions +az extension add --name azure-devops + +# export +AZURE_DEVOPS_ORG_NAME= +AZURE_DEVOPS_ORG=https://dev.azure.com/$AZURE_DEVOPS_ORG_NAME +AZURE_DEVOPS_PROJECT_NAME= +AZURE_REPOS_NAME= +AZURE_PIPELINES_SERVICE_CONN_NAME=default_cicd_service-connection + +# create project or skip this step if you are using an existent Azure DevOps project +az devops project create \ + --name $AZURE_DEVOPS_PROJECT_NAME \ + --organization $AZURE_DEVOPS_ORG + +# create repo +az repos create \ + --name $AZURE_REPOS_NAME \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME + +# create service principal for Azure Pipelines +az ad sp create-for-rbac + +# create Service Connection +az devops service-endpoint create \ + --service-endpoint-type azurerm \ + --name $AZURE_PIPELINES_SERVICE_CONN_NAME \ + --authorization-scheme ServicePrincipal \ + --azure-rm-service-principal-id $SERVICE_PRINCIPAL_ID \ + --azure-rm-service-prinicipal-key $SERVICE_PRINCIPAL_KEY \ + --azure-rm-tenant-id $TENANT_ID \ + --azure-rm-subscription-id $SUBSCRIPTION_ID \ + --azure-rm-subscription-name "$SUBSCRIPTION_NAME" \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME + +# navigate to the repo and add ssh following links below or just skip this step for https +open $AZURE_DEVOPS_ORG/$AZURE_DEVOPS_PROJECT_NAME/_git/$AZURE_REPOS_NAME +``` + +> Follow instructions at [Use SSH Key authentication](https://docs.microsoft.com/en-us/azure/devops/repos/git/use-ssh-keys-to-authenticate?view=azure-devops) to add ssh. +For more information on the different authentication types, please take a look at [Authtentication comparison](https://docs.microsoft.com/en-us/azure/devops/repos/git/auth-overview?view=azure-devops#authentication-comparison) + +![](https://docs.microsoft.com/en-us/azure/devops/repos/git/_img/ssh_add_public_key.gif?view=azure-devops) + +## Add new remote for the new Azure Repo +``` +# get the ssh url. For https just replace sshUrl with remoteUrl below +export NEW_REMOTE=$(az repos show -r $AZURE_REPOS_NAME --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query sshUrl -o tsv) + +# push master from cloned repo to the new remote +cd && \ +git remote add newremote $NEW_REMOTE +``` + +## Obtain Azure DevOps resources + +Extract details from devops, repos and projects + +```bash +export AZURE_DEVOPS_SERVICE_CONN_ID=$(az devops service-endpoint list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='${AZURE_PIPELINES_SERVICE_CONN_NAME}'].id" -o tsv) && \ +export AZURE_DEVOPS_REPOS_ID=$(az repos show --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --repository $AZURE_DEVOPS_REPOS_NAME --query id -o tsv) && \ +export AZURE_DEVOPS_PROJECT_ID=$(az devops project show --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query id -o tsv) +``` + +## Continuous Integration + +### Build pipelines pre-requisites + +``` +# export app paths +export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery && \ +export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package && \ +export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow && \ +export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion && \ +export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler + +# configure build YAML definitions +for pipelinePath in $DELIVERY_PATH $PACKAGE_PATH $WORKFLOW_PATH $INGESTION_PATH $DRONE_PATH; do +sed -i \ + -e "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" \ + -e "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" \ + -e "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" \ + -e "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" \ + -e "s#AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL#$AZURE_PIPELINES_SERVICE_CONN_NAME#g" \ + ${pipelinePath}/azure-pipelines.yml +done + +# push changes to the repo +cd $PROJECT_ROOT && \ +git add -u && \ +git commit -m "set build pipelines variables" && \ +git push newremote master && \ +cd - +``` + +### Create Delivery build definition + +``` +# add build definition +az pipelines create \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME \ + --name delivery-ci \ + --service-connection $AZURE_DEVOPS_SERVICE_CONN_ID \ + --yml-path src/shipping/delivery/azure-pipelines.yml \ + --repository-type tfsgit \ + --repository $AZURE_DEVOPS_REPOS_NAME \ + --branch master +``` + +## Deploy the Delivery service + +Extract resource details from deployment + +```bash +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ +export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ +export COLLECTION_NAME="${DATABASE_NAME}-col" && \ +export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) +``` + +Build the Delivery service + +```bash +export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/delivery:0.1.0 +``` + +Deploy the Delivery service: + +```bash +# Extract pod identity outputs from deployment +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) + +# Deploy the service +helm install $HELM_CHARTS/delivery/ \ + --set image.tag=0.1.0 \ + --set image.repository=delivery \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ + --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ + --set cosmosdb.id=$DATABASE_NAME \ + --set cosmosdb.collectionid=$COLLECTION_NAME \ + --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ + --set reason="Initial deployment" \ + --namespace backend \ + --name delivery-v0.1.0 + +# Verify the pod is created +helm status delivery-v0.1.0 +``` + +## Deploy the Package service + +Extract resource details from deployment + +```bash +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) +``` + +Build the Package service + +```bash +export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package + +# Build the docker image +docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/package:0.1.0 +``` + +Deploy the Package service + +```bash +# Create secret +# Note: Connection strings cannot be exported as outputs in ARM deployments +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') + +# Deploy service +helm install $HELM_CHARTS/package/ \ + --set image.tag=0.1.0 \ + --set image.repository=package \ + --set secrets.appinsights.ikey=$AI_IKEY \ + --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ + --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ + --set dockerregistry=$ACR_SERVER \ + --set reason="Initial deployment" \ + --namespace backend \ + --name package-v0.1.0 + +# Verify the pod is created +helm status package-v0.1.0 +``` + +## Deploy the Workflow service + +Extract resource details from deployment + +```bash +export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) +``` + +Build the workflow service + +```bash +export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow + +# Build the Docker image +docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. + +# Push the image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/workflow:0.1.0 +``` + +Create and set up pod identity + +```bash +# Extract outputs from deployment +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) +``` + +Deploy the Workflow service: + +```bash +# Deploy the service +helm install $HELM_CHARTS/workflow/ \ + --set image.tag=0.1.0 \ + --set image.repository=workflow \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ + --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ + --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ + --set keyvault.resourcegroup=$RESOURCE_GROUP \ + --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ + --set keyvault.tenantid=$TENANT_ID \ + --set reason="Initial deployment" \ + --namespace backend \ + --name workflow-v0.1.0 + +# Verify the pod is created +helm status workflow-v0.1.0 +``` + +## Deploy the Ingestion service + +Extract resource details from deployment + +```bash +export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ +export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) +export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) +``` + +Build the Ingestion service + +```bash +export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion + +# Build the docker image +docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH + +# Push the docker image to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/ingestion:0.1.0 +``` + +Deploy the Ingestion service + +```bash +# Deploy the ngnix ingress controller +helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true + +# Obtain the load balancer ip address and assign a domain name +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" +export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) +INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls + +# Create a self-signed certificate for TLS +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls.crt \ + -keyout ingestion-ingress-tls.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" + +# Deploy service +helm install $HELM_CHARTS/ingestion/ \ + --set image.tag=0.1.0 \ + --set image.repository=ingestion \ + --set dockerregistry=$ACR_SERVER \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=ingestion \ + --set ingress.hosts[0].tls=true \ + --set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ + --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ + --set secrets.appinsights.ikey=${AI_IKEY} \ + --set secrets.queue.keyname=IngestionServiceAccessKey \ + --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ + --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ + --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ + --set reason="Initial deployment" \ + --namespace backend \ + --name ingestion-v0.1.0 + +# Verify the pod is created +helm status ingestion-v0.1.0 +``` + +## Deploy DroneScheduler service + +Extract resource details from deployment + +```bash +export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +``` + +Build the dronescheduler services + +```bash +export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler +``` + +Create and set up pod identity + +```bash +# Extract outputs from deployment +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +``` + +Build and publish the container image + +```bash +# Build the Docker image +docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ + +# Push the images to ACR +az acr login --name $ACR_NAME +docker push $ACR_SERVER/dronescheduler:0.1.0 +``` + +Deploy the dronescheduler service: +```bash +# Deploy the service +helm install $HELM_CHARTS/dronescheduler/ \ + --set image.tag=0.1.0 \ + --set image.repository=dronescheduler \ + --set dockerregistry=$ACR_SERVER \ + --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ + --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ + --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ + --set reason="Initial deployment" \ + --namespace backend \ + --name dronescheduler-v0.1.0 + +# Verify the pod is created +helm status dronescheduler-v0.1.0 +``` + +## Validate the application is running + +You can send delivery requests to the ingestion service using the Swagger UI. + +Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. + +```bash +open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +``` + +> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. + + +## Optional steps + +### Load Test the application + +To run load testing against the solution, follow the steps listed [here](./src/shipping/ingress/readme.md). + +### Fluentd and Elastic Search + +Follow these steps to add logging and monitoring capabilities to the solution. + +Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch + +```bash +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \ +kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml +``` + +Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd + +```bash +# The example elasticsearch yaml files deploy a service named "elasticsearch" +wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml + +# Commenting out X-Pack credentials for demo purposes. +# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads +sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ +sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ +sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ +kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml +``` diff --git a/src/shipping/delivery/azure-pipelines-ci.yml b/src/shipping/delivery/azure-pipelines-ci.yml deleted file mode 100644 index 76b6c494..00000000 --- a/src/shipping/delivery/azure-pipelines-ci.yml +++ /dev/null @@ -1,147 +0,0 @@ -steps: -- script: echo '##vso[task.setvariable variable=repositoryName]delivery' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- script: echo '##vso[task.setvariable variable=chartPath]charts/delivery' - name: setvarChartPath -- script: echo $(chartPath) - name: echovarChartPath - -- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/delivery/Dockerfile' - name: setvarDockerFileName -- script: echo $(dockerFileName) - name: echovarDockerFileName - -- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' - name: setvarReleaseVersion -- script: echo $(releaseVersion) - name: echovarReleaseVersion - -- script: echo '##vso[task.setvariable variable=aksNamespace]backend' - name: setvarAKSNamespace -- script: echo $(aksNamespace) - name: echovarAKSNamespace - -- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' - name: setvarImageName -- script: echo $(imageName) - name: echovarImageName - -- task: Docker@1 - displayName: 'Build testrunner image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - arguments: '--pull --target testrunner' - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - imageName: '$(imageName)-test' - -- task: Docker@1 - displayName: 'Run tests' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'run' - - containerName: testrunner - - volumes: '$(System.DefaultWorkingDirectory)/TestResults:/app/tests/TestResults' - - imageName: '$(imageName)-test' - - runInBackground: false - -- task: PublishTestResults@2 - displayName: 'Publish test results' - inputs: - testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit - - testResultsFiles: 'TestResults/*.trx' - - searchFolder: '$(System.DefaultWorkingDirectory)' - - publishRunAttachments: true - -- task: Docker@1 - condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) - displayName: 'Build runtime image' - inputs: - - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - includeLatestTag: false - - imageName: '$(imageName)' - -- task: Docker@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push runtime image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'Push an image' - - imageName: '$(imageName)' - - includeSourceTags: false - -- task: HelmInstaller@0 - condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.9.1' - inputs: - kubectlVersion: 1.10.3 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm package' - inputs: - command: package - - chartPath: $(chartPath) - - chartVersion: $(Build.SourceBranchName) - - arguments: '--app-version $(Build.SourceBranchName)' - -- task: AzureCLI@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push a helm package' - inputs: - azureSubscription: $(AzureSubscription) - - scriptLocation: inlineScript - - inlineScript: | - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/delivery/azure-pipelines-validation.yml b/src/shipping/delivery/azure-pipelines-validation.yml deleted file mode 100644 index dac4fc24..00000000 --- a/src/shipping/delivery/azure-pipelines-validation.yml +++ /dev/null @@ -1,56 +0,0 @@ -variables: - BuildConfiguration: "Release" - -name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) - -pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy - branches: - include: - - - master - - - release/delivery/v* # for bug fixes - - paths: - include: - - - /src/shipping/delivery/ - -trigger: - batch: true - branches: - include: - - - master - - - feature/* - - - topic/* - - exclude: - - - feature/experimental/* - - - topic/experimental/* - - paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html - include: - - - /src/shipping/delivery/ - -resources: -- repo: self - -jobs: - -# CI -- job: deliverycivalidationjob - displayName: "Delivery CI Validation (build & test)" - pool: - vmImage: 'Ubuntu 16.04' - timeoutInMinutes: 90 - variables: - fullCI: False - buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] - steps: - - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml index f222a652..d0a9be12 100644 --- a/src/shipping/delivery/azure-pipelines.yml +++ b/src/shipping/delivery/azure-pipelines.yml @@ -1,22 +1,39 @@ variables: - BuildConfiguration: "Release" + repositoryName: delivery + chartPath: charts/delivery + dockerFileName: src/shipping/delivery/Dockerfile + imageName: $(repositoryName):$(Build.SourceBranchName) + azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL + azureContainerRegistry: ACR_SERVER_VAR_VAL + azureContainerRegistryName: ACR_NAME_VAR_VAL + azureKubernetesCluster: CLUSTER_NAME_VAR_VAL + resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/delivery/ + + branches: + include: + - master + - release/delivery/v* # for bug fixes + trigger: batch: true branches: include: - # for new release to production: release flow strategy - release/delivery/v* - - refs/relelase/delivery/v* - + - master + - feature/delivery/* + - topic/delivery/* paths: - include: - - - /src/shipping/delivery/ + include: + - /src/shipping/delivery/ resources: - repo: self @@ -31,7 +48,121 @@ jobs: timeoutInMinutes: 90 variables: fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/delivery/v') ] + buildImage: $[ eq(variables['build.sourceBranch'], 'refs/heads/master') ] steps: - - script: echo $(fullCI) - name: echovar - - template: ./azure-pipelines-ci.yml + - task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + + - task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/app/tests/TestResults' + + imageName: '$(imageName)-test' + + runInBackground: false + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/*.trx' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + + - task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + + - task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + + - task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(azureSubscription) + + azureResourceGroup: $(resourceGroup) + + kubernetesCluster: $(azureKubernetesCluster) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + + - task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(azureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(azureContainerRegistryName) --force; From c1e2b863c00b13ae752ccf14b77d6fbdee4c6e31 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 24 Apr 2019 18:04:14 -0300 Subject: [PATCH 145/246] [package] add CI instructions --- deploymentCICD.md | 15 +++ src/shipping/package/azure-pipelines-ci.yml | 108 ----------------- .../package/azure-pipelines-validation.yml | 53 --------- src/shipping/package/azure-pipelines.yml | 112 ++++++++++++++++-- 4 files changed, 117 insertions(+), 171 deletions(-) delete mode 100644 src/shipping/package/azure-pipelines-ci.yml delete mode 100644 src/shipping/package/azure-pipelines-validation.yml diff --git a/deploymentCICD.md b/deploymentCICD.md index 74ee16c4..52f03a5f 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -327,6 +327,21 @@ helm install $HELM_CHARTS/delivery/ \ helm status delivery-v0.1.0 ``` +## Add Package CI + +``` +# add build definitions +az pipelines create \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME \ + --name package-ci \ + --service-connection $AZURE_DEVOPS_SERVICE_CONN_ID \ + --yml-path src/shipping/package/azure-pipelines.yml \ + --repository-type tfsgit \ + --repository $AZURE_DEVOPS_REPOS_NAME \ + --branch master +``` + ## Deploy the Package service Extract resource details from deployment diff --git a/src/shipping/package/azure-pipelines-ci.yml b/src/shipping/package/azure-pipelines-ci.yml deleted file mode 100644 index 7c063986..00000000 --- a/src/shipping/package/azure-pipelines-ci.yml +++ /dev/null @@ -1,108 +0,0 @@ -steps: -- script: echo '##vso[task.setvariable variable=repositoryName]package' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- script: echo '##vso[task.setvariable variable=chartPath]charts/package' - name: setvarChartPath -- script: echo $(chartPath) - name: echovarChartPath - -- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/package/Dockerfile' - name: setvarDockerFileName -- script: echo $(dockerFileName) - name: echovarDockerFileName - -- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' - name: setvarReleaseVersion -- script: echo $(releaseVersion) - name: echovarReleaseVersion - -- script: echo '##vso[task.setvariable variable=aksNamespace]backend' - name: setvarAKSNamespace -- script: echo $(aksNamespace) - name: echovarAKSNamespace - -- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' - name: setvarImageName -- script: echo $(imageName) - name: echovarImageName - -- task: Docker@1 - condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) - displayName: 'Build runtime image' - inputs: - - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - includeLatestTag: true - - imageName: '$(imageName)' - -- task: Docker@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push runtime image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'Push an image' - - imageName: '$(imageName)' - - includeSourceTags: false - -- task: HelmInstaller@0 - condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.9.1' - inputs: - kubectlVersion: 1.10.3 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - namespace: prod - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm package' - inputs: - command: package - - chartPath: $(chartPath) - - chartVersion: $(Build.SourceBranchName) - - arguments: '--app-version $(Build.SourceBranchName)' - -- task: AzureCLI@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push a helm package' - inputs: - azureSubscription: $(AzureSubscription) - - scriptLocation: inlineScript - - inlineScript: | - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/package/azure-pipelines-validation.yml b/src/shipping/package/azure-pipelines-validation.yml deleted file mode 100644 index d511923a..00000000 --- a/src/shipping/package/azure-pipelines-validation.yml +++ /dev/null @@ -1,53 +0,0 @@ -variables: - BuildConfiguration: "Release" - -name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) - -pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy - branches: - include: - - - master - - release/package/v* # for bug fixes - - paths: - include: - - /src/shipping/package/ - -trigger: - batch: true - branches: - include: - - master - - - feature/* - - - topic/* - - exclude: - - - feature/experimental/* - - - topic/experimental/* - - paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html - include: - - - /src/shipping/package/ - -resources: -- repo: self - -jobs: - -# CI -- job: packagecivalidationjob - displayName: "Package CI Validation (build & test)" - pool: - vmImage: 'Ubuntu 16.04' - timeoutInMinutes: 90 - variables: - fullCI: False - buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] - steps: - - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/package/azure-pipelines.yml b/src/shipping/package/azure-pipelines.yml index 2e0a26bb..39558349 100644 --- a/src/shipping/package/azure-pipelines.yml +++ b/src/shipping/package/azure-pipelines.yml @@ -1,22 +1,39 @@ variables: - BuildConfiguration: "Release" + repositoryName: package + chartPath: charts/package + dockerFileName: src/shipping/package/Dockerfile + imageName: $(repositoryName):$(Build.SourceBranchName) + azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL + azureContainerRegistry: ACR_SERVER_VAR_VAL + azureContainerRegistryName: ACR_NAME_VAR_VAL + azureKubernetesCluster: CLUSTER_NAME_VAR_VAL + resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/package/ + + branches: + include: + - master + - release/package/v* # for bug fixes + trigger: batch: true branches: include: - # for new release to production: release flow strategy - release/package/v* - - refs/relelase/package/v* - + - master + - feature/package/* + - topic/package/* paths: - include: - - - /src/shipping/package/ + include: + - /src/shipping/package/ resources: - repo: self @@ -31,7 +48,82 @@ jobs: timeoutInMinutes: 90 variables: fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/package/v') ] + buildImage: $[ eq(variables['build.sourceBranch'], 'refs/heads/master') ] steps: - - script: echo $(fullCI) - name: echovar - - template: ./azure-pipelines-ci.yml + - task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: true + + imageName: '$(imageName)' + + - task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + + - task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(azureSubscription) + + azureresourceGroup: $(resourceGroup) + + kubernetesCluster: $(azureKubernetesCluster) + + namespace: prod + + command: init + + upgradeTiller: false + + arguments: '--client-only' + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + + - task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(azureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(azureContainerRegistryName) --force; From 0ec03cfd95c083afda3a7baa32c30487acef89ac Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 24 Apr 2019 18:38:42 -0300 Subject: [PATCH 146/246] [workflow] add CI instructions --- deploymentCICD.md | 15 ++ src/shipping/workflow/azure-pipelines-ci.yml | 147 ----------------- .../workflow/azure-pipelines-validation.yml | 56 ------- src/shipping/workflow/azure-pipelines.yml | 151 ++++++++++++++++-- 4 files changed, 156 insertions(+), 213 deletions(-) delete mode 100644 src/shipping/workflow/azure-pipelines-ci.yml delete mode 100644 src/shipping/workflow/azure-pipelines-validation.yml diff --git a/deploymentCICD.md b/deploymentCICD.md index 52f03a5f..37f2d56e 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -386,6 +386,21 @@ helm install $HELM_CHARTS/package/ \ helm status package-v0.1.0 ``` +## Add Workflow CI + +``` +# add build definitions +az pipelines create \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME \ + --name workflow-ci \ + --service-connection $AZURE_DEVOPS_SERVICE_CONN_ID \ + --yml-path src/shipping/workflow/azure-pipelines.yml \ + --repository-type tfsgit \ + --repository $AZURE_DEVOPS_REPOS_NAME \ + --branch master +``` + ## Deploy the Workflow service Extract resource details from deployment diff --git a/src/shipping/workflow/azure-pipelines-ci.yml b/src/shipping/workflow/azure-pipelines-ci.yml deleted file mode 100644 index d1ffca8b..00000000 --- a/src/shipping/workflow/azure-pipelines-ci.yml +++ /dev/null @@ -1,147 +0,0 @@ -steps: -- script: echo '##vso[task.setvariable variable=repositoryName]workflow' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- script: echo '##vso[task.setvariable variable=chartPath]charts/workflow' - name: setvarChartPath -- script: echo $(chartPath) - name: echovarChartPath - -- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/workflow/Dockerfile' - name: setvarDockerFileName -- script: echo $(dockerFileName) - name: echovarDockerFileName - -- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' - name: setvarReleaseVersion -- script: echo $(releaseVersion) - name: echovarReleaseVersion - -- script: echo '##vso[task.setvariable variable=aksNamespace]backend' - name: setvarAKSNamespace -- script: echo $(aksNamespace) - name: echovarAKSNamespace - -- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' - name: setvarImageName -- script: echo $(imageName) - name: echovarImageName - -- task: Docker@1 - displayName: 'Build testrunner image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - arguments: '--pull --target testrunner' - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - imageName: '$(imageName)-test' - -- task: Docker@1 - displayName: 'Run tests' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'run' - - containerName: testrunner - - volumes: '$(System.DefaultWorkingDirectory)/TestResults:/src/tests/TestResults' - - imageName: '$(imageName)-test' - - runInBackground: false - -- task: PublishTestResults@2 - displayName: 'Publish test results' - inputs: - testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit - - testResultsFiles: 'TestResults/*.trx' - - searchFolder: '$(System.DefaultWorkingDirectory)' - - publishRunAttachments: true - -- task: Docker@1 - condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) - displayName: 'Build runtime image' - inputs: - - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - includeLatestTag: false - - imageName: '$(imageName)' - -- task: Docker@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push runtime image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'Push an image' - - imageName: '$(imageName)' - - includeSourceTags: false - -- task: HelmInstaller@0 - condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.9.1' - inputs: - kubectlVersion: 1.10.3 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm package' - inputs: - command: package - - chartPath: $(chartPath) - - chartVersion: $(Build.SourceBranchName) - - arguments: '--app-version $(Build.SourceBranchName)' - -- task: AzureCLI@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push a helm package' - inputs: - azureSubscription: $(AzureSubscription) - - scriptLocation: inlineScript - - inlineScript: | - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/workflow/azure-pipelines-validation.yml b/src/shipping/workflow/azure-pipelines-validation.yml deleted file mode 100644 index 21854f43..00000000 --- a/src/shipping/workflow/azure-pipelines-validation.yml +++ /dev/null @@ -1,56 +0,0 @@ -variables: - BuildConfiguration: "Release" - -name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) - -pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy - branches: - include: - - - master - - - release/workflow/v* # for bug fixes - - paths: - include: - - - /src/shipping/workflow/ - -trigger: - batch: true - branches: - include: - - - master - - - feature/* - - - topic/* - - exclude: - - - feature/experimental/* - - - topic/experimental/* - - paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html - include: - - - /src/shipping/workflow/ - -resources: -- repo: self - -jobs: - -# CI -- job: workflowcivalidationjob - displayName: "Workflow CI Validation (build & test)" - pool: - vmImage: 'Ubuntu 16.04' - timeoutInMinutes: 90 - variables: - fullCI: False - buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] - steps: - - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/workflow/azure-pipelines.yml b/src/shipping/workflow/azure-pipelines.yml index bf3e647a..2a9d77c9 100644 --- a/src/shipping/workflow/azure-pipelines.yml +++ b/src/shipping/workflow/azure-pipelines.yml @@ -1,22 +1,39 @@ variables: - BuildConfiguration: "Release" + repositoryName: workflow + chartPath: charts/workflow + dockerFileName: src/shipping/workflow/Dockerfile + imageName: $(repositoryName):$(Build.SourceBranchName) + azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL + azureContainerRegistry: ACR_SERVER_VAR_VAL + azureContainerRegistryName: ACR_NAME_VAR_VAL + azureKubernetesCluster: CLUSTER_NAME_VAR_VAL + resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/workflow/ + + branches: + include: + - master + - release/workflow/v* # for bug fixes + trigger: batch: true branches: include: - # for new release to production: release flow strategy - release/workflow/v* - - refs/relelase/workflow/v* - + - master + - feature/workflow/* + - topic/workflow/* paths: - include: - - - /src/shipping/workflow/ + include: + - /src/shipping/workflow/ resources: - repo: self @@ -31,7 +48,121 @@ jobs: timeoutInMinutes: 90 variables: fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/workflow/v') ] + buildImage: $[ eq(variables['build.sourceBranch'], 'refs/heads/master') ] steps: - - script: echo $(fullCI) - name: echovar - - template: ./azure-pipelines-ci.yml + - task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + + - task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/src/tests/TestResults' + + imageName: '$(imageName)-test' + + runInBackground: false + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'VSTest' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/*.trx' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + + - task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + + - task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + + - task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(azureSubscription) + + azureresourceGroup: $(resourceGroup) + + kubernetesCluster: $(azureKubernetesCluster) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + + - task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(azureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(azureContainerRegistryName) --force; From ec293750d84193f556c48cc5a7e813dec28fd9dd Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 24 Apr 2019 18:54:10 -0300 Subject: [PATCH 147/246] [ingestion] add CI instructions --- deploymentCICD.md | 14 ++ src/shipping/ingestion/azure-pipelines-ci.yml | 147 ----------------- .../ingestion/azure-pipelines-validation.yml | 49 ------ src/shipping/ingestion/azure-pipelines.yml | 149 +++++++++++++++++- 4 files changed, 157 insertions(+), 202 deletions(-) delete mode 100644 src/shipping/ingestion/azure-pipelines-ci.yml delete mode 100644 src/shipping/ingestion/azure-pipelines-validation.yml diff --git a/deploymentCICD.md b/deploymentCICD.md index 37f2d56e..6ae17f72 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -451,6 +451,20 @@ helm install $HELM_CHARTS/workflow/ \ # Verify the pod is created helm status workflow-v0.1.0 ``` +## Add Ingestion CI + +``` +# add build definitions +az pipelines create \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME \ + --name ingestion-ci \ + --service-connection $AZURE_DEVOPS_SERVICE_CONN_ID \ + --yml-path src/shipping/ingestion/azure-pipelines.yml \ + --repository-type tfsgit \ + --repository $AZURE_DEVOPS_REPOS_NAME \ + --branch master +``` ## Deploy the Ingestion service diff --git a/src/shipping/ingestion/azure-pipelines-ci.yml b/src/shipping/ingestion/azure-pipelines-ci.yml deleted file mode 100644 index a8e98c62..00000000 --- a/src/shipping/ingestion/azure-pipelines-ci.yml +++ /dev/null @@ -1,147 +0,0 @@ -steps: -- script: echo '##vso[task.setvariable variable=repositoryName]ingestion' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- script: echo '##vso[task.setvariable variable=chartPath]charts/ingestion' - name: setvarChartPath -- script: echo $(chartPath) - name: echovarChartPath - -- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/ingestion/Dockerfile' - name: setvarDockerFileName -- script: echo $(dockerFileName) - name: echovarDockerFileName - -- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' - name: setvarReleaseVersion -- script: echo $(releaseVersion) - name: echovarReleaseVersion - -- script: echo '##vso[task.setvariable variable=aksNamespace]backend' - name: setvarAKSNamespace -- script: echo $(aksNamespace) - name: echovarAKSNamespace - -- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' - name: setvarImageName -- script: echo $(imageName) - name: echovarImageName - -- task: Docker@1 - displayName: 'Build testrunner image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - arguments: '--pull --target testrunner' - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - imageName: '$(imageName)-test' - -- task: Docker@1 - displayName: 'Run tests' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'run' - - containerName: testrunner - - volumes: '$(System.DefaultWorkingDirectory)/TestResults:/usr/src/app/target/surefire-reports/' - - imageName: '$(imageName)-test' - - runInBackground: false - -- task: PublishTestResults@2 - displayName: 'Publish test results' - inputs: - testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit - - testResultsFiles: 'TestResults/TEST*.xml' - - searchFolder: '$(System.DefaultWorkingDirectory)' - - publishRunAttachments: true - -- task: Docker@1 - condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) - displayName: 'Build runtime image' - inputs: - - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - includeLatestTag: false - - imageName: '$(imageName)' - -- task: Docker@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push runtime image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'Push an image' - - imageName: '$(imageName)' - - includeSourceTags: false - -- task: HelmInstaller@0 - condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' - inputs: - kubectlVersion: 1.12.4 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm package' - inputs: - command: package - - chartPath: $(chartPath) - - chartVersion: $(Build.SourceBranchName) - - arguments: '--app-version $(Build.SourceBranchName)' - -- task: AzureCLI@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push a helm package' - inputs: - azureSubscription: $(AzureSubscription) - - scriptLocation: inlineScript - - inlineScript: | - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/ingestion/azure-pipelines-validation.yml b/src/shipping/ingestion/azure-pipelines-validation.yml deleted file mode 100644 index c3627972..00000000 --- a/src/shipping/ingestion/azure-pipelines-validation.yml +++ /dev/null @@ -1,49 +0,0 @@ -variables: - BuildConfiguration: "Release" - -name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) - -pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy - paths: - include: - - - /src/shipping/ingestion/ - branches: - include: - - - master - - - release/ingestion/v* - - -trigger: - batch: true - paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html - include: - - - /src/shipping/ingestion/ - branches: - include: - - - master - - - feature/* - - - topic/* - -resources: -- repo: self - -jobs: - -# CI -- job: ingestioncivalidationjob - displayName: "ingestion CI Validation (build & test)" - pool: - vmImage: 'Ubuntu 16.04' - timeoutInMinutes: 90 - variables: - fullCI: False - buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/heads/master')) ] - steps: - - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/ingestion/azure-pipelines.yml b/src/shipping/ingestion/azure-pipelines.yml index b5a510d6..f7010ae4 100644 --- a/src/shipping/ingestion/azure-pipelines.yml +++ b/src/shipping/ingestion/azure-pipelines.yml @@ -1,16 +1,39 @@ variables: - BuildConfiguration: "Release" + repositoryName: ingestion + chartPath: charts/ingestion + dockerFileName: src/shipping/ingestion/Dockerfile + imageName: $(repositoryName):$(Build.SourceBranchName) + azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL + azureContainerRegistry: ACR_SERVER_VAR_VAL + azureContainerRegistryName: ACR_NAME_VAR_VAL + azureKubernetesCluster: CLUSTER_NAME_VAR_VAL + resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) -trigger: +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/ingestion/ + branches: include: + - master + - release/ingestion/v* # for bug fixes +trigger: + batch: true + branches: + include: # for new release to production: release flow strategy - release/ingestion/v* - - refs/relelase/ingestion/v* + - master + - feature/ingestion/* + - topic/ingestion/* + paths: + include: + - /src/shipping/ingestion/ resources: - repo: self @@ -25,7 +48,121 @@ jobs: timeoutInMinutes: 90 variables: fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/ingestion/v') ] + buildImage: $[ eq(variables['build.sourceBranch'], 'refs/heads/master') ] steps: - - script: echo $(fullCI) - name: echovar - - template: ./azure-pipelines-ci.yml + - task: Docker@1 + displayName: 'Build testrunner image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + arguments: '--pull --target testrunner' + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + imageName: '$(imageName)-test' + + - task: Docker@1 + displayName: 'Run tests' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'run' + + containerName: testrunner + + volumes: '$(System.DefaultWorkingDirectory)/TestResults:/usr/src/app/target/surefire-reports/' + + imageName: '$(imageName)-test' + + runInBackground: false + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit + + testResultsFiles: 'TestResults/TEST*.xml' + + searchFolder: '$(System.DefaultWorkingDirectory)' + + publishRunAttachments: true + + - task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + + - task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + + - task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(azureSubscription) + + azureresourceGroup: $(resourceGroup) + + kubernetesCluster: $(azureKubernetesCluster) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + + - task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(azureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(azureContainerRegistryName) --force; From 7d50e7d6f475ada0ddb8072081f06a13d9d42aca Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 24 Apr 2019 18:55:40 -0300 Subject: [PATCH 148/246] [drone] add CI instructions --- deploymentCICD.md | 15 +++ .../dronescheduler/azure-pipelines-ci.yml | 115 ----------------- .../azure-pipelines-validation.yml | 57 --------- .../dronescheduler/azure-pipelines.yml | 117 ++++++++++++++++-- 4 files changed, 120 insertions(+), 184 deletions(-) delete mode 100644 src/shipping/dronescheduler/azure-pipelines-ci.yml delete mode 100644 src/shipping/dronescheduler/azure-pipelines-validation.yml diff --git a/deploymentCICD.md b/deploymentCICD.md index 6ae17f72..c1bc93e7 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -534,6 +534,21 @@ helm install $HELM_CHARTS/ingestion/ \ helm status ingestion-v0.1.0 ``` +## Add DroneScheduler CI + +``` +# add build definitions +az pipelines create \ + --organization $AZURE_DEVOPS_ORG \ + --project $AZURE_DEVOPS_PROJECT_NAME \ + --name dronescheduler-ci \ + --service-connection $AZURE_DEVOPS_SERVICE_CONN_ID \ + --yml-path src/shipping/dronescheduler/azure-pipelines.yml \ + --repository-type tfsgit \ + --repository $AZURE_DEVOPS_REPOS_NAME \ + --branch master +``` + ## Deploy DroneScheduler service Extract resource details from deployment diff --git a/src/shipping/dronescheduler/azure-pipelines-ci.yml b/src/shipping/dronescheduler/azure-pipelines-ci.yml deleted file mode 100644 index 3b0e8d57..00000000 --- a/src/shipping/dronescheduler/azure-pipelines-ci.yml +++ /dev/null @@ -1,115 +0,0 @@ -steps: -- script: echo '##vso[task.setvariable variable=repositoryName]dronescheduler' - name: setvarAzureContainerRegistry -- script: echo $(repositoryName) - name: echovarRepositoryName - -- script: echo '##vso[task.setvariable variable=chartPath]charts/dronescheduler' - name: setvarChartPath -- script: echo $(chartPath) - name: echovarChartPath - -- script: echo '##vso[task.setvariable variable=buildContext]src/shipping' - name: setvarBuildContext -- script: echo $(buildContext) - name: echovarBuildContext - -- script: echo '##vso[task.setvariable variable=dockerFileName]src/shipping/dronescheduler/Dockerfile' - name: setvarDockerFileName -- script: echo $(dockerFileName) - name: echovarDockerFileName - -- script: echo '##vso[task.setvariable variable=releaseVersion]$(Build.SourceBranchName)' - name: setvarReleaseVersion -- script: echo $(releaseVersion) - name: echovarReleaseVersion - -- script: echo '##vso[task.setvariable variable=aksNamespace]backend' - name: setvarAKSNamespace -- script: echo $(aksNamespace) - name: echovarAKSNamespace - -- script: echo '##vso[task.setvariable variable=imageName]$(repositoryName):$(releaseVersion)' - name: setvarImageName -- script: echo $(imageName) - name: echovarImageName - -- task: Docker@1 - condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) - displayName: 'Build runtime image' - inputs: - - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) - - includeLatestTag: false - - imageName: '$(imageName)' - - buildContext: $(buildContext) - - useDefaultContext: false - -- task: Docker@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push runtime image' - inputs: - azureSubscriptionEndpoint: $(AzureSubscription) - - azureContainerRegistry: $(AzureContainerRegistry) - - command: 'Push an image' - - imageName: '$(imageName)' - - includeSourceTags: false - -- task: HelmInstaller@0 - condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.9.1' - inputs: - kubectlVersion: 1.10.3 - - checkLatestKubectl: false - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(AzureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(AzureKubernetesService) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - -- task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm package' - inputs: - command: package - - chartPath: $(chartPath) - - chartVersion: $(Build.SourceBranchName) - - arguments: '--app-version $(Build.SourceBranchName)' - -- task: AzureCLI@1 - condition: eq(variables['fullCI'],True) - displayName: 'Push a helm package' - inputs: - azureSubscription: $(AzureSubscription) - - scriptLocation: inlineScript - - inlineScript: | - az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(AzureContainerRegistryHelmRepo) --force; diff --git a/src/shipping/dronescheduler/azure-pipelines-validation.yml b/src/shipping/dronescheduler/azure-pipelines-validation.yml deleted file mode 100644 index c018c02f..00000000 --- a/src/shipping/dronescheduler/azure-pipelines-validation.yml +++ /dev/null @@ -1,57 +0,0 @@ -variables: - BuildConfiguration: "Release" - -name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) - -pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy - paths: - include: - - /src/shipping/dronescheduler/ - exclude: - - / - - /src/ - - branches: - include: - - - master - - - release/dronescheduler/v* # for bug fixes - -trigger: - batch: true - paths: # first commit into a new branch is not filter by path: https://developercommunity.visualstudio.com/content/problem/246323/push-of-new-branch-disregards-path-filters-in-ci-b.html - include: - - /src/shipping/dronescheduler/ - - branches: - include: - - - master - - - feature/* - - - topic/* - - exclude: - - - feature/experimental/* - - - topic/experimental/* - -resources: -- repo: self - -jobs: - -# CI -- job: droneschedulercivalidationjob - displayName: "dronescheduler CI Validation (build & test)" - pool: - vmImage: 'Ubuntu 16.04' - timeoutInMinutes: 90 - variables: - fullCI: False - buildImage: $[ or(startsWith(variables['build.sourceBranch'], 'refs/pull/'),eq(variables['build.sourceBranch'], 'refs/head/master')) ] - steps: - - template: ./azure-pipelines-ci.yml diff --git a/src/shipping/dronescheduler/azure-pipelines.yml b/src/shipping/dronescheduler/azure-pipelines.yml index 2e1e59d2..17260999 100644 --- a/src/shipping/dronescheduler/azure-pipelines.yml +++ b/src/shipping/dronescheduler/azure-pipelines.yml @@ -1,24 +1,40 @@ variables: - BuildConfiguration: "Release" + repositoryName: dronescheduler + chartPath: charts/dronescheduler + dockerFileName: src/shipping/dronescheduler/Dockerfile + buildContext: src/shipping + imageName: $(repositoryName):$(Build.SourceBranchName) + azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL + azureContainerRegistry: ACR_SERVER_VAR_VAL + azureContainerRegistryName: ACR_NAME_VAR_VAL + azureKubernetesCluster: CLUSTER_NAME_VAR_VAL + resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) +pr: # only valid for GitHub. Using Azure repo it must be configure as a Branch Policy + paths: + include: + - /src/shipping/dronescheduler/ + + branches: + include: + - master + - release/dronescheduler/v* # for bug fixes + trigger: batch: true branches: include: - # for new release to production: release flow strategy - release/dronescheduler/v* - - refs/relelase/dronescheduler/v* - + - master + - feature/dronescheduler/* + - topic/dronescheduler/* paths: - include: - - /src/shipping/dronescheduler/ - exclude: - - / - - /src/ + include: + - /src/shipping/dronescheduler/ resources: - repo: self @@ -33,7 +49,84 @@ jobs: timeoutInMinutes: 90 variables: fullCI: $[ startsWith(variables['build.sourceBranch'], 'refs/heads/release/dronescheduler/v') ] + buildImage: $[ eq(variables['build.sourceBranch'], 'refs/heads/master') ] steps: - - script: echo $(fullCI) - name: echovar - - template: ./azure-pipelines-ci.yml + - task: Docker@1 + condition: or(eq(variables['buildImage'],True),eq(variables['fullCI'],True)) + displayName: 'Build runtime image' + inputs: + + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + dockerFile: $(System.DefaultWorkingDirectory)/$(dockerFileName) + + includeLatestTag: false + + imageName: '$(imageName)' + + buildContext: $(buildContext) + + useDefaultContext: false + + - task: Docker@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push runtime image' + inputs: + azureSubscriptionEndpoint: $(azureSubscription) + + azureContainerRegistry: $(azureContainerRegistry) + + command: 'Push an image' + + imageName: '$(imageName)' + + includeSourceTags: false + + - task: HelmInstaller@0 + condition: eq(variables['fullCI'],True) + displayName: 'Install Helm 2.12.3' + inputs: + kubectlVersion: 1.12.4 + + checkLatestKubectl: false + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm init --client-only' + inputs: + azureSubscription: $(azureSubscription) + + azureResourceGroup: $(ResourceGroup) + + kubernetesCluster: $(azureKubernetesCluster) + + command: init + + upgradeTiller: false + + arguments: '--client-only' + + - task: HelmDeploy@0 + condition: eq(variables['fullCI'],True) + displayName: 'helm package' + inputs: + command: package + + chartPath: $(chartPath) + + chartVersion: $(Build.SourceBranchName) + + arguments: '--app-version $(Build.SourceBranchName)' + + - task: AzureCLI@1 + condition: eq(variables['fullCI'],True) + displayName: 'Push a helm package' + inputs: + azureSubscription: $(azureSubscription) + + scriptLocation: inlineScript + + inlineScript: | + az acr helm push $(System.ArtifactsDirectory)/$(repositoryName)-$(Build.SourceBranchName).tgz --name $(azureContainerRegistryName) --force; From 8a007ab8eeb41217e8e8ceea44d47bd5765161f6 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 17 Apr 2019 15:22:40 -0300 Subject: [PATCH 149/246] [delivery] add CD instructions --- deploymentCICD.md | 88 +- src/shipping/delivery/azure-pipelines-cd.json | 1959 +++++++++++++++++ 2 files changed, 2004 insertions(+), 43 deletions(-) create mode 100644 src/shipping/delivery/azure-pipelines-cd.json diff --git a/deploymentCICD.md b/deploymentCICD.md index c1bc93e7..9ee1504b 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -5,7 +5,6 @@ - Azure subscription - [Azure CLI 2.0.49 or later](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Azure DevOps account](https://azure.microsoft.com/services/devops) -- [Docker](https://docs.docker.com/) - [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) - [JQ](https://stedolan.github.io/jq/download/) @@ -168,6 +167,7 @@ az extension add --name azure-devops # export AZURE_DEVOPS_ORG_NAME= AZURE_DEVOPS_ORG=https://dev.azure.com/$AZURE_DEVOPS_ORG_NAME +AZURE_DEVOPS_VSRM_ORG=https://vsrm.dev.azure.com/$AZURE_DEVOPS_ORG_NAME AZURE_DEVOPS_PROJECT_NAME= AZURE_REPOS_NAME= AZURE_PIPELINES_SERVICE_CONN_NAME=default_cicd_service-connection @@ -223,13 +223,20 @@ git remote add newremote $NEW_REMOTE Extract details from devops, repos and projects ```bash +# navigate to the organization tokens and create a new Personal Access Token +open $AZURE_DEVOPS_ORG/_usersSettings/tokens + +# export token for making REST API calls +export AZURE_DEVEOPS_USER= +export AZURE_DEVOPS_PAT= +export AZURE_DEVOPS_AUTHN_BASIC_TOKEN=$(echo -n ${AZURE_DEVOPS_USER}:${AZURE_DEVOPS_PAT} | base64 | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n//g') + export AZURE_DEVOPS_SERVICE_CONN_ID=$(az devops service-endpoint list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='${AZURE_PIPELINES_SERVICE_CONN_NAME}'].id" -o tsv) && \ export AZURE_DEVOPS_REPOS_ID=$(az repos show --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --repository $AZURE_DEVOPS_REPOS_NAME --query id -o tsv) && \ -export AZURE_DEVOPS_PROJECT_ID=$(az devops project show --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query id -o tsv) +export AZURE_DEVOPS_PROJECT_ID=$(az devops project show --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query id -o tsv) && \ +export AZURE_DEVOPS_USER_ID=$(az devops user show --user ${AZURE_DEVEOPS_USER} --organization ${AZURE_DEVOPS_ORG} --query id -o tsv) ``` -## Continuous Integration - ### Build pipelines pre-requisites ``` @@ -259,7 +266,7 @@ git push newremote master && \ cd - ``` -### Create Delivery build definition +## Add Delivery CI/CD ``` # add build definition @@ -272,58 +279,53 @@ az pipelines create \ --repository-type tfsgit \ --repository $AZURE_DEVOPS_REPOS_NAME \ --branch master -``` - -## Deploy the Delivery service - -Extract resource details from deployment -```bash +# query build definition details and resources +export AZURE_DEVOPS_DELIVERY_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='delivery-ci'].id" -o tsv) && \ +export AZURE_DEVOPS_DELIVERY_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='delivery-ci'].queue.id" -o tsv) && \ export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ -export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) -``` +export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) -Build the Delivery service +# add relese definition +cat $DELIVERY_PATH/azure-pipelines-cd.json | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ + sed "s#COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ + sed "s#DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ + sed "s#AZURE_DEVOPS_DELIVERY_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_DELIVERY_BUILD_ID#g" | \ + sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ + sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ + sed "s#AZURE_DEVOPS_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_DELIVERY_QUEUE_ID#g" | \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + > $DELIVERY_PATH/azure-pipelines-cd-0.json -```bash -export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery +curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ + -d@${DELIVERY_PATH}/azure-pipelines-cd-0.json \ + -H "Authorization: Basic ${AZURE_DEVOPS_AUTHN_BASIC_TOKEN}" \ + -H "Content-Type: application/json" \ + -o /dev/null ``` -Build and publish the container image +Kick off CI/CD pipelines ```bash -# Build the Docker image -docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/. - -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/delivery:0.1.0 +git checkout -b release/delivery/v0.1.0 && \ +git push newremote release/delivery/v0.1.0 ``` -Deploy the Delivery service: +Verify delivery was deployed ```bash -# Extract pod identity outputs from deployment -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) - -# Deploy the service -helm install $HELM_CHARTS/delivery/ \ - --set image.tag=0.1.0 \ - --set image.repository=delivery \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ - --set cosmosdb.id=$DATABASE_NAME \ - --set cosmosdb.collectionid=$COLLECTION_NAME \ - --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ - --set reason="Initial deployment" \ - --namespace backend \ - --name delivery-v0.1.0 - -# Verify the pod is created helm status delivery-v0.1.0 ``` diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json new file mode 100644 index 00000000..26c5745b --- /dev/null +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -0,0 +1,1959 @@ +{ + "source": 1, + "revision": 13, + "description": null, + "isDeleted": false, + "variables": { + "REPO_NAME": { + "value": "delivery" + }, + "REASON": { + "value": "Azure DevOps CD Pipeline", + "allowOverride": true + }, + "ACR_SERVER": { + "value": "ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "ACR_NAME_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_CLIENT_ID": { + "value": "DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_RESOURCE_ID": { + "value": "DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "DATABASE_NAME": { + "value": "DATABASE_NAME_VAR_VAL" + }, + "COLLECTION_NAME": { + "value": "COLLECTION_NAME_VAR_VAL" + }, + "DELIVERY_KEYVAULT_URI": { + "value": "DELIVERY_KEYVAULT_URI_VAR_VAL" + } + }, + "variableGroups": [], + "environments": [ + { + "id": 2, + "name": "dev", + "rank": 1, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 4 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 11 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 12 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-delivery", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-dev", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "ReleaseStarted", + "conditionType": 1, + "value": "" + }, + { + "name": "ci-delivery", + "conditionType": 4, + "value": "{\"sourceBranch\":\"release/delivery/v*\",\"tags\":[],\"useBuildDefinitionBranch\":false,\"createReleaseOnBuildTagging\":false}" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "currentRelease": null, + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 3, + "name": "QA", + "rank": 2, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 5 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 10 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 13 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-delivery", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-qa", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "currentRelease": null, + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 4, + "name": "staging", + "rank": 3, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 7 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 8 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 15 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-delivery", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-staging", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 5, + "name": "production", + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "rank": 4, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": false, + "isNotificationOn": false, + "approver": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "id": 14 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": true, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 16 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 17 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-delivery", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Pull a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "pull", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(Build.Repository.Name):$(Build.BuildId)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Tag a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Tag image", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "false", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Push a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Push an image", + "dockerFile": "**/Dockerfile", + "arguments": "", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service delivery-experimental app.kubernetes.io/name=delivery,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 2, + "phaseType": 2, + "name": "Agentless job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "bcb64569-d51a-4af0-9c01-ea5d05b3b622", + "version": "8.*", + "name": "Swap (blue-green)", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 3600, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "instructions": "consider running some canary or just resume for swapping blue and green versions", + "emailRecipients": "", + "onTimeout": "reject" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-delivery", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 3, + "phaseType": 1, + "name": "Agent job (swap)", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl get", + "refName": "BlueVersion", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "get", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "-n backend svc/delivery -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector -n backend svc/delivery-experimental app.kubernetes.io/name=delivery,app.kubernetes.io/instance=$(BlueVersion.KubectlOutput)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set blue", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service delivery app.kubernetes.io/name=delivery,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "QA", + "conditionType": 2, + "value": "4" + }, + { + "name": "staging", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + } + ], + "artifacts": [ + { + "sourceId": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL:AZURE_DEVOPS_DELIVERY_BUILD_ID_VAR_VAL", + "type": "Build", + "alias": "ci-delivery", + "definitionReference": { + "artifactSourceDefinitionUrl": { + "id": "", + "name": "" + }, + "defaultVersionBranch": { + "id": "", + "name": "" + }, + "defaultVersionSpecific": { + "id": "", + "name": "" + }, + "defaultVersionTags": { + "id": "", + "name": "" + }, + "defaultVersionType": { + "id": "selectDuringReleaseCreationType", + "name": "Specify at the time of release creation" + }, + "definition": { + "id": "AZURE_DEVOPS_DELIVERY_BUILD_ID_VAR_VAL", + "name": "aks-ri-ci-delivery" + }, + "definitions": { + "id": "", + "name": "" + }, + "IsMultiDefinitionType": { + "id": "False", + "name": "False" + }, + "project": { + "id": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL", + "name": "roadmap" + }, + "repository": { + "id": "AZURE_DEVOPS_REPOS_ID_VAR_VALL", + "name": "roadmap" + } + }, + "isPrimary": true, + "isRetained": false + } + ], + "triggers": [ + { + "artifactAlias": "ci-delivery", + "triggerConditions": [ + { + "sourceBranch": "release/$(REPO_NAME)/v*", + "tags": [], + "useBuildDefinitionBranch": false, + "createReleaseOnBuildTagging": false + } + ], + "triggerType": 1 + } + ], + "releaseNameFormat": "release-$(rev:r)", + "tags": [], + "pipelineProcess": { + "type": 1 + }, + "properties": { + "DefinitionCreationSource": { + "$type": "System.String", + "$value": "Other" + } + }, + "id": 2, + "name": "delivery-cd", + "path": null, + "projectReference": null, + "url": null +} From c55e0883ab9466a8473f98f376987e8eedbef00f Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 18 Apr 2019 16:51:51 -0300 Subject: [PATCH 150/246] bug fix: missing var to export the package collection name solved: #127 https://github.com/mspnp/microservices-reference-implementation/issues/127 --- deployment.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index b859d810..5a325a1d 100644 --- a/deployment.md +++ b/deployment.md @@ -236,7 +236,8 @@ Deploy the Package service ```bash # Create secret # Note: Connection strings cannot be exported as outputs in ARM deployments -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ +export COSMOSDB_COL_NAME=packages # Deploy service helm install $HELM_CHARTS/package/ \ From f47ead91559c604bb484ecd4f8b66f75f7dd6dc8 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 18 Apr 2019 16:57:14 -0300 Subject: [PATCH 151/246] [package] add CD pipeline --- deploymentCICD.md | 66 +- src/shipping/package/azure-pipelines-cd.json | 1954 ++++++++++++++++++ 2 files changed, 1987 insertions(+), 33 deletions(-) create mode 100644 src/shipping/package/azure-pipelines-cd.json diff --git a/deploymentCICD.md b/deploymentCICD.md index 9ee1504b..0a9e2f16 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -329,8 +329,9 @@ Verify delivery was deployed helm status delivery-v0.1.0 ``` -## Add Package CI +## Add Package CI/CD +Create build and release pipeline definitions ``` # add build definitions az pipelines create \ @@ -342,49 +343,48 @@ az pipelines create \ --repository-type tfsgit \ --repository $AZURE_DEVOPS_REPOS_NAME \ --branch master -``` -## Deploy the Package service +# query build definition details and resources +export AZURE_DEVOPS_PACKAGE_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='package-ci'].id" -o tsv) && \ +export AZURE_DEVOPS_PACKAGE_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='package-ci'].queue.id" -o tsv) && \ +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) && \ +export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ +export COSMOSDB_COL_NAME=packages -Extract resource details from deployment +# add relese definition +cat $PACKAGE_PATH/azure-pipelines-cd.json | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ + sed "s#COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ + sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ + sed "s#AZURE_DEVOPS_PACKAGE_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_PACKAGE_BUILD_ID#g" | \ + sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ + sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ + sed "s#AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_PACKAGE_QUEUE_ID#g" | \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + > $PACKAGE_PATH/azure-pipelines-cd-0.json -```bash -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) +curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ + -d@${PACKAGE_PATH}/azure-pipelines-cd-0.json \ + -H "Authorization: Basic ${AZURE_DEVOPS_AUTHN_BASIC_TOKEN}" \ + -H "Content-Type: application/json" \ + -o /dev/null ``` -Build the Package service +Kick off CI/CD pipeline ```bash -export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package - -# Build the docker image -docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/package:0.1.0 +git checkout -b release/package/v0.1.0 && \ +git push newremote release/package/v0.1.0 ``` -Deploy the Package service +Verify package was deployed ```bash -# Create secret -# Note: Connection strings cannot be exported as outputs in ARM deployments -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') - -# Deploy service -helm install $HELM_CHARTS/package/ \ - --set image.tag=0.1.0 \ - --set image.repository=package \ - --set secrets.appinsights.ikey=$AI_IKEY \ - --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ - --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ - --set dockerregistry=$ACR_SERVER \ - --set reason="Initial deployment" \ - --namespace backend \ - --name package-v0.1.0 - -# Verify the pod is created helm status package-v0.1.0 ``` diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json new file mode 100644 index 00000000..18f3b7b1 --- /dev/null +++ b/src/shipping/package/azure-pipelines-cd.json @@ -0,0 +1,1954 @@ +{ + "source": 1, + "revision": 20, + "description": null, + "isDeleted": false, + "variables": { + "AI_IKEY": { + "value": "AI_IKEY_VAR_VAL", + "isSecret": true + }, + "ACR_SERVER": { + "value": "ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "ACR_NAME_VAR_VAL" + }, + "COSMOSDB_COL_NAME": { + "value": "COSMOSDB_COL_NAME_VAR_VAL" + }, + "COSMOSDB_CONNECTION": { + "value": "COSMOSDB_CONNECTION_VAR_VAL", + "isSecret": true + }, + "REASON": { + "value": "Azure DevOps CD Pipeline", + "allowOverride": true + }, + "REPO_NAME": { + "value": "package" + } + }, + "variableGroups": [], + "environments": [ + { + "id": 6, + "name": "dev", + "rank": 1, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 18 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 25 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 26 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-package", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-dev", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "ReleaseStarted", + "conditionType": 1, + "value": "" + }, + { + "name": "ci-package", + "conditionType": 4, + "value": "{\"sourceBranch\":\"release/package/v*\",\"tags\":[],\"useBuildDefinitionBranch\":false,\"createReleaseOnBuildTagging\":false}" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 7, + "name": "QA", + "rank": 2, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 19 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 24 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 27 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-package", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-qa", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 8, + "name": "staging", + "rank": 3, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 20 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 23 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 28 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-package", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-staging", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 9, + "name": "production", + "rank": 4, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": false, + "approver": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "isNotificationOn": false, + "approver": null, + "id": 21 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": true, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 22 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 29 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-package", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Pull a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "command": "pull", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(Build.Repository.Name):$(Build.BuildId)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Tag a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "command": "Tag image", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "false", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Push a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "command": "Push an image", + "dockerFile": "**/Dockerfile", + "arguments": "", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=prod/$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service package-experimental app.kubernetes.io/name=package,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 2, + "phaseType": 2, + "name": "Agentless job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "bcb64569-d51a-4af0-9c01-ea5d05b3b622", + "version": "8.*", + "name": "Swap (blue-green)", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 3600, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "instructions": "consider running some canary or just resume for swapping blue and green versions", + "emailRecipients": "", + "onTimeout": "reject" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-package", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 3, + "phaseType": 1, + "name": "Agent job (swap)", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl get", + "refName": "BlueVersion", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "get", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "-n backend svc/package -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector -n backend svc/package-experimental app.kubernetes.io/name=package,app.kubernetes.io/instance=$(BlueVersion.KubectlOutput)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set blue", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service package app.kubernetes.io/name=package,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "QA", + "conditionType": 2, + "value": "4" + }, + { + "name": "staging", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + } + ], + "artifacts": [ + { + "sourceId": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL:AZURE_DEVOPS_PACKAGE_BUILD_ID_VAR_VAL", + "type": "Build", + "alias": "ci-package", + "definitionReference": { + "artifactSourceDefinitionUrl": { + "id": "", + "name": "" + }, + "defaultVersionBranch": { + "id": "", + "name": "" + }, + "defaultVersionSpecific": { + "id": "", + "name": "" + }, + "defaultVersionTags": { + "id": "", + "name": "" + }, + "defaultVersionType": { + "id": "selectDuringReleaseCreationType", + "name": "Specify at the time of release creation" + }, + "definition": { + "id": "AZURE_DEVOPS_PACKAGE_BUILD_ID_VAR_VAL", + "name": "aks-ri-ci-package" + }, + "definitions": { + "id": "", + "name": "" + }, + "IsMultiDefinitionType": { + "id": "False", + "name": "False" + }, + "project": { + "id": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL", + "name": "roadmap" + }, + "repository": { + "id": "AZURE_DEVOPS_REPOS_ID_VAR_VALL", + "name": "roadmap" + } + }, + "isPrimary": true, + "isRetained": false + } + ], + "triggers": [ + { + "artifactAlias": "ci-package", + "triggerConditions": [ + { + "sourceBranch": "release/$(REPO_NAME)/v*", + "tags": [], + "useBuildDefinitionBranch": false, + "createReleaseOnBuildTagging": false + } + ], + "triggerType": 1 + } + ], + "releaseNameFormat": "release-$(rev:r)", + "tags": [], + "pipelineProcess": { + "type": 1 + }, + "properties": { + "DefinitionCreationSource": { + "$type": "System.String", + "$value": "Other" + } + }, + "id": 3, + "name": "package-cd", + "path": null, + "projectReference": null, + "url": null +} From e7f5c040cafaaf6a7a18efb422d58ea174ee1b8c Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 18 Apr 2019 18:12:20 -0300 Subject: [PATCH 152/246] [workflow] add CD pipeline --- deploymentCICD.md | 75 +- src/shipping/workflow/azure-pipelines-cd.json | 1676 +++++++++++++++++ 2 files changed, 1711 insertions(+), 40 deletions(-) create mode 100644 src/shipping/workflow/azure-pipelines-cd.json diff --git a/deploymentCICD.md b/deploymentCICD.md index 0a9e2f16..edae96ff 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -388,7 +388,7 @@ Verify package was deployed helm status package-v0.1.0 ``` -## Add Workflow CI +## Add Workflow CI/CD ``` # add build definitions @@ -401,58 +401,53 @@ az pipelines create \ --repository-type tfsgit \ --repository $AZURE_DEVOPS_REPOS_NAME \ --branch master -``` - -## Deploy the Workflow service - -Extract resource details from deployment - -```bash -export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) -``` - -Build the workflow service -```bash -export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow +# query build definition details and resources +export AZURE_DEVOPS_WORKFLOW_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='workflow-ci'].id" -o tsv) && \ +export AZURE_DEVOPS_WORKFLOW_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='workflow-ci'].queue.id" -o tsv) && \ +export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) -# Build the Docker image -docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/. +# add relese definition +cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ + sed "s#TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ + sed "s#AZURE_DEVOPS_WORKFLOW_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_WORKFLOW_BUILD_ID#g" | \ + sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ + sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ + sed "s#AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_WORKFLOW_QUEUE_ID#g" | \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + > $WORKFLOW_PATH/azure-pipelines-cd-0.json -# Push the image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/workflow:0.1.0 +curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ + -d@${WORKFLOW_PATH}/azure-pipelines-cd-0.json \ + -H "Authorization: Basic ${AZURE_DEVOPS_AUTHN_BASIC_TOKEN}" \ + -H "Content-Type: application/json" \ + -o /dev/null ``` -Create and set up pod identity +Kick off CI/CD pipeline ```bash -# Extract outputs from deployment -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) +git checkout -b release/workflow/v0.1.0 && \ +git push newremote release/workflow/v0.1.0 ``` -Deploy the Workflow service: +Verify workflow was deployed ```bash -# Deploy the service -helm install $HELM_CHARTS/workflow/ \ - --set image.tag=0.1.0 \ - --set image.repository=workflow \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ - --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ - --set keyvault.resourcegroup=$RESOURCE_GROUP \ - --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ - --set keyvault.tenantid=$TENANT_ID \ - --set reason="Initial deployment" \ - --namespace backend \ - --name workflow-v0.1.0 - -# Verify the pod is created helm status workflow-v0.1.0 ``` + ## Add Ingestion CI ``` diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json new file mode 100644 index 00000000..dc58d789 --- /dev/null +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -0,0 +1,1676 @@ +{ + "source": 1, + "revision": 3, + "description": null, + "isDeleted": false, + "variables": { + "ACR_NAME": { + "value": "ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "ACR_SERVER_VAR_VAL" + }, + "REASON": { + "value": "Azure DevOps CD Pipeline", + "allowOverride": true + }, + "REPO_NAME": { + "value": "workflow" + }, + "RESOURCE_GROUP": { + "value": "RESOURCE_GROUP_VAR_VAL" + }, + "SUBSCRIPTION_ID": { + "value": "SUBSCRIPTION_ID_VAR_VAL" + }, + "TENANT_ID": { + "value": "TENANT_ID_VAR_VAL" + }, + "WORKFLOW_KEYVAULT_NAME": { + "value": "WORKFLOW_KEYVAULT_NAME_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_CLIENT_ID": { + "value": "WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_RESOURCE_ID": { + "value": "WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, + "variableGroups": [], + "environments": [ + { + "id": 34, + "name": "dev", + "rank": 1, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 102 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 109 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 110 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-dev", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "ReleaseStarted", + "conditionType": 1, + "value": "" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 35, + "name": "QA", + "rank": 2, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 103 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 108 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 111 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-qa", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 36, + "name": "staging", + "rank": 3, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 104 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 107 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 112 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-staging", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 37, + "name": "production", + "rank": 4, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": false, + "isNotificationOn": false, + "approver": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "approver": null, + "id": 105 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": true, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 106 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 113 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-workflow", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Pull a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "pull", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(Build.Repository.Name):$(Build.BuildId)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Tag a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Tag image", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "false", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Push a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Push an image", + "dockerFile": "**/Dockerfile", + "arguments": "", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "QA", + "conditionType": 2, + "value": "4" + }, + { + "name": "staging", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + } + ], + "artifacts": [ + { + "sourceId": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL:AZURE_DEVOPS_WORKFLOW_BUILD_ID_VAR_VAL", + "type": "Build", + "alias": "ci-workflow", + "definitionReference": { + "artifactSourceDefinitionUrl": { + "id": "", + "name": "" + }, + "defaultVersionBranch": { + "id": "", + "name": "" + }, + "defaultVersionSpecific": { + "id": "", + "name": "" + }, + "defaultVersionTags": { + "id": "", + "name": "" + }, + "defaultVersionType": { + "id": "selectDuringReleaseCreationType", + "name": "Specify at the time of release creation" + }, + "definition": { + "id": "AZURE_DEVOPS_WORKFLOW_BUILD_ID_VAR_VAL", + "name": "aks-ri-ci-workflow" + }, + "definitions": { + "id": "", + "name": "" + }, + "IsMultiDefinitionType": { + "id": "False", + "name": "False" + }, + "project": { + "id": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL", + "name": "roadmap" + }, + "repository": { + "id": "AZURE_DEVOPS_REPOS_ID_VAR_VALL", + "name": "roadmap" + } + }, + "isPrimary": true, + "isRetained": false + } + ], + "triggers": [ + { + "artifactAlias": "ci-workflow", + "triggerConditions": [ + { + "sourceBranch": "release/$(REPO_NAME)/v*", + "tags": [], + "useBuildDefinitionBranch": false, + "createReleaseOnBuildTagging": false + } + ], + "triggerType": 1 + } + ], + "releaseNameFormat": "release-$(rev:r)", + "tags": [], + "pipelineProcess": { + "type": 1 + }, + "properties": { + "DefinitionCreationSource": { + "$type": "System.String", + "$value": "ReleaseImport" + } + }, + "id": 17, + "name": "workflow-cd", + "path": null, + "projectReference": null, + "url": null +} From 0d42b02513829860d33bbe30028b62d375c65b69 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Thu, 18 Apr 2019 19:23:09 -0300 Subject: [PATCH 153/246] [dronescheduler] add CD pipeline --- deploymentCICD.md | 71 +- .../dronescheduler/azure-pipelines-cd.json | 1946 +++++++++++++++++ 2 files changed, 1978 insertions(+), 39 deletions(-) create mode 100644 src/shipping/dronescheduler/azure-pipelines-cd.json diff --git a/deploymentCICD.md b/deploymentCICD.md index edae96ff..1741c2f7 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -531,8 +531,9 @@ helm install $HELM_CHARTS/ingestion/ \ helm status ingestion-v0.1.0 ``` -## Add DroneScheduler CI +## Add DroneScheduler CI/CD +Create build and release pipeline definitions ``` # add build definitions az pipelines create \ @@ -544,56 +545,48 @@ az pipelines create \ --repository-type tfsgit \ --repository $AZURE_DEVOPS_REPOS_NAME \ --branch master -``` - -## Deploy DroneScheduler service - -Extract resource details from deployment -```bash +# query build definition details and resources +export AZURE_DEVOPS_DRONE_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='dronescheduler-ci'].id" -o tsv) && \ +export AZURE_DEVOPS_DRONE_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='dronescheduler-ci'].queue.id" -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) && \ export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) -``` -Build the dronescheduler services +# add relese definition +cat $DRONE_PATH/azure-pipelines-cd.json | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ + sed "s#AZURE_DEVOPS_DRONE_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_DRONE_BUILD_ID#g" | \ + sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ + sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ + sed "s#AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_DRONE_QUEUE_ID#g" | \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + > $DRONE_PATH/azure-pipelines-cd-0.json -```bash -export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler +curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ + -d@${DRONE_PATH}/azure-pipelines-cd-0.json \ + -H "Authorization: Basic ${AZURE_DEVOPS_AUTHN_BASIC_TOKEN}" \ + -H "Content-Type: application/json" \ + -o /dev/null ``` -Create and set up pod identity +Kick off CI/CD pipeline ```bash -# Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +git checkout -b release/dronescheduler/v0.1.0 && \ +git push newremote release/dronescheduler/v0.1.0 ``` -Build and publish the container image - -```bash -# Build the Docker image -docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../ - -# Push the images to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/dronescheduler:0.1.0 -``` +Verify dronescheduler was deployed -Deploy the dronescheduler service: ```bash -# Deploy the service -helm install $HELM_CHARTS/dronescheduler/ \ - --set image.tag=0.1.0 \ - --set image.repository=dronescheduler \ - --set dockerregistry=$ACR_SERVER \ - --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ - --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ - --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ - --set reason="Initial deployment" \ - --namespace backend \ - --name dronescheduler-v0.1.0 - -# Verify the pod is created helm status dronescheduler-v0.1.0 ``` diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json new file mode 100644 index 00000000..dedc8f7f --- /dev/null +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -0,0 +1,1946 @@ +{ + "source": 1, + "revision": 3, + "description": null, + "isDeleted": false, + "variables": { + "ACR_NAME": { + "value": "ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "ACR_SERVER_VAR_VAL" + }, + "DRONESCHEDULER_KEYVAULT_URI": { + "value": "DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { + "value": "DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { + "value": "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "REASON": { + "value": "Azure DevOps CD Pipeline", + "allowOverride": true + }, + "REPO_NAME": { + "value": "dronescheduler" + } + }, + "variableGroups": [], + "environments": [ + { + "id": 30, + "name": "dev", + "rank": 1, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 90 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 97 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 98 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-dronescheduler", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-dev", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "ReleaseStarted", + "conditionType": 1, + "value": "" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 31, + "name": "QA", + "rank": 2, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 91 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 96 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 99 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-dronescheduler", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-qa", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 32, + "name": "staging", + "rank": 3, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 92 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 95 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 100 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-dronescheduler", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-staging", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 33, + "name": "production", + "rank": 4, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": false, + "isNotificationOn": false, + "approver": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "id": 93 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": true, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 94 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 101 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-dronescheduler", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Pull a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "pull", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(Build.Repository.Name):$(Build.BuildId)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Tag a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Tag image", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "false", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Push a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Push an image", + "dockerFile": "**/Dockerfile", + "arguments": "", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service dronescheduler-experimental app.kubernetes.io/name=dronescheduler,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 2, + "phaseType": 2, + "name": "Agentless job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "bcb64569-d51a-4af0-9c01-ea5d05b3b622", + "version": "8.*", + "name": "Swap (blue-green)", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 3600, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "instructions": "consider running some canary or just resume for swapping blue and green versions", + "emailRecipients": "", + "onTimeout": "reject" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [ + { + "alias": "ci-dronescheduler", + "artifactType": "Build", + "artifactDownloadMode": "All", + "artifactItems": [] + } + ] + }, + "queueId": AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 3, + "phaseType": 1, + "name": "Agent job (swap)", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl get", + "refName": "BlueVersion", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "get", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "-n backend svc/dronescheduler -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector -n backend svc/dronescheduler-experimental app.kubernetes.io/name=dronescheduler,app.kubernetes.io/instance=$(BlueVersion.KubectlOutput)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set blue", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service dronescheduler app.kubernetes.io/name=dronescheduler,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "QA", + "conditionType": 2, + "value": "4" + }, + { + "name": "staging", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + } + ], + "artifacts": [ + { + "sourceId": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL:AZURE_DEVOPS_DRONE_BUILD_ID_VAR_VAL", + "type": "Build", + "alias": "ci-dronescheduler", + "definitionReference": { + "artifactSourceDefinitionUrl": { + "id": "", + "name": "" + }, + "defaultVersionBranch": { + "id": "", + "name": "" + }, + "defaultVersionSpecific": { + "id": "", + "name": "" + }, + "defaultVersionTags": { + "id": "", + "name": "" + }, + "defaultVersionType": { + "id": "selectDuringReleaseCreationType", + "name": "Specify at the time of release creation" + }, + "definition": { + "id": "AZURE_DEVOPS_DRONE_BUILD_ID_VAR_VAL", + "name": "aks-ri-ci-dronescheduler" + }, + "definitions": { + "id": "", + "name": "" + }, + "IsMultiDefinitionType": { + "id": "False", + "name": "False" + }, + "project": { + "id": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL", + "name": "roadmap" + }, + "repository": { + "id": "AZURE_DEVOPS_REPOS_ID_VAR_VALL", + "name": "roadmap" + } + }, + "isPrimary": true, + "isRetained": false + } + ], + "triggers": [ + { + "artifactAlias": "ci-dronescheduler", + "triggerConditions": [ + { + "sourceBranch": "release/$(REPO_NAME)/v*", + "tags": [], + "useBuildDefinitionBranch": false, + "createReleaseOnBuildTagging": false + } + ], + "triggerType": 1 + } + ], + "releaseNameFormat": "release-$(rev:r)", + "tags": [], + "pipelineProcess": { + "type": 1 + }, + "properties": { + "DefinitionCreationSource": { + "$type": "System.String", + "$value": "ReleaseImport" + } + }, + "id": 16, + "name": "dronescheduler-cd", + "path": null, + "projectReference": null, + "url": null +} From 51add4e7630c5d1ac671437672e8ab8c7c80c8ec Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 22 Apr 2019 17:21:18 -0300 Subject: [PATCH 154/246] [ingestion] add CD pipeline --- deploymentCICD.md | 118 +- .../ingestion/azure-pipelines-cd.json | 1932 +++++++++++++++++ 2 files changed, 1993 insertions(+), 57 deletions(-) create mode 100644 src/shipping/ingestion/azure-pipelines-cd.json diff --git a/deploymentCICD.md b/deploymentCICD.md index 1741c2f7..21abfc16 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -448,8 +448,29 @@ Verify workflow was deployed helm status workflow-v0.1.0 ``` -## Add Ingestion CI +## Add Ingestion CI/CD +Ingestion pre-requisites + +```bash +# Deploy the ngnix ingress controller +helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true + +# Obtain the load balancer ip address and assign a domain name +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" +export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) +INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls + +# Create a self-signed certificate for TLS +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls.crt \ + -keyout ingestion-ingress-tls.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" +``` + +Create build and release pipeline definitions ``` # add build definitions az pipelines create \ @@ -461,73 +482,56 @@ az pipelines create \ --repository-type tfsgit \ --repository $AZURE_DEVOPS_REPOS_NAME \ --branch master -``` -## Deploy the Ingestion service +# query build definition details and resources +export AZURE_DEVOPS_INGESTION_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='ingestion-ci'].id" -o tsv) && \ +export AZURE_DEVOPS_INGESTION_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='ingestion-ci'].queue.id" -o tsv) && \ +export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ +export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) && \ +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) && \ +export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) && \ +export INGRESS_TLS_SECRET_CERT=$(echo $(cat ingestion-ingress-tls.crt) | tr '\n' "\\n") && \ +export INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls.key) | tr '\n' "\\n") -Extract resource details from deployment +# add relese definition +cat $INGESTION_PATH/azure-pipelines-cd.json | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ + sed "s#INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ + sed "s#INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ + sed "s#AZURE_DEVOPS_INGESTION_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_INGESTION_BUILD_ID#g" | \ + sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ + sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ + sed "s#AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_INGESTION_QUEUE_ID#g" | \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + > $INGESTION_PATH/azure-pipelines-cd-0.json -```bash -export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ -export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) -export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) -export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) +curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ + -d@${INGESTION_PATH}/azure-pipelines-cd-0.json \ + -H "Authorization: Basic ${AZURE_DEVOPS_AUTHN_BASIC_TOKEN}" \ + -H "Content-Type: application/json" \ + -o /dev/null ``` -Build the Ingestion service +Kick off CI/CD pipeline ```bash -export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion - -# Build the docker image -docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH - -# Push the docker image to ACR -az acr login --name $ACR_NAME -docker push $ACR_SERVER/ingestion:0.1.0 +git checkout -b release/ingestion/v0.1.0 && \ +git push newremote release/ingestion/v0.1.0 ``` -Deploy the Ingestion service +Verify ingestion was deployed ```bash -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true - -# Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done -export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" -export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) -INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls - -# Create a self-signed certificate for TLS -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -out ingestion-ingress-tls.crt \ - -keyout ingestion-ingress-tls.key \ - -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" - -# Deploy service -helm install $HELM_CHARTS/ingestion/ \ - --set image.tag=0.1.0 \ - --set image.repository=ingestion \ - --set dockerregistry=$ACR_SERVER \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=ingestion \ - --set ingress.hosts[0].tls=true \ - --set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \ - --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ - --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ - --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ - --set secrets.appinsights.ikey=${AI_IKEY} \ - --set secrets.queue.keyname=IngestionServiceAccessKey \ - --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ - --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ - --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ - --set reason="Initial deployment" \ - --namespace backend \ - --name ingestion-v0.1.0 - -# Verify the pod is created helm status ingestion-v0.1.0 ``` diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json new file mode 100644 index 00000000..88b2e507 --- /dev/null +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -0,0 +1,1932 @@ +{ + "source": 1, + "revision": 5, + "description": null, + "isDeleted": false, + "variables": { + "ACR_NAME": { + "value": "ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "ACR_SERVER_VAR_VAL" + }, + "AI_IKEY": { + "value": "AI_IKEY_VAR_VAL", + "isSecret": true + }, + "EXTERNAL_INGEST_FQDN": { + "value": "EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGESTION_ACCESS_KEY_VALUE": { + "value": "INGESTION_ACCESS_KEY_VALUE_VAR_VAL", + "isSecret": true + }, + "INGESTION_QUEUE_NAME": { + "value": "INGESTION_QUEUE_NAME_VAR_VAL" + }, + "INGESTION_QUEUE_NAMESPACE": { + "value": "INGESTION_QUEUE_NAMESPACE_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "INGRESS_TLS_SECRET_NAME_VAR_VAL" + }, + "REASON": { + "value": "Azure DevOps CD Pipeline", + "allowOverride": true + }, + "REPO_NAME": { + "value": "ingestion" + }, + "SERVICE_NAME": { + "value": "ingestion" + } + }, + "variableGroups": [], + "environments": [ + { + "id": 26, + "name": "dev", + "rank": 1, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 78 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 85 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 86 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-dev", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "ReleaseStarted", + "conditionType": 1, + "value": "" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 27, + "name": "QA", + "rank": 2, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 79 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 84 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 87 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-qa", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", + "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 28, + "name": "staging", + "rank": 3, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 80 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 83 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 88 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend-staging", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "dev", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + }, + { + "id": 29, + "name": "production", + "rank": 4, + "owner": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "variables": {}, + "variableGroups": [], + "preDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": false, + "isNotificationOn": false, + "approver": { + "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" + }, + "id": 81 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": true, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 1 + } + }, + "deployStep": { + "id": 82 + }, + "postDeployApprovals": { + "approvals": [ + { + "rank": 1, + "isAutomated": true, + "isNotificationOn": false, + "id": 89 + } + ], + "approvalOptions": { + "requiredApproverCount": null, + "releaseCreatorCanBeApprover": false, + "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, + "enforceIdentityRevalidation": false, + "timeoutInMinutes": 0, + "executionOrder": 2 + } + }, + "deployPhases": [ + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 1, + "phaseType": 1, + "name": "Agent job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Pull a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "pull", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(Build.Repository.Name):$(Build.BuildId)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Tag a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Tag image", + "dockerFile": "**/Dockerfile", + "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "false", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "version": "1.*", + "name": "Push a docker image", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "containerregistrytype": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", + "command": "Push an image", + "dockerFile": "**/Dockerfile", + "arguments": "", + "pushMultipleImages": "false", + "tagMultipleImages": "false", + "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", + "imageNamesPath": "", + "qualifyImageName": "true", + "includeSourceTags": "false", + "includeLatestTag": "false", + "addDefaultLabels": "true", + "useDefaultContext": "true", + "buildContext": "", + "imageDigestFile": "", + "containerName": "", + "ports": "", + "volumes": "", + "envVars": "", + "workingDirectory": "", + "entrypointOverride": "", + "containerCommand": "", + "runInBackground": "true", + "restartPolicy": "no", + "maxRestartRetries": "", + "dockerHostEndpoint": "", + "enforceDockerNamingConvention": "true", + "memoryLimit": "" + } + }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "backend", + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service ingestion-experimental app.kubernetes.io/name=ingestion,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 2, + "phaseType": 2, + "name": "Agentless job", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "bcb64569-d51a-4af0-9c01-ea5d05b3b622", + "version": "8.*", + "name": "Swap (blue-green)", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 3600, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "instructions": "consider running some canary or just resume for swapping blue and green versions", + "emailRecipients": "", + "onTimeout": "reject" + } + } + ] + }, + { + "deploymentInput": { + "parallelExecution": { + "parallelExecutionType": 0 + }, + "skipArtifactsDownload": false, + "artifactsDownloadInput": { + "downloadInputs": [] + }, + "queueId": AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL, + "demands": [], + "enableAccessToken": false, + "timeoutInMinutes": 0, + "jobCancelTimeoutInMinutes": 1, + "condition": "succeeded()", + "overrideInputs": {} + }, + "rank": 3, + "phaseType": 1, + "name": "Agent job (swap)", + "refName": null, + "workflowTasks": [ + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl get", + "refName": "BlueVersion", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "get", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "-n backend svc/ingestion -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set green", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector -n backend svc/ingestion-experimental app.kubernetes.io/name=ingestion,app.kubernetes.io/instance=$(BlueVersion.KubectlOutput)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + }, + { + "environment": {}, + "taskId": "cbc316a2-586f-4def-be79-488a1f503564", + "version": "1.*", + "name": "kubectl set blue", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "Azure Resource Manager", + "kubernetesServiceEndpoint": "", + "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", + "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", + "useClusterAdmin": "false", + "namespace": "backend", + "command": "set", + "useConfigurationFile": "false", + "configurationType": "configuration", + "configuration": "", + "inline": "", + "arguments": "selector service ingestion app.kubernetes.io/name=ingestion,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", + "secretType": "dockerRegistry", + "secretArguments": "", + "containerRegistryType": "Azure Container Registry", + "dockerRegistryEndpoint": "", + "azureSubscriptionEndpointForSecrets": "", + "azureContainerRegistry": "", + "secretName": "", + "forceUpdate": "true", + "configMapName": "", + "forceUpdateConfigMap": "false", + "useConfigMapFile": "false", + "configMapFile": "", + "configMapArguments": "", + "versionOrLocation": "version", + "versionSpec": "1.12.4", + "checkLatest": "false", + "specifyLocation": "", + "cwd": "$(System.DefaultWorkingDirectory)", + "outputFormat": "json" + } + } + ] + } + ], + "environmentOptions": { + "emailNotificationType": "OnlyOnFailure", + "emailRecipients": "release.environment.owner;release.creator", + "skipArtifactsDownload": false, + "timeoutInMinutes": 0, + "enableAccessToken": false, + "publishDeploymentStatus": true, + "badgeEnabled": false, + "autoLinkWorkItems": false, + "pullRequestDeploymentEnabled": false + }, + "demands": [], + "conditions": [ + { + "name": "QA", + "conditionType": 2, + "value": "4" + }, + { + "name": "staging", + "conditionType": 2, + "value": "4" + } + ], + "executionPolicy": { + "concurrencyCount": 1, + "queueDepthCount": 0 + }, + "schedules": [], + "retentionPolicy": { + "daysToKeep": 30, + "releasesToKeep": 3, + "retainBuild": true + }, + "processParameters": { + "inputs": [ + { + "aliases": [], + "options": { + "Azure Resource Manager": "Azure Resource Manager", + "Kubernetes Service Connection": "Kubernetes Service Connection" + }, + "properties": { + "EditableOptions": "false" + }, + "name": "connectionType", + "label": "Connection Type.", + "defaultValue": "Azure Resource Manager", + "type": "pickList", + "helpMarkDown": "", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "azureSubscriptionEndpoint", + "label": "Azure subscription", + "defaultValue": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "required": true, + "type": "connectedService:AzureRM", + "helpMarkDown": "Select an Azure subscription, which has your Azure Container Registry.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "azureResourceGroup", + "label": "Resource group", + "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Resource Group.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": { + "EditableOptions": "True" + }, + "name": "kubernetesCluster", + "label": "Kubernetes cluster", + "defaultValue": "CLUSTER_NAME_VAR_VAL", + "required": true, + "type": "pickList", + "helpMarkDown": "Select an Azure Kubernetes Service cluster.", + "visibleRule": "connectionType = Azure Resource Manager", + "groupName": "" + }, + { + "aliases": [], + "options": {}, + "properties": {}, + "name": "kubernetesServiceEndpoint", + "label": "Kubernetes Service Connection", + "defaultValue": "", + "required": true, + "type": "connectedService:kubernetes", + "helpMarkDown": "Select a Kubernetes service connection.", + "visibleRule": "connectionType = Kubernetes Service Connection", + "groupName": "" + } + ], + "dataSourceBindings": [ + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "kubernetesCluster", + "resultTemplate": "{{{name}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/resourceGroups/$(azureResourceGroup)/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + }, + { + "parameters": {}, + "endpointId": "$(azureSubscriptionEndpoint)", + "target": "azureResourceGroup", + "resultTemplate": "{{{ #extractResource id resourcegroups}}}", + "endpointUrl": "{{{endpoint.url}}}/subscriptions/{{{endpoint.subscriptionId}}}/providers/Microsoft.ContainerService/managedClusters?api-version=2017-08-31", + "resultSelector": "jsonpath:$.value[*]" + } + ] + }, + "properties": {}, + "preDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "postDeploymentGates": { + "id": 0, + "gatesOptions": null, + "gates": [] + }, + "environmentTriggers": [], + "badgeUrl": null + } + ], + "artifacts": [ + { + "sourceId": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL:AZURE_DEVOPS_INGESTION_BUILD_ID_VAR_VAL", + "type": "Build", + "alias": "ci-ingestion", + "definitionReference": { + "artifactSourceDefinitionUrl": { + "id": "", + "name": "" + }, + "defaultVersionBranch": { + "id": "", + "name": "" + }, + "defaultVersionSpecific": { + "id": "", + "name": "" + }, + "defaultVersionTags": { + "id": "", + "name": "" + }, + "defaultVersionType": { + "id": "selectDuringReleaseCreationType", + "name": "Specify at the time of release creation" + }, + "definition": { + "id": "AZURE_DEVOPS_INGESTION_BUILD_ID_VAR_VAL", + "name": "aks-ri-ci-ingestion" + }, + "definitions": { + "id": "", + "name": "" + }, + "IsMultiDefinitionType": { + "id": "False", + "name": "False" + }, + "project": { + "id": "AZURE_DEVOPS_PROJECT_ID_VAR_VAL", + "name": "roadmap" + }, + "repository": { + "id": "AZURE_DEVOPS_REPOS_ID_VAR_VALL", + "name": "roadmap" + } + }, + "isPrimary": true, + "isRetained": false + } + ], + "triggers": [ + { + "artifactAlias": "ci-ingestion", + "triggerConditions": [ + { + "sourceBranch": "release/$(REPO_NAME)/v*", + "tags": [], + "useBuildDefinitionBranch": false, + "createReleaseOnBuildTagging": false + } + ], + "triggerType": 1 + } + ], + "releaseNameFormat": "release-$(rev:r)", + "tags": [], + "pipelineProcess": { + "type": 1 + }, + "properties": { + "DefinitionCreationSource": { + "$type": "System.String", + "$value": "ReleaseImport" + } + }, + "id": 15, + "name": "ingestion-cd", + "path": null, + "projectReference": null, + "url": null +} From 68ab7b43a46a0f1c8783ee2bd6e9f873633eda97 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 3 May 2019 15:29:29 -0300 Subject: [PATCH 155/246] Applying feeback - unify manual and automated deployment experiences - format, content and typos --- deployment.md | 6 ++ deploymentCICD.md | 201 +--------------------------------------------- 2 files changed, 10 insertions(+), 197 deletions(-) diff --git a/deployment.md b/deployment.md index 5a325a1d..ca1cedb3 100644 --- a/deployment.md +++ b/deployment.md @@ -157,6 +157,12 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` +## Optional: Set up CI/CD with Azure DevOps + +Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). + +> Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application as follows. + ## Deploy the Delivery service Extract resource details from deployment diff --git a/deploymentCICD.md b/deploymentCICD.md index 21abfc16..28d03bfd 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -1,4 +1,4 @@ -# Deploying the Reference Implementation +# Setup Reference Implementation CI/CD with Azure DevOps ## Prerequisites @@ -6,157 +6,7 @@ - [Azure CLI 2.0.49 or later](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Azure DevOps account](https://azure.microsoft.com/services/devops) - [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) -- [JQ](https://stedolan.github.io/jq/download/) - -> Note: in linux systems, it is possible to run the docker command without prefacing -> with sudo. For more information, please refer to [the Post-installation steps -> for linux](https://docs.docker.com/install/linux/linux-postinstall/) - -Clone or download this repo locally. - -```bash -git clone https://github.com/mspnp/microservices-reference-implementation.git -``` - -The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. - -## Generate a SSH rsa public/private key pair - -the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see [How to create and use an SSH key pair](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys). -> Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure. - -## Azure Resources Provisioning - -Set environment variables. - -```bash -export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE] - -export LOCATION=[YOUR_LOCATION_HERE] - -export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] - -export SUBSCRIPTION_ID=$(az account show --query id --output tsv) -export SUBSCRIPTION_NAME=$(az account show --query name --output tsv) -export TENANT_ID=$(az account show --query tenantId --output tsv) - -export PROJECT_ROOT=./microservices-reference-implementation -export K8S=$PROJECT_ROOT/k8s -export HELM_CHARTS=$PROJECT_ROOT/charts -``` - -Infrastructure Prerequisites - -```bash -# Log in to Azure -az login - -# Create a resource group and service principal for AKS -az group create --name $RESOURCE_GROUP --location $LOCATION && \ -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ -export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ -export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ -export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) -``` - -Deployment - -> Note: this deployment might take up to 20 minutes - -```bash -# Deploy the managed identities -# These are deployed first in a separate template to avoid propagation delays with AAD -az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file ${PROJECT_ROOT}/azuredeploy-identities.json -export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) -export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) -export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) -export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) -export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) -export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) - -# Wait for AAD propagation -until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done -until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done -until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done - -# Deploy all other resources -# The version of kubernetes must be supported in the target region -export KUBERNETES_VERSION='1.12.6' -az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file ${PROJECT_ROOT}/azuredeploy.json \ ---parameters servicePrincipalClientId=${SP_APP_ID} \ - servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ - servicePrincipalId=${SP_OBJECT_ID} \ - kubernetesVersion=${KUBERNETES_VERSION} \ - sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ - deliveryIdName=${DELIVERY_ID_NAME} \ - deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \ - droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ - droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ - workflowIdName=${WORKFLOW_ID_NAME} \ - workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} -``` - -Get outputs from Azure Deploy -```bash -# Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) -``` - -Enable Azure Monitoring for the AKS cluster -```bash -az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -``` - -Download kubectl and create a k8s namespace -```bash -# Install kubectl -sudo az aks install-cli - -# Get the Kubernetes cluster credentials -az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME - -# Create namespaces -kubectl create namespace backend && \ -kubectl create namespace frontend -``` - -Setup Helm in the container - -```bash -kubectl apply -f $K8S/tiller-rbac.yaml -helm init --service-account tiller -``` - -Integrate Application Insights instance - -```bash -# Acquire Instrumentation Key -export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) -export AI_IKEY=$(az resource show \ - -g $RESOURCE_GROUP \ - -n $AI_NAME \ - --resource-type "Microsoft.Insights/components" \ - --query properties.InstrumentationKey \ - -o tsv) - -# add RBAC for AppInsights -kubectl apply -f $K8S/k8s-rbac-ai.yaml -``` - -## Setup AAD pod identity and key vault flexvol infrastructure - -Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol - -Note: the tested nmi version was 1.4. It enables namespaced pod identity. - -```bash -# setup AAD pod identity -kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml - -kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml -``` +- [Values from deployment instructions](./deployment.md) ## Setup Azure DevOps @@ -204,7 +54,7 @@ open $AZURE_DEVOPS_ORG/$AZURE_DEVOPS_PROJECT_NAME/_git/$AZURE_REPOS_NAME ``` > Follow instructions at [Use SSH Key authentication](https://docs.microsoft.com/en-us/azure/devops/repos/git/use-ssh-keys-to-authenticate?view=azure-devops) to add ssh. -For more information on the different authentication types, please take a look at [Authtentication comparison](https://docs.microsoft.com/en-us/azure/devops/repos/git/auth-overview?view=azure-devops#authentication-comparison) +For more information on the different authentication types, please take a look at [Authentication comparison](https://docs.microsoft.com/en-us/azure/devops/repos/git/auth-overview?view=azure-devops#authentication-comparison) ![](https://docs.microsoft.com/en-us/azure/devops/repos/git/_img/ssh_add_public_key.gif?view=azure-devops) @@ -596,47 +446,4 @@ helm status dronescheduler-v0.1.0 ## Validate the application is running -You can send delivery requests to the ingestion service using the Swagger UI. - -Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. - -```bash -open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" -``` - -> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. - - -## Optional steps - -### Load Test the application - -To run load testing against the solution, follow the steps listed [here](./src/shipping/ingress/readme.md). - -### Fluentd and Elastic Search - -Follow these steps to add logging and monitoring capabilities to the solution. - -Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch - -```bash -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \ -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \ -kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml -``` - -Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd - -```bash -# The example elasticsearch yaml files deploy a service named "elasticsearch" -wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \ -sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml - -# Commenting out X-Pack credentials for demo purposes. -# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads -sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \ -sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \ -sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \ -sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ -kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml -``` +You can resume the [the original deployment instructions](./deployment.md#validate-the-application-is-running) to validate the application is running. From e5c9fbbc15b2a92321c1c3cbdfd9744b521d6b1a Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 26 Apr 2019 12:55:06 -0300 Subject: [PATCH 156/246] [delivery] add envs to helm chart --- .../delivery/charts/delivery-dev/Chart.yaml | 4 +++ .../delivery/charts/delivery-dev/values.yaml | 10 ++++++ .../delivery/charts/delivery-prod/Chart.yaml | 4 +++ .../delivery/charts/delivery-prod/values.yaml | 11 +++++++ charts/delivery/charts/delivery-qa/Chart.yaml | 4 +++ .../delivery/charts/delivery-qa/values.yaml | 10 ++++++ .../charts/delivery-staging/Chart.yaml | 4 +++ .../charts/delivery-staging/values.yaml | 10 ++++++ charts/delivery/requirements.yaml | 32 +++++++++++++++++++ .../delivery/templates/delivery-deploy.yaml | 2 +- charts/delivery/values.yaml | 9 +++++- deployment.md | 1 + src/shipping/delivery/azure-pipelines-cd.json | 8 ++--- 13 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 charts/delivery/charts/delivery-dev/Chart.yaml create mode 100644 charts/delivery/charts/delivery-dev/values.yaml create mode 100644 charts/delivery/charts/delivery-prod/Chart.yaml create mode 100644 charts/delivery/charts/delivery-prod/values.yaml create mode 100644 charts/delivery/charts/delivery-qa/Chart.yaml create mode 100644 charts/delivery/charts/delivery-qa/values.yaml create mode 100644 charts/delivery/charts/delivery-staging/Chart.yaml create mode 100644 charts/delivery/charts/delivery-staging/values.yaml create mode 100644 charts/delivery/requirements.yaml diff --git a/charts/delivery/charts/delivery-dev/Chart.yaml b/charts/delivery/charts/delivery-dev/Chart.yaml new file mode 100644 index 00000000..7ad76f0b --- /dev/null +++ b/charts/delivery/charts/delivery-dev/Chart.yaml @@ -0,0 +1,4 @@ +name: delivery-dev +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Service diff --git a/charts/delivery/charts/delivery-dev/values.yaml b/charts/delivery/charts/delivery-dev/values.yaml new file mode 100644 index 00000000..0ccafb8f --- /dev/null +++ b/charts/delivery/charts/delivery-dev/values.yaml @@ -0,0 +1,10 @@ +# Dev values for delivery. +nameOverride: delivery +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new dev deploy" diff --git a/charts/delivery/charts/delivery-prod/Chart.yaml b/charts/delivery/charts/delivery-prod/Chart.yaml new file mode 100644 index 00000000..98682ddf --- /dev/null +++ b/charts/delivery/charts/delivery-prod/Chart.yaml @@ -0,0 +1,4 @@ +name: delivery-prod +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Service diff --git a/charts/delivery/charts/delivery-prod/values.yaml b/charts/delivery/charts/delivery-prod/values.yaml new file mode 100644 index 00000000..a02382e0 --- /dev/null +++ b/charts/delivery/charts/delivery-prod/values.yaml @@ -0,0 +1,11 @@ +# Production values for delivery. +nameOverride: delivery +exports: + data: + replicaCount: 3 + dockerregistrynamespace: "/prod" + image: + pullPolicy: IfNotPresent + telemetry: + level: "Error" + reason: "new prod deploy" diff --git a/charts/delivery/charts/delivery-qa/Chart.yaml b/charts/delivery/charts/delivery-qa/Chart.yaml new file mode 100644 index 00000000..8174cf05 --- /dev/null +++ b/charts/delivery/charts/delivery-qa/Chart.yaml @@ -0,0 +1,4 @@ +name: delivery-qa +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Service diff --git a/charts/delivery/charts/delivery-qa/values.yaml b/charts/delivery/charts/delivery-qa/values.yaml new file mode 100644 index 00000000..433bc215 --- /dev/null +++ b/charts/delivery/charts/delivery-qa/values.yaml @@ -0,0 +1,10 @@ +# QA values for delivery. +nameOverride: delivery +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new qa deploy" diff --git a/charts/delivery/charts/delivery-staging/Chart.yaml b/charts/delivery/charts/delivery-staging/Chart.yaml new file mode 100644 index 00000000..69219341 --- /dev/null +++ b/charts/delivery/charts/delivery-staging/Chart.yaml @@ -0,0 +1,4 @@ +name: delivery-staging +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Service diff --git a/charts/delivery/charts/delivery-staging/values.yaml b/charts/delivery/charts/delivery-staging/values.yaml new file mode 100644 index 00000000..0f4427e7 --- /dev/null +++ b/charts/delivery/charts/delivery-staging/values.yaml @@ -0,0 +1,10 @@ +# Staging values for delivery. +nameOverride: delivery +exports: + data: + replicaCount: 3 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new staging deploy" diff --git a/charts/delivery/requirements.yaml b/charts/delivery/requirements.yaml new file mode 100644 index 00000000..9fd8c214 --- /dev/null +++ b/charts/delivery/requirements.yaml @@ -0,0 +1,32 @@ +dependencies: + - name: delivery-dev + repository: "file://charts/delivery-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data + + - name: delivery-prod + repository: "file://charts/delivery-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data + + - name: delivery-qa + repository: "file://charts/delivery-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + + - name: delivery-staging + repository: "file://charts/delivery-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 0efcaa43..deea1345 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -44,7 +44,7 @@ spec: fsGroup: 1 containers: - name: fabrikam-delivery - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: DOCDB_DATABASEID diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index 11a0eda1..02ffbb26 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -1,8 +1,10 @@ # Default values for dronedelivery. -replicaCount: 3 +nameOverride: delivery +replicaCount: 1 identity: clientid: resourceid: +dockerregistrynamespace: dockerregistry: image: repository: @@ -23,3 +25,8 @@ resources: limits: cpu: 500m memory: 500Mi +tags: + dev: false + prod: false + qa: false + staging: false diff --git a/deployment.md b/deployment.md index ca1cedb3..9d0a058d 100644 --- a/deployment.md +++ b/deployment.md @@ -209,6 +209,7 @@ helm install $HELM_CHARTS/delivery/ \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ --set reason="Initial deployment" \ + --set tags.prod=true \ --namespace backend \ --name delivery-v0.1.0 diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index 26c5745b..ba3a65b7 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -231,7 +231,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -593,7 +593,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -950,7 +950,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1450,7 +1450,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From f69c4ec368ac9937e23e72d909f4d4c0aa7d5882 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 26 Apr 2019 13:07:21 -0300 Subject: [PATCH 157/246] [package] add envs to helm chart --- charts/package/charts/package-dev/Chart.yaml | 4 +++ charts/package/charts/package-dev/values.yaml | 10 ++++++ charts/package/charts/package-prod/Chart.yaml | 4 +++ .../package/charts/package-prod/values.yaml | 11 +++++++ charts/package/charts/package-qa/Chart.yaml | 4 +++ charts/package/charts/package-qa/values.yaml | 10 ++++++ .../package/charts/package-staging/Chart.yaml | 4 +++ .../charts/package-staging/values.yaml | 10 ++++++ charts/package/requirements.yaml | 32 +++++++++++++++++++ charts/package/templates/package-deploy.yaml | 2 +- charts/package/values.yaml | 9 +++++- deployment.md | 1 + src/shipping/package/azure-pipelines-cd.json | 8 ++--- 13 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 charts/package/charts/package-dev/Chart.yaml create mode 100644 charts/package/charts/package-dev/values.yaml create mode 100644 charts/package/charts/package-prod/Chart.yaml create mode 100644 charts/package/charts/package-prod/values.yaml create mode 100644 charts/package/charts/package-qa/Chart.yaml create mode 100644 charts/package/charts/package-qa/values.yaml create mode 100644 charts/package/charts/package-staging/Chart.yaml create mode 100644 charts/package/charts/package-staging/values.yaml create mode 100644 charts/package/requirements.yaml diff --git a/charts/package/charts/package-dev/Chart.yaml b/charts/package/charts/package-dev/Chart.yaml new file mode 100644 index 00000000..9b9e3a5d --- /dev/null +++ b/charts/package/charts/package-dev/Chart.yaml @@ -0,0 +1,4 @@ +name: package-dev +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Package Service diff --git a/charts/package/charts/package-dev/values.yaml b/charts/package/charts/package-dev/values.yaml new file mode 100644 index 00000000..d9d1e84c --- /dev/null +++ b/charts/package/charts/package-dev/values.yaml @@ -0,0 +1,10 @@ +# Dev values for package. +nameOverride: package +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + log: + level: "info" + reason: "new dev deploy" diff --git a/charts/package/charts/package-prod/Chart.yaml b/charts/package/charts/package-prod/Chart.yaml new file mode 100644 index 00000000..141e274d --- /dev/null +++ b/charts/package/charts/package-prod/Chart.yaml @@ -0,0 +1,4 @@ +name: package-prod +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Package Service diff --git a/charts/package/charts/package-prod/values.yaml b/charts/package/charts/package-prod/values.yaml new file mode 100644 index 00000000..5f897e06 --- /dev/null +++ b/charts/package/charts/package-prod/values.yaml @@ -0,0 +1,11 @@ +# Production values for package. +nameOverride: package +exports: + data: + replicaCount: 3 + dockerregistrynamespace: "/prod" + image: + pullPolicy: IfNotPresent + log: + level: "error" + reason: "new prod deploy" diff --git a/charts/package/charts/package-qa/Chart.yaml b/charts/package/charts/package-qa/Chart.yaml new file mode 100644 index 00000000..43e1f25a --- /dev/null +++ b/charts/package/charts/package-qa/Chart.yaml @@ -0,0 +1,4 @@ +name: package-qa +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Package Service diff --git a/charts/package/charts/package-qa/values.yaml b/charts/package/charts/package-qa/values.yaml new file mode 100644 index 00000000..5f8b06e7 --- /dev/null +++ b/charts/package/charts/package-qa/values.yaml @@ -0,0 +1,10 @@ +# QA values for package. +nameOverride: package +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + log: + level: "info" + reason: "new qa deploy" diff --git a/charts/package/charts/package-staging/Chart.yaml b/charts/package/charts/package-staging/Chart.yaml new file mode 100644 index 00000000..60024b05 --- /dev/null +++ b/charts/package/charts/package-staging/Chart.yaml @@ -0,0 +1,4 @@ +name: package-staging +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Package Service diff --git a/charts/package/charts/package-staging/values.yaml b/charts/package/charts/package-staging/values.yaml new file mode 100644 index 00000000..5b7777a8 --- /dev/null +++ b/charts/package/charts/package-staging/values.yaml @@ -0,0 +1,10 @@ +# Staging values for package. +nameOverride: package +exports: + data: + replicaCount: 3 + image: + pullPolicy: Always + log: + level: "info" + reason: "new staging deploy" diff --git a/charts/package/requirements.yaml b/charts/package/requirements.yaml new file mode 100644 index 00000000..debe028e --- /dev/null +++ b/charts/package/requirements.yaml @@ -0,0 +1,32 @@ +dependencies: + - name: package-dev + repository: "file://charts/package-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data + + - name: package-prod + repository: "file://charts/package-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data + + - name: package-qa + repository: "file://charts/package-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + + - name: package-staging + repository: "file://charts/package-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index a6bb9045..dc162481 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -39,7 +39,7 @@ spec: spec: containers: - name: &package-container_name fabrikam-package - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CONNECTION_STRING diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 1de9dab4..39f6f777 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -1,5 +1,7 @@ # Default values for package service. -replicaCount: 3 +nameOverride: package +replicaCount: 1 +dockerregistrynamespace: dockerregistry: image: repository: @@ -17,3 +19,8 @@ resources: limits: cpu: 410m memory: 140Mi +tags: + dev: false + prod: false + qa: false + staging: false diff --git a/deployment.md b/deployment.md index 9d0a058d..912d963c 100644 --- a/deployment.md +++ b/deployment.md @@ -255,6 +255,7 @@ helm install $HELM_CHARTS/package/ \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ + --set tags.prod=true \ --namespace backend \ --name package-v0.1.0 diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 18f3b7b1..06ec271e 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -227,7 +227,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -588,7 +588,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -944,7 +944,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1445,7 +1445,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=prod/$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From 46d3da1384360ec649314ca6e2dbc8aa57a394ee Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 26 Apr 2019 13:39:46 -0300 Subject: [PATCH 158/246] [workflow] add envs to helm chart --- charts/workflow/Chart.yaml | 2 +- .../workflow/charts/workflow-dev/Chart.yaml | 4 +++ .../workflow/charts/workflow-dev/values.yaml | 10 ++++++ .../workflow/charts/workflow-prod/Chart.yaml | 4 +++ .../workflow/charts/workflow-prod/values.yaml | 11 +++++++ charts/workflow/charts/workflow-qa/Chart.yaml | 4 +++ .../workflow/charts/workflow-qa/values.yaml | 10 ++++++ .../charts/workflow-staging/Chart.yaml | 4 +++ .../charts/workflow-staging/values.yaml | 10 ++++++ charts/workflow/requirements.yaml | 32 +++++++++++++++++++ .../workflow/templates/workflow-deploy.yaml | 2 +- charts/workflow/values.yaml | 9 +++++- deployment.md | 1 + src/shipping/workflow/azure-pipelines-cd.json | 8 ++--- 14 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 charts/workflow/charts/workflow-dev/Chart.yaml create mode 100644 charts/workflow/charts/workflow-dev/values.yaml create mode 100644 charts/workflow/charts/workflow-prod/Chart.yaml create mode 100644 charts/workflow/charts/workflow-prod/values.yaml create mode 100644 charts/workflow/charts/workflow-qa/Chart.yaml create mode 100644 charts/workflow/charts/workflow-qa/values.yaml create mode 100644 charts/workflow/charts/workflow-staging/Chart.yaml create mode 100644 charts/workflow/charts/workflow-staging/values.yaml create mode 100644 charts/workflow/requirements.yaml diff --git a/charts/workflow/Chart.yaml b/charts/workflow/Chart.yaml index 60565da6..15d87905 100644 --- a/charts/workflow/Chart.yaml +++ b/charts/workflow/Chart.yaml @@ -1,4 +1,4 @@ name: workflow version: v0.1.0 appVersion: v0.1.0 -description: Fabrikam Drone Workflow Service +description: Fabrikam Drone Delivery Workflow Service diff --git a/charts/workflow/charts/workflow-dev/Chart.yaml b/charts/workflow/charts/workflow-dev/Chart.yaml new file mode 100644 index 00000000..2a132cae --- /dev/null +++ b/charts/workflow/charts/workflow-dev/Chart.yaml @@ -0,0 +1,4 @@ +name: workflow-dev +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Workflow Service diff --git a/charts/workflow/charts/workflow-dev/values.yaml b/charts/workflow/charts/workflow-dev/values.yaml new file mode 100644 index 00000000..6f4a6a53 --- /dev/null +++ b/charts/workflow/charts/workflow-dev/values.yaml @@ -0,0 +1,10 @@ +# Dev values for workflow. +nameOverride: workflow +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new dev deploy" diff --git a/charts/workflow/charts/workflow-prod/Chart.yaml b/charts/workflow/charts/workflow-prod/Chart.yaml new file mode 100644 index 00000000..8de90dcc --- /dev/null +++ b/charts/workflow/charts/workflow-prod/Chart.yaml @@ -0,0 +1,4 @@ +name: workflow-prod +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Workflow Service diff --git a/charts/workflow/charts/workflow-prod/values.yaml b/charts/workflow/charts/workflow-prod/values.yaml new file mode 100644 index 00000000..9d027222 --- /dev/null +++ b/charts/workflow/charts/workflow-prod/values.yaml @@ -0,0 +1,11 @@ +# Production values for workflow. +nameOverride: workflow +exports: + data: + replicaCount: 3 + dockerregistrynamespace: "/prod" + image: + pullPolicy: IfNotPresent + telemetry: + level: "Error" + reason: "new prod deploy" diff --git a/charts/workflow/charts/workflow-qa/Chart.yaml b/charts/workflow/charts/workflow-qa/Chart.yaml new file mode 100644 index 00000000..52c2c9fc --- /dev/null +++ b/charts/workflow/charts/workflow-qa/Chart.yaml @@ -0,0 +1,4 @@ +name: workflow-qa +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Workflow Service diff --git a/charts/workflow/charts/workflow-qa/values.yaml b/charts/workflow/charts/workflow-qa/values.yaml new file mode 100644 index 00000000..78f674e6 --- /dev/null +++ b/charts/workflow/charts/workflow-qa/values.yaml @@ -0,0 +1,10 @@ +# QA values for workflow. +nameOverride: workflow +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new qa deploy" diff --git a/charts/workflow/charts/workflow-staging/Chart.yaml b/charts/workflow/charts/workflow-staging/Chart.yaml new file mode 100644 index 00000000..47f70e49 --- /dev/null +++ b/charts/workflow/charts/workflow-staging/Chart.yaml @@ -0,0 +1,4 @@ +name: workflow-staging +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Delivery Workflow Service diff --git a/charts/workflow/charts/workflow-staging/values.yaml b/charts/workflow/charts/workflow-staging/values.yaml new file mode 100644 index 00000000..92a59fca --- /dev/null +++ b/charts/workflow/charts/workflow-staging/values.yaml @@ -0,0 +1,10 @@ +# Staging values for workflow. +nameOverride: workflow +exports: + data: + replicaCount: 3 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new staging deploy" diff --git a/charts/workflow/requirements.yaml b/charts/workflow/requirements.yaml new file mode 100644 index 00000000..10232128 --- /dev/null +++ b/charts/workflow/requirements.yaml @@ -0,0 +1,32 @@ +dependencies: + - name: workflow-dev + repository: "file://charts/workflow-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data + + - name: workflow-prod + repository: "file://charts/workflow-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data + + - name: workflow-qa + repository: "file://charts/workflow-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + + - name: workflow-staging + repository: "file://charts/workflow-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index f42ed266..2a342aaa 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -44,7 +44,7 @@ spec: fsGroup: 1 containers: - name: fabrikam-workflow - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: workflow diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 9ab718f3..f21754a5 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -1,5 +1,7 @@ # Default values for workflow. -replicaCount: 3 +nameOverride: workflow +replicaCount: 1 +dockerregistrynamespace: dockerregistry: identity: clientid: @@ -35,3 +37,8 @@ resources: limits: cpu: 2300m memory: 140Mi +tags: + dev: false + prod: false + qa: false + staging: false diff --git a/deployment.md b/deployment.md index 912d963c..fd4075ee 100644 --- a/deployment.md +++ b/deployment.md @@ -307,6 +307,7 @@ helm install $HELM_CHARTS/workflow/ \ --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ --set keyvault.tenantid=$TENANT_ID \ --set reason="Initial deployment" \ + --set tags.prod=true \ --namespace backend \ --name workflow-v0.1.0 diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json index dc58d789..ce5b3ac1 100644 --- a/src/shipping/workflow/azure-pipelines-cd.json +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -227,7 +227,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -576,7 +576,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -925,7 +925,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1426,7 +1426,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From 36393eb88d11f393b0a5eaf9aaec4973b6982983 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 29 Apr 2019 10:17:10 -0300 Subject: [PATCH 159/246] [dronescheduler] add envs to helm chart --- .../charts/dronescheduler-dev/Chart.yaml | 5 +++ .../charts/dronescheduler-dev/values.yaml | 10 ++++++ .../charts/dronescheduler-prod/Chart.yaml | 4 +++ .../charts/dronescheduler-prod/values.yaml | 11 +++++++ .../charts/dronescheduler-qa/Chart.yaml | 4 +++ .../charts/dronescheduler-qa/values.yaml | 10 ++++++ .../charts/dronescheduler-staging/Chart.yaml | 4 +++ .../charts/dronescheduler-staging/values.yaml | 10 ++++++ charts/dronescheduler/requirements.yaml | 32 +++++++++++++++++++ .../templates/dronescheduler-deploy.yaml | 2 +- charts/dronescheduler/values.yaml | 6 ++++ deployment.md | 1 + .../dronescheduler/azure-pipelines-cd.json | 8 ++--- 13 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-dev/values.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-prod/values.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-qa/values.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml create mode 100644 charts/dronescheduler/charts/dronescheduler-staging/values.yaml create mode 100644 charts/dronescheduler/requirements.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml b/charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml new file mode 100644 index 00000000..d9adf749 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml @@ -0,0 +1,5 @@ +name: dronescheduler-dev +appName: dronescheduler +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Scheduler Service diff --git a/charts/dronescheduler/charts/dronescheduler-dev/values.yaml b/charts/dronescheduler/charts/dronescheduler-dev/values.yaml new file mode 100644 index 00000000..0b4c2579 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-dev/values.yaml @@ -0,0 +1,10 @@ +# Dev values for dronescheduler. +nameOverride: dronescheduler +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new dev deploy" diff --git a/charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml b/charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml new file mode 100644 index 00000000..e61d1d8f --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml @@ -0,0 +1,4 @@ +name: dronescheduler-prod +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Scheduler Service diff --git a/charts/dronescheduler/charts/dronescheduler-prod/values.yaml b/charts/dronescheduler/charts/dronescheduler-prod/values.yaml new file mode 100644 index 00000000..51f75da5 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-prod/values.yaml @@ -0,0 +1,11 @@ +# Production values for dronescheduler. +nameOverride: dronescheduler +exports: + data: + replicaCount: 3 + dockerregistrynamespace: "/prod" + image: + pullPolicy: IfNotPresent + telemetry: + level: "Error" + reason: "new prod deploy" diff --git a/charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml b/charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml new file mode 100644 index 00000000..1629a692 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml @@ -0,0 +1,4 @@ +name: dronescheduler-qa +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Scheduler Service diff --git a/charts/dronescheduler/charts/dronescheduler-qa/values.yaml b/charts/dronescheduler/charts/dronescheduler-qa/values.yaml new file mode 100644 index 00000000..83db9d26 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-qa/values.yaml @@ -0,0 +1,10 @@ +# QA values for dronescheduler. +nameOverride: dronescheduler +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new qa deploy" diff --git a/charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml b/charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml new file mode 100644 index 00000000..fb63d4ea --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml @@ -0,0 +1,4 @@ +name: dronescheduler-staging +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Scheduler Service diff --git a/charts/dronescheduler/charts/dronescheduler-staging/values.yaml b/charts/dronescheduler/charts/dronescheduler-staging/values.yaml new file mode 100644 index 00000000..d6deacb9 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-staging/values.yaml @@ -0,0 +1,10 @@ +# Staging values for dronescheduler. +nameOverride: dronescheduler +exports: + data: + replicaCount: 3 + image: + pullPolicy: Always + telemetry: + level: "Information" + reason: "new staging deploy" diff --git a/charts/dronescheduler/requirements.yaml b/charts/dronescheduler/requirements.yaml new file mode 100644 index 00000000..9fdc5a0b --- /dev/null +++ b/charts/dronescheduler/requirements.yaml @@ -0,0 +1,32 @@ +dependencies: + - name: dronescheduler-dev + repository: "file://charts/dronescheduler-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data + + - name: dronescheduler-prod + repository: "file://charts/dronescheduler-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data + + - name: dronescheduler-qa + repository: "file://charts/dronescheduler-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + + - name: dronescheduler-staging + repository: "file://charts/dronescheduler-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index 9040f6df..ccadbced 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -44,7 +44,7 @@ spec: fsGroup: 1 containers: - name: fabrikam-dronescheduler - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: KEY_VAULT_URI diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 44b1d731..c747574f 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -3,6 +3,7 @@ replicaCount: 3 identity: clientid: resourceid: +dockerregistrynamespace: dockerregistry: image: repository: @@ -20,3 +21,8 @@ resources: limits: cpu: 600m memory: 500Mi +tags: + dev: false + prod: false + qa: false + staging: false diff --git a/deployment.md b/deployment.md index fd4075ee..f6797f1d 100644 --- a/deployment.md +++ b/deployment.md @@ -427,6 +427,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --set reason="Initial deployment" \ + --set tags.prod=true \ --namespace backend \ --name dronescheduler-v0.1.0 diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index dedc8f7f..d5c9b6da 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -225,7 +225,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -581,7 +581,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -937,7 +937,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1437,7 +1437,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From d09a21707072e003b2a7132bf733bcc3af260bfa Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 29 Apr 2019 10:38:51 -0300 Subject: [PATCH 160/246] [ingestion] add envs to helm chart --- .../ingestion/charts/ingestion-dev/Chart.yaml | 4 +++ .../charts/ingestion-dev/values.yaml | 10 ++++++ .../charts/ingestion-prod/Chart.yaml | 4 +++ .../charts/ingestion-prod/values.yaml | 11 +++++++ .../ingestion/charts/ingestion-qa/Chart.yaml | 4 +++ .../ingestion/charts/ingestion-qa/values.yaml | 10 ++++++ .../charts/ingestion-staging/Chart.yaml | 4 +++ .../charts/ingestion-staging/values.yaml | 10 ++++++ charts/ingestion/requirements.yaml | 32 +++++++++++++++++++ .../ingestion/templates/ingestion-deploy.yaml | 2 +- charts/ingestion/values.yaml | 6 ++++ deployment.md | 1 + .../ingestion/azure-pipelines-cd.json | 8 ++--- 13 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 charts/ingestion/charts/ingestion-dev/Chart.yaml create mode 100644 charts/ingestion/charts/ingestion-dev/values.yaml create mode 100644 charts/ingestion/charts/ingestion-prod/Chart.yaml create mode 100644 charts/ingestion/charts/ingestion-prod/values.yaml create mode 100644 charts/ingestion/charts/ingestion-qa/Chart.yaml create mode 100644 charts/ingestion/charts/ingestion-qa/values.yaml create mode 100644 charts/ingestion/charts/ingestion-staging/Chart.yaml create mode 100644 charts/ingestion/charts/ingestion-staging/values.yaml create mode 100644 charts/ingestion/requirements.yaml diff --git a/charts/ingestion/charts/ingestion-dev/Chart.yaml b/charts/ingestion/charts/ingestion-dev/Chart.yaml new file mode 100644 index 00000000..30475d3f --- /dev/null +++ b/charts/ingestion/charts/ingestion-dev/Chart.yaml @@ -0,0 +1,4 @@ +name: ingestion-dev +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Ingestion Service diff --git a/charts/ingestion/charts/ingestion-dev/values.yaml b/charts/ingestion/charts/ingestion-dev/values.yaml new file mode 100644 index 00000000..8eb41d5d --- /dev/null +++ b/charts/ingestion/charts/ingestion-dev/values.yaml @@ -0,0 +1,10 @@ +# Dev values for ingestion. +nameOverride: ingestion +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "info" + reason: "new dev deploy" diff --git a/charts/ingestion/charts/ingestion-prod/Chart.yaml b/charts/ingestion/charts/ingestion-prod/Chart.yaml new file mode 100644 index 00000000..288b4ec7 --- /dev/null +++ b/charts/ingestion/charts/ingestion-prod/Chart.yaml @@ -0,0 +1,4 @@ +name: ingestion-prod +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Ingestion Service diff --git a/charts/ingestion/charts/ingestion-prod/values.yaml b/charts/ingestion/charts/ingestion-prod/values.yaml new file mode 100644 index 00000000..26c36037 --- /dev/null +++ b/charts/ingestion/charts/ingestion-prod/values.yaml @@ -0,0 +1,11 @@ +# Production values for ingestion. +nameOverride: ingestion +exports: + data: + replicaCount: 1 + dockerregistrynamespace: "/prod" + image: + pullPolicy: IfNotPresent + telemetry: + level: "error" + reason: "new prod deploy" diff --git a/charts/ingestion/charts/ingestion-qa/Chart.yaml b/charts/ingestion/charts/ingestion-qa/Chart.yaml new file mode 100644 index 00000000..0cdac0fa --- /dev/null +++ b/charts/ingestion/charts/ingestion-qa/Chart.yaml @@ -0,0 +1,4 @@ +name: ingestion-qa +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Ingestion Service diff --git a/charts/ingestion/charts/ingestion-qa/values.yaml b/charts/ingestion/charts/ingestion-qa/values.yaml new file mode 100644 index 00000000..bdcc0af6 --- /dev/null +++ b/charts/ingestion/charts/ingestion-qa/values.yaml @@ -0,0 +1,10 @@ +# QA values for ingestion. +nameOverride: ingestion +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "info" + reason: "new qa deploy" diff --git a/charts/ingestion/charts/ingestion-staging/Chart.yaml b/charts/ingestion/charts/ingestion-staging/Chart.yaml new file mode 100644 index 00000000..c5f33a04 --- /dev/null +++ b/charts/ingestion/charts/ingestion-staging/Chart.yaml @@ -0,0 +1,4 @@ +name: ingestion-staging +version: v0.1.0 +appVersion: v0.1.0 +description: Fabrikam Drone Ingestion Service diff --git a/charts/ingestion/charts/ingestion-staging/values.yaml b/charts/ingestion/charts/ingestion-staging/values.yaml new file mode 100644 index 00000000..640812ec --- /dev/null +++ b/charts/ingestion/charts/ingestion-staging/values.yaml @@ -0,0 +1,10 @@ +# Staging values for ingestion. +nameOverride: ingestion +exports: + data: + replicaCount: 1 + image: + pullPolicy: Always + telemetry: + level: "info" + reason: "new staging deploy" diff --git a/charts/ingestion/requirements.yaml b/charts/ingestion/requirements.yaml new file mode 100644 index 00000000..1abb4a62 --- /dev/null +++ b/charts/ingestion/requirements.yaml @@ -0,0 +1,32 @@ +dependencies: + - name: ingestion-dev + repository: "file://charts/ingestion-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data + + - name: ingestion-prod + repository: "file://charts/ingestion-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data + + - name: ingestion-qa + repository: "file://charts/ingestion-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + + - name: ingestion-staging + repository: "file://charts/ingestion-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index f14e04ba..8f85dc29 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -39,7 +39,7 @@ spec: spec: containers: - name: &ingestion-container_name fabrikam-ingestion - image: {{ .Values.dockerregistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} livenessProbe: httpGet: diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 62c23ccc..0770fad2 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -1,5 +1,6 @@ # Default values for ingestion. replicaCount: 1 +dockerregistrynamespace: dockerregistry: image: repository: @@ -15,3 +16,8 @@ resources: limits: cpu: 650m memory: 800Mi +tags: + dev: false + prod: false + qa: false + staging: false diff --git a/deployment.md b/deployment.md index f6797f1d..fc881215 100644 --- a/deployment.md +++ b/deployment.md @@ -376,6 +376,7 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ --set reason="Initial deployment" \ + --set tags.prod=true \ --namespace backend \ --name ingestion-v0.1.0 diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 88b2e507..dfd9ef95 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -239,7 +239,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -588,7 +588,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "replicaCount=1,image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -937,7 +937,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1430,7 +1430,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\"", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From c253e58762bbc4e485857eb258b0f5877eecb228 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 3 May 2019 19:12:23 -0300 Subject: [PATCH 161/246] make service names env agnostic. name overriding was not enough because release name --- charts/delivery/templates/_helpers.tpl | 8 ++++++++ charts/delivery/templates/delivery-service.yaml | 2 +- charts/dronescheduler/templates/_helpers.tpl | 8 ++++++++ .../dronescheduler/templates/dronescheduler-service.yaml | 2 +- charts/ingestion/templates/_helpers.tpl | 8 ++++++++ charts/ingestion/templates/ingestion-service.yaml | 2 +- charts/package/templates/_helpers.tpl | 8 ++++++++ charts/package/templates/package-service.yaml | 2 +- charts/workflow/templates/_helpers.tpl | 8 ++++++++ 9 files changed, 44 insertions(+), 4 deletions(-) diff --git a/charts/delivery/templates/_helpers.tpl b/charts/delivery/templates/_helpers.tpl index 44000f85..d02ec5d1 100644 --- a/charts/delivery/templates/_helpers.tpl +++ b/charts/delivery/templates/_helpers.tpl @@ -6,6 +6,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create app name with version. +*/}} +{{- define "delivery.versionappname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" $name .Chart.AppVersion | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 11d87d93..25548700 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -12,7 +12,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "delivery.fullname" . | replace "." "" }} + name: {{ include "delivery.versionappname" . | replace "." "" }} labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/dronescheduler/templates/_helpers.tpl b/charts/dronescheduler/templates/_helpers.tpl index 1cd08a99..caf6b7ac 100644 --- a/charts/dronescheduler/templates/_helpers.tpl +++ b/charts/dronescheduler/templates/_helpers.tpl @@ -6,6 +6,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create app name with version. +*/}} +{{- define "dronescheduler.versionappname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" $name .Chart.AppVersion | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index a25d1990..cb02903c 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -12,7 +12,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "dronescheduler.fullname" . | replace "." "" }} + name: {{ include "dronescheduler.versionappname" . | replace "." "" }} labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/ingestion/templates/_helpers.tpl b/charts/ingestion/templates/_helpers.tpl index fb15086a..353284bc 100644 --- a/charts/ingestion/templates/_helpers.tpl +++ b/charts/ingestion/templates/_helpers.tpl @@ -6,6 +6,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create app name with version. +*/}} +{{- define "ingestion.versionappname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" $name .Chart.AppVersion | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index 68df3612..966e9bd5 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -12,7 +12,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "ingestion.fullname" . | replace "." "" }} + name: {{ include "ingestion.versionappname" . | replace "." "" }} labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/package/templates/_helpers.tpl b/charts/package/templates/_helpers.tpl index 166a742e..c8a2b8a8 100644 --- a/charts/package/templates/_helpers.tpl +++ b/charts/package/templates/_helpers.tpl @@ -6,6 +6,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create app name with version. +*/}} +{{- define "package.versionappname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" $name .Chart.AppVersion | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index b6195a8f..072f741f 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -12,7 +12,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "package.fullname" . | replace "." "" }} + name: {{ include "package.versionappname" . | replace "." "" }} labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/workflow/templates/_helpers.tpl b/charts/workflow/templates/_helpers.tpl index 051ea827..aeefc2b1 100644 --- a/charts/workflow/templates/_helpers.tpl +++ b/charts/workflow/templates/_helpers.tpl @@ -6,6 +6,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create app name with version. +*/}} +{{- define "workflow.versionappname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" $name .Chart.AppVersion | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). From 3d3ba8d6f3a30853c82334ec7ce541774fade237 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 6 May 2019 09:53:35 -0300 Subject: [PATCH 162/246] Adress PR Comments: scope vars per environment --- deploymentCICD.md | 190 +++++++++++++----- src/shipping/delivery/azure-pipelines-cd.json | 87 ++++++-- .../dronescheduler/azure-pipelines-cd.json | 57 ++++-- .../ingestion/azure-pipelines-cd.json | 127 +++++++++--- src/shipping/package/azure-pipelines-cd.json | 46 ++++- src/shipping/workflow/azure-pipelines-cd.json | 118 ++++++++--- 6 files changed, 476 insertions(+), 149 deletions(-) diff --git a/deploymentCICD.md b/deploymentCICD.md index 28d03bfd..e5b2c552 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -142,21 +142,40 @@ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROU # add relese definition cat $DELIVERY_PATH/azure-pipelines-cd.json | \ - sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ - sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ - sed "s#COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ - sed "s#DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ sed "s#AZURE_DEVOPS_DELIVERY_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_DELIVERY_BUILD_ID#g" | \ sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ sed "s#AZURE_DEVOPS_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_DELIVERY_QUEUE_ID#g" | \ - sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + # development resources + sed "s#DEV_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ + sed "s#DEV_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ + sed "s#DEV_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + # qa resources + sed "s#QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ + sed "s#QA_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ + sed "s#QA_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + # staging resources + sed "s#STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ + sed "s#STAGING_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ + sed "s#STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + # production resources + sed "s#PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ + sed "s#PROD_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ + sed "s#PROD_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" \ > $DELIVERY_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -203,19 +222,29 @@ export COSMOSDB_COL_NAME=packages # add relese definition cat $PACKAGE_PATH/azure-pipelines-cd.json | \ - sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ - sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ - sed "s#COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ - sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ sed "s#AZURE_DEVOPS_PACKAGE_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_PACKAGE_BUILD_ID#g" | \ sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ sed "s#AZURE_DEVOPS_PACKAGE_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_PACKAGE_QUEUE_ID#g" | \ - sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ + # development resources + sed "s#DEV_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ + sed "s#DEV_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + # qa resources + sed "s#QA_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ + sed "s#QA_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + # staging resources + sed "s#STAGING_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ + sed "s#STAGING_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + # production resources + sed "s#PROD_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ + sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" \ > $PACKAGE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -261,22 +290,45 @@ export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROU # add relese definition cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ - sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ - sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ - sed "s#SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ - sed "s#TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ sed "s#AZURE_DEVOPS_WORKFLOW_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_WORKFLOW_BUILD_ID#g" | \ sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ sed "s#AZURE_DEVOPS_WORKFLOW_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_WORKFLOW_QUEUE_ID#g" | \ - sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ - > $WORKFLOW_PATH/azure-pipelines-cd-0.json + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#CLUSTER_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + # development resources + sed "s#DEV_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#DEV_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#DEV_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ + sed "s#DEV_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + # qa resources + sed "s#QA_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#QA_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#QA_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ + sed "s#QA_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + # staging resources + sed "s#STAGING_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#STAGING_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ + sed "s#STAGING_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + # production resources + sed "s#PROD_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#PROD_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#PROD_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ + sed "s#PROD_TENANT_ID_VAR_VAL#$TENANT_ID#g" \ + > $WORKFLOW_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ -d@${WORKFLOW_PATH}/azure-pipelines-cd-0.json \ @@ -345,24 +397,49 @@ export INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls.key) | tr '\n' # add relese definition cat $INGESTION_PATH/azure-pipelines-cd.json | \ - sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ - sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ - sed "s#INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ - sed "s#INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ - sed "s#INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ - sed "s#INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ sed "s#AZURE_DEVOPS_INGESTION_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_INGESTION_BUILD_ID#g" | \ sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ sed "s#AZURE_DEVOPS_INGESTION_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_INGESTION_QUEUE_ID#g" | \ - sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ + # development resources + sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#DEV_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ + sed "s#DEV_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + # qa resources + sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#QA_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ + sed "s#QA_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + # staging resources + sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#STAGING_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ + sed "s#STAGING_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + # production resources + sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#PROD_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ + sed "s#PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" \ > $INGESTION_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -409,19 +486,32 @@ export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP # add relese definition cat $DRONE_PATH/azure-pipelines-cd.json | \ - sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ - sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ sed "s#AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL#$AZURE_DEVOPS_SERVICE_CONN_ID#g" | \ sed "s#AZURE_DEVOPS_DRONE_BUILD_ID_VAR_VAL#$AZURE_DEVOPS_DRONE_BUILD_ID#g" | \ sed "s#AZURE_DEVOPS_REPOS_ID_VAR_VAL#$AZURE_DEVOPS_REPOS_ID#g" | \ sed "s#AZURE_DEVOPS_PROJECT_ID_VAR_VAL#$AZURE_DEVOPS_PROJECT_ID#g" | \ sed "s#AZURE_DEVOPS_DRONE_QUEUE_ID_VAR_VAL#$AZURE_DEVOPS_DRONE_QUEUE_ID#g" | \ - sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" \ + sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ + sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ + sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ + sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ + sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ + # development resources + sed "s#DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + # qa resources + sed "s#QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + # staging resources + sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + # production resources + sed "s#PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" \ > $DRONE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index ba3a65b7..01c2a351 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -16,21 +16,6 @@ }, "ACR_NAME": { "value": "ACR_NAME_VAR_VAL" - }, - "DELIVERY_PRINCIPAL_CLIENT_ID": { - "value": "DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" - }, - "DELIVERY_PRINCIPAL_RESOURCE_ID": { - "value": "DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" - }, - "DATABASE_NAME": { - "value": "DATABASE_NAME_VAR_VAL" - }, - "COLLECTION_NAME": { - "value": "COLLECTION_NAME_VAR_VAL" - }, - "DELIVERY_KEYVAULT_URI": { - "value": "DELIVERY_KEYVAULT_URI_VAR_VAL" } }, "variableGroups": [], @@ -42,7 +27,23 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DELIVERY_PRINCIPAL_CLIENT_ID": { + "value": "DEV_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_RESOURCE_ID": { + "value": "DEV_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "DATABASE_NAME": { + "value": "DEV_DATABASE_NAME_VAR_VAL" + }, + "COLLECTION_NAME": { + "value": "DEV_COLLECTION_NAME_VAR_VAL" + }, + "DELIVERY_KEYVAULT_URI": { + "value": "DEV_DELIVERY_KEYVAULT_URI_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -404,7 +405,23 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DELIVERY_PRINCIPAL_CLIENT_ID": { + "value": "QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_RESOURCE_ID": { + "value": "QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "DATABASE_NAME": { + "value": "QA_DATABASE_NAME_VAR_VAL" + }, + "COLLECTION_NAME": { + "value": "QA_COLLECTION_NAME_VAR_VAL" + }, + "DELIVERY_KEYVAULT_URI": { + "value": "QA_DELIVERY_KEYVAULT_URI_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -761,7 +778,23 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DELIVERY_PRINCIPAL_CLIENT_ID": { + "value": "STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_RESOURCE_ID": { + "value": "STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "DATABASE_NAME": { + "value": "STAGING_DATABASE_NAME_VAR_VAL" + }, + "COLLECTION_NAME": { + "value": "STAGING_COLLECTION_NAME_VAR_VAL" + }, + "DELIVERY_KEYVAULT_URI": { + "value": "STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1117,7 +1150,23 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "rank": 4, - "variables": {}, + "variables": { + "DELIVERY_PRINCIPAL_CLIENT_ID": { + "value": "PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DELIVERY_PRINCIPAL_RESOURCE_ID": { + "value": "PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "DATABASE_NAME": { + "value": "PROD_DATABASE_NAME_VAR_VAL" + }, + "COLLECTION_NAME": { + "value": "PROD_COLLECTION_NAME_VAR_VAL" + }, + "DELIVERY_KEYVAULT_URI": { + "value": "PROD_DELIVERY_KEYVAULT_URI_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index d5c9b6da..e3ff9c3d 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -10,15 +10,6 @@ "ACR_SERVER": { "value": "ACR_SERVER_VAR_VAL" }, - "DRONESCHEDULER_KEYVAULT_URI": { - "value": "DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" - }, - "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { - "value": "DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" - }, - "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { - "value": "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -36,7 +27,17 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DRONESCHEDULER_KEYVAULT_URI": { + "value": "DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { + "value": "DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { + "value": "DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -392,7 +393,17 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DRONESCHEDULER_KEYVAULT_URI": { + "value": "QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { + "value": "QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { + "value": "QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -748,7 +759,17 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DRONESCHEDULER_KEYVAULT_URI": { + "value": "STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { + "value": "STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { + "value": "STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1104,7 +1125,17 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "DRONESCHEDULER_KEYVAULT_URI": { + "value": "PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_CLIENT_ID": { + "value": "PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { + "value": "PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index dfd9ef95..93964472 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -14,29 +14,6 @@ "value": "AI_IKEY_VAR_VAL", "isSecret": true }, - "EXTERNAL_INGEST_FQDN": { - "value": "EXTERNAL_INGEST_FQDN_VAR_VAL" - }, - "INGESTION_ACCESS_KEY_VALUE": { - "value": "INGESTION_ACCESS_KEY_VALUE_VAR_VAL", - "isSecret": true - }, - "INGESTION_QUEUE_NAME": { - "value": "INGESTION_QUEUE_NAME_VAR_VAL" - }, - "INGESTION_QUEUE_NAMESPACE": { - "value": "INGESTION_QUEUE_NAMESPACE_VAR_VAL" - }, - "INGRESS_TLS_SECRET_CERT": { - "value": "INGRESS_TLS_SECRET_CERT_VAR_VAL" - }, - "INGRESS_TLS_SECRET_KEY": { - "value": "INGRESS_TLS_SECRET_KEY_VAR_VAL", - "isSecret": true - }, - "INGRESS_TLS_SECRET_NAME": { - "value": "INGRESS_TLS_SECRET_NAME_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -57,7 +34,31 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "EXTERNAL_INGEST_FQDN": { + "value": "DEV_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGESTION_ACCESS_KEY_VALUE": { + "value": "DEV_INGESTION_ACCESS_KEY_VALUE_VAR_VAL", + "isSecret": true + }, + "INGESTION_QUEUE_NAME": { + "value": "DEV_INGESTION_QUEUE_NAME_VAR_VAL" + }, + "INGESTION_QUEUE_NAMESPACE": { + "value": "DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -406,7 +407,31 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "EXTERNAL_INGEST_FQDN": { + "value": "QA_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGESTION_ACCESS_KEY_VALUE": { + "value": "QA_INGESTION_ACCESS_KEY_VALUE_VAR_VAL", + "isSecret": true + }, + "INGESTION_QUEUE_NAME": { + "value": "QA_INGESTION_QUEUE_NAME_VAR_VAL" + }, + "INGESTION_QUEUE_NAMESPACE": { + "value": "QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "QA_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "QA_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "QA_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -755,7 +780,31 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "EXTERNAL_INGEST_FQDN": { + "value": "STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGESTION_ACCESS_KEY_VALUE": { + "value": "STAGING_INGESTION_ACCESS_KEY_VALUE_VAR_VAL", + "isSecret": true + }, + "INGESTION_QUEUE_NAME": { + "value": "STAGING_INGESTION_QUEUE_NAME_VAR_VAL" + }, + "INGESTION_QUEUE_NAMESPACE": { + "value": "STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1104,7 +1153,31 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "EXTERNAL_INGEST_FQDN": { + "value": "PROD_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGESTION_ACCESS_KEY_VALUE": { + "value": "PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL", + "isSecret": true + }, + "INGESTION_QUEUE_NAME": { + "value": "PROD_INGESTION_QUEUE_NAME_VAR_VAL" + }, + "INGESTION_QUEUE_NAMESPACE": { + "value": "PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 06ec271e..ddc848f5 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -14,13 +14,6 @@ "ACR_NAME": { "value": "ACR_NAME_VAR_VAL" }, - "COSMOSDB_COL_NAME": { - "value": "COSMOSDB_COL_NAME_VAR_VAL" - }, - "COSMOSDB_CONNECTION": { - "value": "COSMOSDB_CONNECTION_VAR_VAL", - "isSecret": true - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -38,7 +31,15 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "COSMOSDB_COL_NAME": { + "value": "DEV_COSMOSDB_COL_NAME_VAR_VAL" + }, + "COSMOSDB_CONNECTION": { + "value": "DEV_COSMOSDB_CONNECTION_VAR_VAL", + "isSecret": true + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -399,6 +400,15 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, + "variables": { + "COSMOSDB_COL_NAME": { + "value": "QA_COSMOSDB_COL_NAME_VAR_VAL" + }, + "COSMOSDB_CONNECTION": { + "value": "QA_COSMOSDB_CONNECTION_VAR_VAL", + "isSecret": true + } + }, "variables": {}, "variableGroups": [], "preDeployApprovals": { @@ -755,7 +765,15 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "COSMOSDB_COL_NAME": { + "value": "STAGING_COSMOSDB_COL_NAME_VAR_VAL" + }, + "COSMOSDB_CONNECTION": { + "value": "STAGING_COSMOSDB_CONNECTION_VAR_VAL", + "isSecret": true + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1111,7 +1129,15 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "COSMOSDB_COL_NAME": { + "value": "PROD_COSMOSDB_COL_NAME_VAR_VAL" + }, + "COSMOSDB_CONNECTION": { + "value": "PROD_COSMOSDB_CONNECTION_VAR_VAL", + "isSecret": true + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json index ce5b3ac1..847d68b3 100644 --- a/src/shipping/workflow/azure-pipelines-cd.json +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -16,24 +16,6 @@ }, "REPO_NAME": { "value": "workflow" - }, - "RESOURCE_GROUP": { - "value": "RESOURCE_GROUP_VAR_VAL" - }, - "SUBSCRIPTION_ID": { - "value": "SUBSCRIPTION_ID_VAR_VAL" - }, - "TENANT_ID": { - "value": "TENANT_ID_VAR_VAL" - }, - "WORKFLOW_KEYVAULT_NAME": { - "value": "WORKFLOW_KEYVAULT_NAME_VAR_VAL" - }, - "WORKFLOW_PRINCIPAL_CLIENT_ID": { - "value": "WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" - }, - "WORKFLOW_PRINCIPAL_RESOURCE_ID": { - "value": "WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" } }, "variableGroups": [], @@ -45,7 +27,26 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "WORKFLOW_KEYVAULT_RESOURCE_GROUP": { + "value": "DEV_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL" + }, + "SUBSCRIPTION_ID": { + "value": "DEV_SUBSCRIPTION_ID_VAR_VAL" + }, + "TENANT_ID": { + "value": "DEV_TENANT_ID_VAR_VAL" + }, + "WORKFLOW_KEYVAULT_NAME": { + "value": "DEV_WORKFLOW_KEYVAULT_NAME_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_CLIENT_ID": { + "value": "DEV_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_RESOURCE_ID": { + "value": "DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -227,7 +228,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -318,7 +319,7 @@ }, "name": "azureResourceGroup", "label": "Resource group", - "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "defaultValue": "CLUSTER_RESOURCE_GROUP_VAR_VAL", "required": true, "type": "pickList", "helpMarkDown": "Select an Azure Resource Group.", @@ -394,7 +395,26 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "WORKFLOW_KEYVAULT_RESOURCE_GROUP": { + "value": "QA_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL" + }, + "SUBSCRIPTION_ID": { + "value": "QA_SUBSCRIPTION_ID_VAR_VAL" + }, + "TENANT_ID": { + "value": "QA_TENANT_ID_VAR_VAL" + }, + "WORKFLOW_KEYVAULT_NAME": { + "value": "QA_WORKFLOW_KEYVAULT_NAME_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_CLIENT_ID": { + "value": "QA_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_RESOURCE_ID": { + "value": "QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -576,7 +596,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -667,7 +687,7 @@ }, "name": "azureResourceGroup", "label": "Resource group", - "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "defaultValue": "CLUSTER_RESOURCE_GROUP_VAR_VAL", "required": true, "type": "pickList", "helpMarkDown": "Select an Azure Resource Group.", @@ -743,7 +763,26 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "WORKFLOW_KEYVAULT_RESOURCE_GROUP": { + "value": "STAGING_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL" + }, + "SUBSCRIPTION_ID": { + "value": "STAGING_SUBSCRIPTION_ID_VAR_VAL" + }, + "TENANT_ID": { + "value": "STAGING_TENANT_ID_VAR_VAL" + }, + "WORKFLOW_KEYVAULT_NAME": { + "value": "STAGING_WORKFLOW_KEYVAULT_NAME_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_CLIENT_ID": { + "value": "STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_RESOURCE_ID": { + "value": "STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -925,7 +964,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1016,7 +1055,7 @@ }, "name": "azureResourceGroup", "label": "Resource group", - "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "defaultValue": "CLUSTER_RESOURCE_GROUP_VAR_VAL", "required": true, "type": "pickList", "helpMarkDown": "Select an Azure Resource Group.", @@ -1092,7 +1131,26 @@ "owner": { "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, - "variables": {}, + "variables": { + "WORKFLOW_KEYVAULT_RESOURCE_GROUP": { + "value": "PROD_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL" + }, + "SUBSCRIPTION_ID": { + "value": "PROD_SUBSCRIPTION_ID_VAR_VAL" + }, + "TENANT_ID": { + "value": "PROD_TENANT_ID_VAR_VAL" + }, + "WORKFLOW_KEYVAULT_NAME": { + "value": "PROD_WORKFLOW_KEYVAULT_NAME_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_CLIENT_ID": { + "value": "PROD_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL" + }, + "WORKFLOW_PRINCIPAL_RESOURCE_ID": { + "value": "PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + } + }, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1426,7 +1484,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1522,7 +1580,7 @@ }, "name": "azureResourceGroup", "label": "Resource group", - "defaultValue": "RESOURCE_GROUP_VAR_VAL", + "defaultValue": "CLUSTER_RESOURCE_GROUP_VAR_VAL", "required": true, "type": "pickList", "helpMarkDown": "Select an Azure Resource Group.", From 31a1394e3afbcd30d737c0cf51f6e0a5332f40ac Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 14 May 2019 16:24:34 +0000 Subject: [PATCH 163/246] Merged PR 761: Expose deliveries status query Adds a public alias for the get delivery action. adds ingress for delivery rewriting to the public endpoint updates ingestion ingress to coexist with the delivery ingress updates routing attributes to follow recommendations https://docs.microsoft.com/en-us/aspnet/core/mvc/controllers/routing?view=aspnetcore-2.2#attribute-routing-with-httpverb-attributes adds routing tests Related work items: #9188 --- .../delivery/templates/delivery-ingress.yaml | 49 +++++ .../delivery-secret-ingress-tls.yaml | 14 ++ .../templates/ingestion-ingress.yaml | 10 +- deployment.md | 72 +++++-- .../DeliveriesControllerRoutingFixture.cs | 200 ++++++++++++++++++ ...DroneDelivery.DeliveryService.Tests.csproj | 5 + .../Controllers/DeliveriesController.cs | 39 ++-- 7 files changed, 344 insertions(+), 45 deletions(-) create mode 100644 charts/delivery/templates/delivery-ingress.yaml create mode 100644 charts/delivery/templates/delivery-secret-ingress-tls.yaml create mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs diff --git a/charts/delivery/templates/delivery-ingress.yaml b/charts/delivery/templates/delivery-ingress.yaml new file mode 100644 index 00000000..d90c0b85 --- /dev/null +++ b/charts/delivery/templates/delivery-ingress.yaml @@ -0,0 +1,49 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingress +################################################################################################### +{{- $svcversion := .Chart.AppVersion | replace "." "" }} +{{- $appversion := .Chart.AppVersion }} +{{- $defaultversionedpath := printf "/%s/" $appversion }} +{{- $relname := .Release.Name }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $relname }}-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries/public$1 +spec: + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.hosts }} + {{- if .tls }} + - hosts: + - {{ .name }} + secretName: {{ $relname }}-{{ .tlsSecretName }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + {{- if .path }} + - path: {{ printf "%s/%s/" .path $appversion }}api/deliveries(.*) + {{- else }} + - path: {{ $defaultversionedpath }}api/deliveries(.*) + {{- end }} + backend: + serviceName: "{{ .serviceName }}-{{ $svcversion }}" + servicePort: http + {{- if (eq $appversion "v0.1.0") }} + - path: {{ default "/" .path }}api/deliveries(.*) + backend: + serviceName: "{{ .serviceName }}" + servicePort: http + {{- end }} + {{ end }} diff --git a/charts/delivery/templates/delivery-secret-ingress-tls.yaml b/charts/delivery/templates/delivery-secret-ingress-tls.yaml new file mode 100644 index 00000000..3feb02a6 --- /dev/null +++ b/charts/delivery/templates/delivery-secret-ingress-tls.yaml @@ -0,0 +1,14 @@ +{{- if .Values.ingress.tls}} +{{- $relname := .Release.Name }} +{{- range .Values.ingress.tls.secrets }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ $relname }}-{{ .name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +--- +{{ end }} +{{ end }} diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index e28a80e6..4ad01fc1 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -8,14 +8,14 @@ ################################################################################################### {{- $svcversion := .Chart.AppVersion | replace "." "" }} {{- $appversion := .Chart.AppVersion }} -{{- $defaultversionedpath := printf "/%s" $appversion }} +{{- $defaultversionedpath := printf "/%s/" $appversion }} {{- $relname := .Release.Name }} apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $relname }}-ingress annotations: - nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/rewrite-target: /api/deliveryrequests$1 spec: {{- if .Values.ingress.tls }} tls: @@ -33,15 +33,15 @@ spec: http: paths: {{ if .path }} - - path: {{ printf "%s/%s" .path $appversion | quote }} + - path: {{ printf "%s/%s/" .path $appversion }}api/deliveryrequests(.*) {{ else }} - - path: {{ $defaultversionedpath | quote }} + - path: {{ $defaultversionedpath }}api/deliveryrequests(.*) {{ end }} backend: serviceName: "{{ .serviceName }}-{{ $svcversion }}" servicePort: http {{ if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }} + - path: {{ default "/" .path }}api/deliveryrequests(.*) backend: serviceName: "{{ .serviceName }}" servicePort: http diff --git a/deployment.md b/deployment.md index fc881215..a3ea390b 100644 --- a/deployment.md +++ b/deployment.md @@ -157,6 +157,25 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` +## Deploy the ingress controller + +```bash +# Deploy the ngnix ingress controller +helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true + +# Obtain the load balancer ip address and assign a domain name +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" +export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) + +# Create a self-signed certificate for TLS +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls.crt \ + -keyout ingestion-ingress-tls.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" +``` + ## Optional: Set up CI/CD with Azure DevOps Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). @@ -197,12 +216,20 @@ Deploy the Delivery service: # Extract pod identity outputs from deployment export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +export DELIVERY_INGRESS_TLS_SECRET_NAME=delivery-ingress-tls # Deploy the service helm install $HELM_CHARTS/delivery/ \ --set image.tag=0.1.0 \ --set image.repository=delivery \ --set dockerregistry=$ACR_SERVER \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=delivery \ + --set ingress.hosts[0].tls=true \ + --set ingress.hosts[0].tlsSecretName=$DELIVERY_INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].name=$DELIVERY_INGRESS_TLS_SECRET_NAME \ + --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ + --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ --set cosmosdb.id=$DATABASE_NAME \ @@ -342,21 +369,8 @@ docker push $ACR_SERVER/ingestion:0.1.0 Deploy the Ingestion service ```bash -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true - -# Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done -export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" -export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) -INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls - -# Create a self-signed certificate for TLS -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -out ingestion-ingress-tls.crt \ - -keyout ingestion-ingress-tls.key \ - -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" +# Set secreat name +export INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # Deploy service helm install $HELM_CHARTS/ingestion/ \ @@ -438,16 +452,36 @@ helm status dronescheduler-v0.1.0 ## Validate the application is running -You can send delivery requests to the ingestion service using the Swagger UI. +You can send delivery requests and check their statuses using curl. + +### Send a request -Use a web browser to navigate to `https://[EXTERNAL_INGEST_FQDN]/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST` and use the **Try it out** button to submit a delivery request. +Since the certificate used for TLS is self-signed, the request disables TLS validation using the '-k' option. ```bash -open "https://$EXTERNAL_INGEST_FQDN/swagger-ui.html#/ingestion45controller/scheduleDeliveryAsyncUsingPOST" +curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -i -d '{ + "confirmationRequired": "None", + "deadline": "", + "deliveryId": "mydelivery", + "dropOffLocation": "drop off", + "expedited": true, + "ownerId": "myowner", + "packageInfo": { + "packageId": "mypackage", + "size": "Small", + "tag": "mytag", + "weight": 10 + }, + "pickupLocation": "my pickup", + "pickupTime": "2019-05-08T20:00:00.000Z" + }' ``` -> We recommended putting an API Gateway in front of all public APIs. For convenience, the Ingestion service is directly exposed with a public IP address. +### Check the request status +```bash +curl "https://$EXTERNAL_INGEST_FQDN/api/deliveries/mydelivery" --header 'Accept: application/json' -k -i +``` ## Optional steps diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs new file mode 100644 index 00000000..a69b6ee6 --- /dev/null +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs @@ -0,0 +1,200 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Fabrikam.DroneDelivery.Common; +using Fabrikam.DroneDelivery.DeliveryService.Middlewares.Builder; +using Fabrikam.DroneDelivery.DeliveryService.Models; +using Fabrikam.DroneDelivery.DeliveryService.Services; +using Moq; + +namespace Fabrikam.DroneDelivery.DeliveryService.Tests +{ + [TestClass] + public class DeliveriesControllerRoutingFixture + { + private readonly TestServer _testServer; + private readonly Mock _deliveryRepositoryMock; + private readonly Mock _notifyMeRequestRepository; + private readonly Mock _notificationService; + private readonly Mock _deliveryTrackingRepository; + + public DeliveriesControllerRoutingFixture() + { + _deliveryRepositoryMock = new Mock(); + _notifyMeRequestRepository = new Mock(); + _notificationService = new Mock(); + _deliveryTrackingRepository = new Mock(); + + _testServer = + new TestServer( + new WebHostBuilder() + .Configure(builder => + { + builder.UseGlobalExceptionHandler(); + builder.UseMvc(); + }) + .ConfigureServices(builder => + { + builder.AddMvc(); + + builder.AddSingleton(_deliveryRepositoryMock.Object); + builder.AddSingleton(_notifyMeRequestRepository.Object); + builder.AddSingleton(_notificationService.Object); + builder.AddSingleton(_deliveryTrackingRepository.Object); + })); + } + + [TestCleanup] + public void TearDown() + { + _testServer?.Dispose(); + } + + [TestMethod] + public async Task GetDelivery_GetsResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + + _deliveryRepositoryMock + .Setup(r => r.GetAsync(deliveryId)) + .ReturnsAsync(new InternalDelivery(deliveryId, null, null, null, null, false, ConfirmationType.None, null)) + .Verifiable(); + + using (var client = _testServer.CreateClient()) + { + var response = await client.GetAsync($"http://localhost/api/deliveries/{deliveryId}"); + response.EnsureSuccessStatusCode(); + + var delivery = await response.Content.ReadAsAsync(); + + Assert.AreEqual(deliveryId, delivery.Id); + } + + _deliveryRepositoryMock.VerifyAll(); + } + + [TestMethod] + public async Task GetDeliveryThroughPublicRoute_GetsResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + + _deliveryRepositoryMock + .Setup(r => r.GetAsync(deliveryId)) + .ReturnsAsync(new InternalDelivery(deliveryId, null, null, null, null, false, ConfirmationType.None, null)) + .Verifiable(); + + using (var client = _testServer.CreateClient()) + { + var response = await client.GetAsync($"http://localhost/api/deliveries/public/{deliveryId}"); + response.EnsureSuccessStatusCode(); + + var delivery = await response.Content.ReadAsAsync(); + + Assert.AreEqual(deliveryId, delivery.Id); + } + + _deliveryRepositoryMock.VerifyAll(); + } + + [TestMethod] + public async Task PutDelivery_GetsResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + var delivery = new Delivery(deliveryId, new UserAccount("user", "accound"), null, null, null, false, ConfirmationType.None, null); + + _deliveryRepositoryMock + .Setup(r => r.CreateAsync(It.Is(d => d.Id == deliveryId))) + .Returns(Task.CompletedTask) + .Verifiable(); + _deliveryTrackingRepository + .Setup(r => r.AddAsync(It.Is(e => e.DeliveryId == deliveryId))) + .Returns(Task.CompletedTask) + .Verifiable(); + + using (var client = _testServer.CreateClient()) + { + var response = await client.PutAsJsonAsync($"http://localhost/api/deliveries/{deliveryId}", delivery); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); + + var createdDelivery = await response.Content.ReadAsAsync(); + Assert.AreEqual(deliveryId, delivery.Id); + } + + _deliveryRepositoryMock.VerifyAll(); + _deliveryTrackingRepository.VerifyAll(); + } + + [TestMethod] + public async Task PutDeliveryThroughPublicRoute_GetsNotFoundResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + var delivery = new Delivery(deliveryId, new UserAccount("user", "accound"), null, null, null, false, ConfirmationType.None, null); + + using (var client = _testServer.CreateClient()) + { + var response = await client.PutAsJsonAsync($"http://localhost/api/deliveries/public/{deliveryId}", delivery); + Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); + } + + _deliveryRepositoryMock.VerifyAll(); + _deliveryTrackingRepository.VerifyAll(); + } + + [TestMethod] + public async Task GetOwner_GetsResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + + _deliveryRepositoryMock + .Setup(r => r.GetAsync(deliveryId)) + .ReturnsAsync(new InternalDelivery(deliveryId, new UserAccount("user", "account"), null, null, null, false, ConfirmationType.None, null)) + .Verifiable(); + + using (var client = _testServer.CreateClient()) + { + var response = await client.GetAsync($"http://localhost/api/deliveries/{deliveryId}/owner"); + response.EnsureSuccessStatusCode(); + + var userAccount = await response.Content.ReadAsAsync(); + + Assert.AreEqual("user", userAccount.UserId); + } + + _deliveryRepositoryMock.VerifyAll(); + } + + [TestMethod] + public async Task GetStatus_GetsResponse() + { + var deliveryId = Guid.NewGuid().ToString(); + + _deliveryRepositoryMock + .Setup(r => r.GetAsync(deliveryId)) + .ReturnsAsync(new InternalDelivery(deliveryId, new UserAccount("user", "account"), null, null, null, false, ConfirmationType.None, null)) + .Verifiable(); + + using (var client = _testServer.CreateClient()) + { + var response = await client.GetAsync($"http://localhost/api/deliveries/{deliveryId}/status"); + response.EnsureSuccessStatusCode(); + + var status = await response.Content.ReadAsAsync(); + + Assert.AreEqual(DeliveryStage.Created, status.Stage); // exposes deserialization issue + } + + _deliveryRepositoryMock.VerifyAll(); + } + } +} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj index 863eb49d..9e2c8428 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj @@ -5,10 +5,15 @@ + + + + + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs index 43259035..8f8e9d2e 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -39,8 +38,8 @@ public DeliveriesController(IDeliveryRepository deliveryRepository, } // GET api/deliveries/5 - [Route("/api/[controller]/{id}", Name = "GetDelivery")] - [HttpGet] + [HttpGet("{id}", Name = "GetDelivery")] + [HttpGet("public/{id}")] [ProducesResponseType(typeof(Delivery), 200)] public async Task Get(string id) { @@ -48,7 +47,8 @@ public async Task Get(string id) var internalDelivery = await deliveryRepository.GetAsync(id); - if (internalDelivery == null) { + if (internalDelivery == null) + { logger.LogDebug("Delivery id: {Id} not found", id); return NotFound(); } @@ -57,8 +57,7 @@ public async Task Get(string id) } // GET api/deliveries/5/owner - [Route("/api/[controller]/{id}/owner")] - [HttpGet] + [HttpGet("{id}/owner")] [ProducesResponseType(typeof(UserAccount), 200)] public async Task GetOwner(string id) { @@ -75,13 +74,12 @@ public async Task GetOwner(string id) } // GET api/deliveries/5/status - [Route("/api/[controller]/{id}/status")] - [HttpGet] + [HttpGet("{id}/status")] [ProducesResponseType(typeof(DeliveryStatus), 200)] public async Task GetStatus(string id) { logger.LogInformation("In GetStatus action with id: {Id}", id); - + var delivery = await deliveryRepository.GetAsync(id); if (delivery == null) { @@ -89,7 +87,7 @@ public async Task GetStatus(string id) return NotFound(); } - var status = new DeliveryStatus(DeliveryStage.HeadedToDropoff, new Location(0,0,0), DateTime.Now.AddMinutes(10).ToString(), DateTime.Now.AddHours(1).ToString()); + var status = new DeliveryStatus(DeliveryStage.HeadedToDropoff, new Location(0, 0, 0), DateTime.Now.AddMinutes(10).ToString(), DateTime.Now.AddHours(1).ToString()); return Ok(status); } @@ -112,7 +110,7 @@ public async Task Put([FromBody]Delivery delivery, string id) var deliveryTrackingEvent = new DeliveryTrackingEvent { DeliveryId = delivery.Id, Stage = DeliveryStage.Created }; await deliveryTrackingRepository.AddAsync(deliveryTrackingEvent); - return CreatedAtRoute("GetDelivery", new { id= delivery.Id }, delivery); + return CreatedAtRoute("GetDelivery", new { id = delivery.Id }, delivery); } catch (DuplicateResourceException) { @@ -149,7 +147,7 @@ public async Task Patch(string id, [FromBody]RescheduledDelivery delivery.Expedited, delivery.ConfirmationRequired, delivery.DroneId); - + // Adds the delivery rescheduled status event var deliveryTrackingEvent = new DeliveryTrackingEvent { DeliveryId = id, Stage = DeliveryStage.Rescheduled }; await deliveryTrackingRepository.AddAsync(deliveryTrackingEvent); @@ -183,8 +181,7 @@ public async Task Delete(string id) } // POST api/deliveries/5/notifymerequests - [Route("/api/[controller]/{id}/notifymerequests")] - [HttpPost] + [HttpPost("{id}/notifymerequests")] public async Task NotifyMe(string id, [FromBody]NotifyMeRequest notifyMeRequest) { logger.LogInformation("In NotifyMe action with id: {Id} and notifyMeRequest: {@NotifyMeRequest}", id, notifyMeRequest.ToLogInfo()); @@ -218,8 +215,7 @@ public async Task NotifyMe(string id, [FromBody]NotifyMeRequest n /// /// // POST api/deliveries/5/confirmations - [Route("/api/[controller]/{id}/confirmations")] - [HttpPost] + [HttpPost("{id}/confirmations")] public async Task Confirm(string id, [FromBody]Confirmation confirmation) { logger.LogInformation("In Confirm action with id: {Id} and confirmation: {@Confirmation}", id, confirmation.ToLogInfo()); @@ -249,11 +245,12 @@ public async Task Confirm(string id, [FromBody]Confirmation confi }; // Adds the delivery complete status event - await deliveryTrackingRepository.AddAsync(new DeliveryTrackingEvent - { - DeliveryId = id, - Stage = DeliveryStage.Completed - }); + await deliveryTrackingRepository.AddAsync( + new DeliveryTrackingEvent + { + DeliveryId = id, + Stage = DeliveryStage.Completed + }); // sends notifications var notifyMeRequests = await notifyMeRequestRepository.GetAllByDeliveryIdAsync(id); From d4ea0c1326cfae81da8d571db6b0e9083f188342 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 11:43:20 -0300 Subject: [PATCH 164/246] [delivery] deploy experimental in production only --- .../templates/delivery-service.yaml | 31 ++++++++++++ .../delivery/templates/delivery-service.yaml | 2 +- src/shipping/delivery/azure-pipelines-cd.json | 48 ------------------- 3 files changed, 32 insertions(+), 49 deletions(-) create mode 100644 charts/delivery/charts/delivery-prod/templates/delivery-service.yaml diff --git a/charts/delivery/charts/delivery-prod/templates/delivery-service.yaml b/charts/delivery/charts/delivery-prod/templates/delivery-service.yaml new file mode 100644 index 00000000..46595d00 --- /dev/null +++ b/charts/delivery/charts/delivery-prod/templates/delivery-service.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery service +################################################################################################### +{{- $appname := include "delivery.name" . -}} +{{- $chart := include "delivery.chart" . -}} +{{- $instancename := .Release.Name }} +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +apiVersion: v1 +kind: Service +metadata: + name: delivery-experimental + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 25548700..380bf205 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -30,7 +30,7 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ if (eq .Release.Name "delivery-v0.1.0") }} -{{- range tuple "delivery" "delivery-experimental" }} +{{- range tuple "delivery" }} {{- $svcname := . }} --- # the following object is meant ot be created first time only. diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index 01c2a351..e048e4fe 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -1518,54 +1518,6 @@ "privatekey": "", "tillernamespace": "" } - }, - { - "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set green", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", - "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service delivery-experimental app.kubernetes.io/name=delivery,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" - } } ] }, From 0d1156fe597929c311a106d4833b8100788304d7 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 12:58:06 -0300 Subject: [PATCH 165/246] [dronescheduler] deploy experimental in production only --- .../templates/dronescheduler-service.yaml | 31 ++++++++++++ .../templates/dronescheduler-service.yaml | 2 +- .../dronescheduler/azure-pipelines-cd.json | 48 ------------------- 3 files changed, 32 insertions(+), 49 deletions(-) create mode 100644 charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml b/charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml new file mode 100644 index 00000000..48c10e93 --- /dev/null +++ b/charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler service experimental +################################################################################################### +{{- $appname := include "dronescheduler.name" . -}} +{{- $chart := include "dronescheduler.chart" . -}} +{{- $instancename := .Release.Name }} +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +apiVersion: v1 +kind: Service +metadata: + name: dronescheduler-experimental + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index cb02903c..ccba4bab 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -30,7 +30,7 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ if (eq .Release.Name "dronescheduler-v0.1.0") }} -{{- range tuple "dronescheduler" "dronescheduler-experimental" }} +{{- range tuple "dronescheduler" }} {{- $svcname := . }} --- # the following object is meant ot be created first time only. diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index e3ff9c3d..33a2c028 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -1487,54 +1487,6 @@ "privatekey": "", "tillernamespace": "" } - }, - { - "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set green", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", - "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service dronescheduler-experimental app.kubernetes.io/name=dronescheduler,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" - } } ] }, From 000c2d86e6e5c8019c2ba6d75ca999ab8bd6f543 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 13:09:19 -0300 Subject: [PATCH 166/246] [ingestion] deploy experimental in production only --- .../templates/ingestion-service.yaml | 31 ++++++++++++ .../templates/ingestion-service.yaml | 2 +- .../ingestion/azure-pipelines-cd.json | 48 ------------------- 3 files changed, 32 insertions(+), 49 deletions(-) create mode 100644 charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml diff --git a/charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml b/charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml new file mode 100644 index 00000000..e03157e8 --- /dev/null +++ b/charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingestion service experimental +################################################################################################### +{{- $appname := include "ingestion.name" . -}} +{{- $chart := include "ingestion.chart" . -}} +{{- $instancename := .Release.Name }} +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +apiVersion: v1 +kind: Service +metadata: + name: ingestion-experimental + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index 966e9bd5..615d2017 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -30,7 +30,7 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ if (eq .Release.Name "ingestion-v0.1.0") }} -{{- range tuple "ingestion" "ingestion-experimental" }} +{{- range tuple "ingestion" }} {{- $svcname := . }} --- # the following object is meant ot be created first time only. diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 93964472..9f364818 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -1522,54 +1522,6 @@ "privatekey": "", "tillernamespace": "" } - }, - { - "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set green", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", - "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service ingestion-experimental app.kubernetes.io/name=ingestion,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" - } } ] }, From 6fb529d152623d359f97106b2d123f093c647849 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Tue, 7 May 2019 13:14:57 -0300 Subject: [PATCH 167/246] [package] deploy experimental in production only --- .../templates/package-service.yaml | 31 ++++++++++++ charts/package/templates/package-service.yaml | 2 +- src/shipping/package/azure-pipelines-cd.json | 48 ------------------- 3 files changed, 32 insertions(+), 49 deletions(-) create mode 100644 charts/package/charts/package-prod/templates/package-service.yaml diff --git a/charts/package/charts/package-prod/templates/package-service.yaml b/charts/package/charts/package-prod/templates/package-service.yaml new file mode 100644 index 00000000..e2f8ee4d --- /dev/null +++ b/charts/package/charts/package-prod/templates/package-service.yaml @@ -0,0 +1,31 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# package service experimental +################################################################################################### +{{- $appname := include "package.name" . -}} +{{- $chart := include "package.chart" . -}} +{{- $instancename := .Release.Name }} +# the following object is meant ot be created first time only. +# its configuration will be later managed by CI/CD +apiVersion: v1 +kind: Service +metadata: + name: package-experimental + labels: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/managed-by: azuredeveops + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ $chart }} +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/name: {{ $appname }} + app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index 072f741f..38b13369 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -30,7 +30,7 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ if (eq .Release.Name "package-v0.1.0") }} -{{- range tuple "package" "package-experimental" }} +{{- range tuple "package" }} {{- $svcname := . }} --- # the following object is meant ot be created first time only. diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index ddc848f5..c3814cb3 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -1490,54 +1490,6 @@ "privatekey": "", "tillernamespace": "" } - }, - { - "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set green", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", - "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service package-experimental app.kubernetes.io/name=package,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" - } } ] }, From f3bd10ffe17b9d904bdb454e6e00ad72bc5f81d8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 16 May 2019 21:53:27 +0000 Subject: [PATCH 168/246] Merged PR 771: upgrade/install default service provided it is considered current upgrade/install default service provided it is considered current before it configured the service version agnostic provided it is v0.1.0, now it will be possible to arbitrary promote any release as the "current" version. It enables the following: - promotion-like API management approach - remove previous limited configuration approach release-name-based - blue-green turns into a more helm approach enabling other scenarios (i.e. rollback to previous current version using helm) --- .../delivery/templates/delivery-service.yaml | 9 +-- charts/delivery/values.yaml | 1 + .../templates/dronescheduler-service.yaml | 9 +-- charts/dronescheduler/values.yaml | 1 + .../templates/ingestion-service.yaml | 9 +-- charts/ingestion/values.yaml | 1 + charts/package/templates/package-service.yaml | 9 +-- charts/package/values.yaml | 1 + deployment.md | 4 ++ src/shipping/delivery/azure-pipelines-cd.json | 68 +++++++++---------- .../dronescheduler/azure-pipelines-cd.json | 68 +++++++++---------- .../ingestion/azure-pipelines-cd.json | 68 +++++++++---------- src/shipping/package/azure-pipelines-cd.json | 68 +++++++++---------- 13 files changed, 148 insertions(+), 168 deletions(-) diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 380bf205..4dd39004 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -29,16 +29,12 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} -{{ if (eq .Release.Name "delivery-v0.1.0") }} -{{- range tuple "delivery" }} -{{- $svcname := . }} +{{ if .Values.current }} --- -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD apiVersion: v1 kind: Service metadata: - name: {{ $svcname }} + name: delivery labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -54,4 +50,3 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ end }} -{{ end }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index 02ffbb26..2da6c55b 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -30,3 +30,4 @@ tags: prod: false qa: false staging: false +current: false diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index ccba4bab..bb8c149b 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -29,16 +29,12 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} -{{ if (eq .Release.Name "dronescheduler-v0.1.0") }} -{{- range tuple "dronescheduler" }} -{{- $svcname := . }} +{{ if .Values.current }} --- -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD apiVersion: v1 kind: Service metadata: - name: {{ $svcname }} + name: dronescheduler labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -54,4 +50,3 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ end }} -{{ end }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index c747574f..02b3f73c 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -26,3 +26,4 @@ tags: prod: false qa: false staging: false +current: false diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index 615d2017..e0e7f32b 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -29,16 +29,12 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} -{{ if (eq .Release.Name "ingestion-v0.1.0") }} -{{- range tuple "ingestion" }} -{{- $svcname := . }} +{{ if .Values.current }} --- -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD apiVersion: v1 kind: Service metadata: - name: {{ $svcname }} + name: ingestion labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -54,4 +50,3 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ end }} -{{ end }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 0770fad2..ed657d9c 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -21,3 +21,4 @@ tags: prod: false qa: false staging: false +current: false diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index 38b13369..998b58c7 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -29,16 +29,12 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} -{{ if (eq .Release.Name "package-v0.1.0") }} -{{- range tuple "package" }} -{{- $svcname := . }} +{{ if .Values.current }} --- -# the following object is meant ot be created first time only. -# its configuration will be later managed by CI/CD apiVersion: v1 kind: Service metadata: - name: {{ $svcname }} + name: package labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -54,4 +50,3 @@ spec: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} {{ end }} -{{ end }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 39f6f777..239908f0 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -24,3 +24,4 @@ tags: prod: false qa: false staging: false +current: false diff --git a/deployment.md b/deployment.md index a3ea390b..d9dea15c 100644 --- a/deployment.md +++ b/deployment.md @@ -237,6 +237,7 @@ helm install $HELM_CHARTS/delivery/ \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ --set reason="Initial deployment" \ --set tags.prod=true \ + --set current=true \ --namespace backend \ --name delivery-v0.1.0 @@ -283,6 +284,7 @@ helm install $HELM_CHARTS/package/ \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ --set tags.prod=true \ + --set current=true \ --namespace backend \ --name package-v0.1.0 @@ -391,6 +393,7 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ --set reason="Initial deployment" \ --set tags.prod=true \ + --set current=true \ --namespace backend \ --name ingestion-v0.1.0 @@ -443,6 +446,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --set reason="Initial deployment" \ --set tags.prod=true \ + --set current=true \ --namespace backend \ --name dronescheduler-v0.1.0 diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index e048e4fe..b16d88a2 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -1684,50 +1684,48 @@ }, { "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set blue", + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade (mark as current blue)", "refName": "", "enabled": true, "alwaysRun": false, "continueOnError": false, "timeoutInMinutes": 0, - "definitionType": "task", + "definitionType": null, "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service delivery app.kubernetes.io/name=delivery,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" } } ] diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index 33a2c028..db331a07 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -1653,50 +1653,48 @@ }, { "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set blue", + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade (mark as current blue)", "refName": "", "enabled": true, "alwaysRun": false, "continueOnError": false, "timeoutInMinutes": 0, - "definitionType": "task", + "definitionType": null, "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service dronescheduler app.kubernetes.io/name=dronescheduler,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" } } ] diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 9f364818..7f312bbb 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -1681,50 +1681,48 @@ }, { "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set blue", + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade (mark as current blue)", "refName": "", "enabled": true, "alwaysRun": false, "continueOnError": false, "timeoutInMinutes": 0, - "definitionType": "task", + "definitionType": null, "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service ingestion app.kubernetes.io/name=ingestion,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" } } ] diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index c3814cb3..2adb897a 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -1656,50 +1656,48 @@ }, { "environment": {}, - "taskId": "cbc316a2-586f-4def-be79-488a1f503564", - "version": "1.*", - "name": "kubectl set blue", + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm upgrade (set current as blue)", "refName": "", "enabled": true, "alwaysRun": false, "continueOnError": false, "timeoutInMinutes": 0, - "definitionType": "task", + "definitionType": null, "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "connectionType": "Azure Resource Manager", - "kubernetesServiceEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureResourceGroup": "RESOURCE_GROUP_VAR_VAL", - "kubernetesCluster": "CLUSTER_NAME_VAR_VAL", - "useClusterAdmin": "false", + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", "namespace": "backend", - "command": "set", - "useConfigurationFile": "false", - "configurationType": "configuration", - "configuration": "", - "inline": "", - "arguments": "selector service package app.kubernetes.io/name=package,app.kubernetes.io/instance=$(REPO_NAME)-$(Build.SourceBranchName)", - "secretType": "dockerRegistry", - "secretArguments": "", - "containerRegistryType": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpointForSecrets": "", - "azureContainerRegistry": "", - "secretName": "", - "forceUpdate": "true", - "configMapName": "", - "forceUpdateConfigMap": "false", - "useConfigMapFile": "false", - "configMapFile": "", - "configMapArguments": "", - "versionOrLocation": "version", - "versionSpec": "1.12.4", - "checkLatest": "false", - "specifyLocation": "", - "cwd": "$(System.DefaultWorkingDirectory)", - "outputFormat": "json" + "command": "upgrade", + "chartType": "Name", + "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartPath": "", + "version": "", + "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true,current=true", + "valueFile": "", + "destination": "", + "canaryimage": "false", + "upgradetiller": "false", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "true", + "waitForExecution": "false", + "arguments": "--version $(Build.SourceBranchName)", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" } } ] From e75d59187d99a1232344835752eae64e04ad6c31 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 24 May 2019 13:13:02 +0000 Subject: [PATCH 169/246] Merged PR 784: deploy isolated ngnix per environment and/or namespace deploy isolated ngnix per environment and/or namespace - solve problem ngnix watching multiple namespace issue - strealine instructions for cicd steps - update cd - one fqdn per env - add missing header additional best practices achieved as a consequence: - isolate production ingress - add redundancy in prod and staging solved: #9166 Related work items: #9166 --- .../charts/ingestion-prod/values.yaml | 3 + .../charts/ingestion-staging/values.yaml | 3 + charts/ingestion/requirements.lock | 6 ++ charts/ingestion/requirements.yaml | 61 ++++++++-------- .../ingestion-secret-ingress-tls.yaml | 5 ++ charts/ingestion/values.yaml | 8 +++ deployment.md | 45 ++++++------ deploymentCICD.md | 69 ++++++++++--------- .../ingestion/azure-pipelines-cd.json | 22 ++++-- 9 files changed, 134 insertions(+), 88 deletions(-) create mode 100644 charts/ingestion/requirements.lock diff --git a/charts/ingestion/charts/ingestion-prod/values.yaml b/charts/ingestion/charts/ingestion-prod/values.yaml index 26c36037..431bb060 100644 --- a/charts/ingestion/charts/ingestion-prod/values.yaml +++ b/charts/ingestion/charts/ingestion-prod/values.yaml @@ -9,3 +9,6 @@ exports: telemetry: level: "error" reason: "new prod deploy" + nginx-ingress: + controller: + replicaCount: 2 diff --git a/charts/ingestion/charts/ingestion-staging/values.yaml b/charts/ingestion/charts/ingestion-staging/values.yaml index 640812ec..c42d1d88 100644 --- a/charts/ingestion/charts/ingestion-staging/values.yaml +++ b/charts/ingestion/charts/ingestion-staging/values.yaml @@ -8,3 +8,6 @@ exports: telemetry: level: "info" reason: "new staging deploy" + nginx-ingress: + controller: + replicaCount: 2 diff --git a/charts/ingestion/requirements.lock b/charts/ingestion/requirements.lock new file mode 100644 index 00000000..4de12479 --- /dev/null +++ b/charts/ingestion/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: nginx-ingress + repository: https://kubernetes-charts.storage.googleapis.com + version: 1.6.5 +digest: sha256:ac4894123e00ec6c55fa3cd2a659a66b13882fbe825706afb4613a2a8f507bcf +generated: 2019-05-17T12:50:32.882555938-03:00 diff --git a/charts/ingestion/requirements.yaml b/charts/ingestion/requirements.yaml index 1abb4a62..2b7b48a2 100644 --- a/charts/ingestion/requirements.yaml +++ b/charts/ingestion/requirements.yaml @@ -1,32 +1,37 @@ dependencies: - - name: ingestion-dev - repository: "file://charts/ingestion-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data +- name: nginx-ingress + repository: https://kubernetes-charts.storage.googleapis.com + version: ">=1.6.0" + condition: nginx-ingress.enabled - - name: ingestion-prod - repository: "file://charts/ingestion-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data +- name: ingestion-dev + repository: "file://charts/ingestion-dev" + version: ">= 0.0.1" + tags: + - dev + import-values: + - data - - name: ingestion-qa - repository: "file://charts/ingestion-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data +- name: ingestion-prod + repository: "file://charts/ingestion-prod" + version: ">= 0.0.1" + tags: + - prod + import-values: + - data - - name: ingestion-staging - repository: "file://charts/ingestion-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data +- name: ingestion-qa + repository: "file://charts/ingestion-qa" + version: ">= 0.0.1" + tags: + - qa + import-values: + - data + +- name: ingestion-staging + repository: "file://charts/ingestion-staging" + version: ">= 0.0.1" + tags: + - staging + import-values: + - data diff --git a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml index 3feb02a6..6312aa6a 100644 --- a/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml +++ b/charts/ingestion/templates/ingestion-secret-ingress-tls.yaml @@ -1,3 +1,8 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + {{- if .Values.ingress.tls}} {{- $relname := .Release.Name }} {{- range .Values.ingress.tls.secrets }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index ed657d9c..fb4468d4 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -16,6 +16,14 @@ resources: limits: cpu: 650m memory: 800Mi +nginx-ingress: + fullnameOverride: nginx-ingress + rbac: + create: true + controller: + scope: + enabled: true + enabled: false tags: dev: false prod: false diff --git a/deployment.md b/deployment.md index d9dea15c..518003ee 100644 --- a/deployment.md +++ b/deployment.md @@ -157,6 +157,12 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` +## Optional: Set up CI/CD with Azure DevOps + +Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). + +> Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application as follows. + ## Deploy the ingress controller ```bash @@ -175,13 +181,6 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ingestion-ingress-tls.key \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` - -## Optional: Set up CI/CD with Azure DevOps - -Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). - -> Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application as follows. - ## Deploy the Delivery service Extract resource details from deployment @@ -463,22 +462,22 @@ You can send delivery requests and check their statuses using curl. Since the certificate used for TLS is self-signed, the request disables TLS validation using the '-k' option. ```bash -curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -i -d '{ - "confirmationRequired": "None", - "deadline": "", - "deliveryId": "mydelivery", - "dropOffLocation": "drop off", - "expedited": true, - "ownerId": "myowner", - "packageInfo": { - "packageId": "mypackage", - "size": "Small", - "tag": "mytag", - "weight": 10 - }, - "pickupLocation": "my pickup", - "pickupTime": "2019-05-08T20:00:00.000Z" - }' +curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -i -d '{ + "confirmationRequired": "None", + "deadline": "", + "deliveryId": "mydelivery", + "dropOffLocation": "drop off", + "expedited": true, + "ownerId": "myowner", + "packageInfo": { + "packageId": "mypackage", + "size": "Small", + "tag": "mytag", + "weight": 10 + }, + "pickupLocation": "my pickup", + "pickupTime": "2019-05-08T20:00:00.000Z" + }' ``` ### Check the request status diff --git a/deploymentCICD.md b/deploymentCICD.md index e5b2c552..7d72d041 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -89,7 +89,25 @@ export AZURE_DEVOPS_USER_ID=$(az devops user show --user ${AZURE_DEVEOPS_USER} - ### Build pipelines pre-requisites -``` +```bash +# Create a self-signed certificate for TLS and public ip addresses +export RESOURCE_GROUP_NODE=$(az aks show -g $RESOURCE_GROUP -n $CLUSTER_NAME --query "nodeResourceGroup" -o tsv) && \ +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" + +for env in dev qa staging prod;do +ENV=${env^^} +az network public-ip create --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --dns-name ${EXTERNAL_INGEST_DNS_NAME}-${env} --allocation-method static -g $RESOURCE_GROUP_NODE +EXTERNAL_INGEST_FQDN=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "dnsSettings.fqdn" -g $RESOURCE_GROUP_NODE --output tsv) +export ${ENV}_EXTERNAL_INGEST_FQDN=${EXTERNAL_INGEST_FQDN} +export ${ENV}_INGRESS_LOAD_BALANCER_IP=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "ipAddress" -g $RESOURCE_GROUP_NODE --output tsv) +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -out ingestion-ingress-tls-${env}.crt \ + -keyout ingestion-ingress-tls-${env}.key \ + -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" +export "${ENV}_INGRESS_TLS_SECRET_CERT=$(echo $(cat ingestion-ingress-tls-${env}.crt) | tr '\n' "\\n")" +export "${ENV}_INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls-${env}.key) | tr '\n' "\\n")" +done + # export app paths export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery && \ export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package && \ @@ -354,24 +372,6 @@ helm status workflow-v0.1.0 Ingestion pre-requisites -```bash -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true - -# Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done -export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" -export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) -INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls - -# Create a self-signed certificate for TLS -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -out ingestion-ingress-tls.crt \ - -keyout ingestion-ingress-tls.key \ - -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" -``` - Create build and release pipeline definitions ``` # add build definitions @@ -392,8 +392,7 @@ export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP - export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) && \ export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) && \ export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) && \ -export INGRESS_TLS_SECRET_CERT=$(echo $(cat ingestion-ingress-tls.crt) | tr '\n' "\\n") && \ -export INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls.key) | tr '\n' "\\n") +export INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # add relese definition cat $INGESTION_PATH/azure-pipelines-cd.json | \ @@ -412,34 +411,38 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#DEV_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ sed "s#DEV_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$DEV_INGRESS_LOAD_BALANCER_IP#g" | \ + sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$DEV_EXTERNAL_INGEST_FQDN#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$DEV_INGRESS_TLS_SECRET_CERT#g" | \ # qa resources sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#QA_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ sed "s#QA_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$QA_INGRESS_LOAD_BALANCER_IP#g" | \ + sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$QA_EXTERNAL_INGEST_FQDN#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$QA_INGRESS_TLS_SECRET_CERT#g" | \ # staging resources sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#STAGING_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ sed "s#STAGING_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$STAGING_INGRESS_LOAD_BALANCER_IP#g" | \ + sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$STAGING_EXTERNAL_INGEST_FQDN#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_CERT#g" | \ # production resources sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#PROD_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ sed "s#PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$EXTERNAL_INGEST_FQDN#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$PROD_INGRESS_LOAD_BALANCER_IP#g" | \ + sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$INGRESS_TLS_SECRET_CERT#g" \ + sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" \ > $INGESTION_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 7f312bbb..e6252f9d 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -35,6 +35,9 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "INGRESS_LOAD_BALANCER_IP": { + "value": "DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL" + }, "EXTERNAL_INGEST_FQDN": { "value": "DEV_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -240,7 +243,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -408,6 +411,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "INGRESS_LOAD_BALANCER_IP": { + "value": "QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL" + }, + "EXTERNAL_INGEST_FQDN": { "value": "QA_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -613,7 +620,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -781,6 +788,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "INGRESS_LOAD_BALANCER_IP": { + "value": "STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL" + }, + "EXTERNAL_INGEST_FQDN": { "value": "STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -986,7 +997,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1154,6 +1165,9 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "INGRESS_LOAD_BALANCER_IP": { + "value": "PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL" + }, "EXTERNAL_INGEST_FQDN": { "value": "PROD_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -1503,7 +1517,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From c9fc7a00b98391294e3ddda76fd5f8880b6640ee Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 4 Jun 2019 15:46:53 +0000 Subject: [PATCH 170/246] Merged PR 787: provision infrastructure for dev, qa, staging and production provision infrastructure for dev, qa, staging and production now resources are being provisioned targeting multiple environment with a new naming convention revealing interesting views from the single resource group current approach: - arm(s) are now environment aware - normalize resource names - remove kv name dep with rg - change manual instructions to default dev - remove unused front-end namespace - discover orchestrator version - change redis tier for dev | | dev | qa | staging | production | |-------------------------|:---------:|:---------:|:---------:|:----------:| | aks_std_d2v2_x3 | shared | shared | shared | shared | | acr_basic | shared | shared | shared | shared | | redis_basic_c0 | dedicated | dedicated | dedicated | dedicated | | cosmosdb_1k_ru | dedicated | dedicated | dedicated | dedicated | | servicebus_std_1gig | dedicated | dedicated | dedicated | dedicated | | mongo_2.6k_ru | dedicated | dedicated | dedicated | dedicated | | stoaccount_std_lrs_v1 | dedicated | dedicated | dedicated | dedicated | | appinsights | dedicated | dedicated | dedicated | dedicated | | keyvault_std_x3 | dedicated | dedicated | dedicated | dedicated | solved: #9198 Related work items: #9198 --- azuredeploy-identities.json | 96 ++++- azuredeploy.json | 406 +++++++++++++----- deployment.md | 101 +++-- deploymentCICD.md | 305 +++++++++---- .../ingestion/azure-pipelines-cd.json | 22 +- src/shipping/package/azure-pipelines-cd.json | 20 +- 6 files changed, 668 insertions(+), 282 deletions(-) diff --git a/azuredeploy-identities.json b/azuredeploy-identities.json index 9e12ac7f..1b606431 100644 --- a/azuredeploy-identities.json +++ b/azuredeploy-identities.json @@ -1,79 +1,131 @@ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "type": "string", + "defaultValue": "dev", + "allowedValues": [ + "dev", + "qa", + "staging", + "prod" + ] + } + }, "variables": { - "deliveryIdName": "delivery", - "workflowIdName": "workflow", - "droneSchedulerIdName": "dronescheduler" + "environmentSettings": { + "dev": { + "deliveryIdName": "dev-d", + "workflowIdName": "dev-wf", + "droneSchedulerIdName": "dev-ds" + }, + "qa": { + "deliveryIdName": "qa-d", + "workflowIdName": "qa-wf", + "droneSchedulerIdName": "qa-ds" + }, + "staging": { + "deliveryIdName": "staging-d", + "workflowIdName": "staging-wf", + "droneSchedulerIdName": "staging-ds" + }, + "prod": { + "deliveryIdName": "prod-d", + "workflowIdName": "prod-wf", + "droneSchedulerIdName": "prod-ds" + } + } }, "resources": [ { "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('workflowIdName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" + "location": "[resourceGroup().location]", + "tags": { + "displayName": "workflow managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-workflow", + "tier": "[parameters('environmentName')]" + } }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('deliveryIdName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" + "location": "[resourceGroup().location]", + "tags": { + "displayName": "delivery managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-delivery", + "tier": "[parameters('environmentName')]" + } }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('droneSchedulerIdName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]" + "location": "[resourceGroup().location]", + "tags": { + "displayName": "dronescheduler managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-dronescheduler", + "tier": "[parameters('environmentName')]" + } } ], "outputs": { "deliveryIdName": { - "value": "[variables('deliveryIdName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", "type": "string" }, "deliveryPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).principalId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)).principalId]", "type": "string" }, "deliveryPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))]", + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)]", "type": "string" }, "deliveryPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('deliveryIdName'))).clientId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)).clientId]", "type": "string" }, "droneSchedulerIdName": { - "value": "[variables('droneSchedulerIdName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", "type": "string" }, "droneSchedulerPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).principalId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)).principalId]", "type": "string" }, "droneSchedulerPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))]", + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)]", "type": "string" }, "droneSchedulerPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('droneSchedulerIdName'))).clientId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)).clientId]", "type": "string" }, "workflowIdName": { - "value": "[variables('workflowIdName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", "type": "string" }, "workflowPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).principalId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)).principalId]", "type": "string" }, "workflowPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))]", + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)]", "type": "string" }, "workflowPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('workflowIdName'))).clientId]", + "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)).clientId]", "type": "string" } } -} \ No newline at end of file +} diff --git a/azuredeploy.json b/azuredeploy.json index c8dbf3f1..41613548 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -2,6 +2,16 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { + "environmentName": { + "type": "string", + "defaultValue": "dev", + "allowedValues": [ + "dev", + "qa", + "staging", + "prod" + ] + }, "deliveryIdName": { "metadata": { "description": "Name of the delivery managed identity" @@ -133,14 +143,14 @@ "Standard", "Premium" ], - "defaultValue": "Premium", + "defaultValue": "Basic", "metadata": { "description": "The pricing tier of the new Azure Redis Cache." } }, "deliveryRedisCacheFamily": { "type": "string", - "defaultValue": "P", + "defaultValue": "C", "metadata": { "description": "The family for the sku." }, @@ -152,12 +162,13 @@ "deliveryRedisCacheCapacity": { "type": "int", "allowedValues": [ + 0, 1, 2, 3, 4 ], - "defaultValue": 4, + "defaultValue": 0, "metadata": { "description": "The size of the new Azure Redis Cache instance. " } @@ -175,36 +186,97 @@ } }, "variables": { - "aksClusterName": "[concat('aksCluster-', uniqueString(resourceGroup().id))]", - "aksClusterDnsNamePrefix": "[concat('cluster-',uniqueString(resourceGroup().id))]", - "acrName": "[concat('acr', uniqueString(resourceGroup().id))]", - "appInsightsName": "[concat('ai', uniqueString(resourceGroup().id))]", - "deliveryRedisStorageName": "[concat('redsto', uniqueString(resourceGroup().id))]", - "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts', variables('deliveryRedisStorageName'))]", - "deliveryCosmosDbName": "[concat('delivery-service-cosmosdb-', uniqueString(resourceGroup().id))]", - "deliveryRedisName": "[concat('delivery-service-redis-', uniqueString(resourceGroup().id))]", - "deliveryKeyVaultName": "[concat(resourceGroup().name, '-d-kv')]", - "packageMongoDbName": "[concat('package-service-cosmosdb-', uniqueString(resourceGroup().id))]", - "ingestionSBNamespace": "[concat('ingsbns-', uniqueString(resourceGroup().id))]", - "ingestionSBName": "[concat('ingsb-', uniqueString(resourceGroup().id))]", - "ingestionServiceAccessKey": "IngestionServiceAccessKey", - "droneSchedulerKeyVaultName": "[concat(resourceGroup().name, '-ds-kv')]", - "workflowKeyVaultName": "[concat(resourceGroup().name, '-wf-kv')]", - "workflowServiceAccessKey": "WorkflowServiceAccessKey", + "clusterNamePrefix": "aks", + "acrNamePrefix": "acr", + "aiNamePrefix": "ai", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", - "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", - "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]" + "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", + "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]", + "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", + "environmentSettings": { + "dev": { + "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", + "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", + "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionServiceAccessKey": "IngestionServiceAccessKey", + "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", + "workflowServiceAccessKey": "WorkflowServiceAccessKey" + }, + "qa": { + "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", + "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", + "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionServiceAccessKey": "IngestionServiceAccessKey", + "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", + "workflowServiceAccessKey": "WorkflowServiceAccessKey" + }, + "staging": { + "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", + "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", + "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionServiceAccessKey": "IngestionServiceAccessKey", + "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", + "workflowServiceAccessKey": "WorkflowServiceAccessKey" + }, + "prod": { + "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", + "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", + "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", + "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", + "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionServiceAccessKey": "IngestionServiceAccessKey", + "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", + "workflowServiceAccessKey": "WorkflowServiceAccessKey" + } + } }, "resources": [ { - "name": "[variables('aksClusterName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", "apiVersion": "2018-03-31", "location": "[resourceGroup().location]", + "tags": { + "environment": "shared cluster" + }, "properties": { "kubernetesVersion": "[parameters('kubernetesVersion')]", - "dnsPrefix": "[variables('aksClusterDnsNamePrefix')]", + "dnsPrefix": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "agentPoolProfiles": [ { "name": "agentpool", @@ -233,7 +305,7 @@ } }, { - "name": "[variables('acrName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].acrName]", "type": "Microsoft.ContainerRegistry/registries", "apiVersion": "2017-10-01", "sku": { @@ -242,44 +314,56 @@ }, "location": "[resourceGroup().location]", "tags": { - "displayName": "Container Registry", - "container.registry": "[variables('acrName')]" + "displayName": "Shared Container Registry for dev, qa, staging and production", + "container.registry": "[variables('environmentSettings')[parameters('environmentName')].acrName]", + "clusterName": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]" }, "properties": { "adminUserEnabled": false } }, { - "name": "[variables('appInsightsName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].appInsightsName]", "type": "Microsoft.Insights/components", "apiVersion": "2015-05-01", "kind": "other", "location": "[resourceGroup().location]", + "tags": { + "displayName": "App Insights instance - Distributed Tracing", + "environment": "[parameters('environmentName')]" + }, "properties": { "Application_Type": "other" } }, { - "name": "[variables('deliveryRedisStorageName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageName]", "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2015-06-15", "location": "[resourceGroup().location]", "comments": "This storage account is used by Delivery Redis", "dependsOn": [], "tags": { - "displayName": "Delivery Redis storage" + "displayName": "Storage account for inflight deliveries", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" }, "properties": { "accountType": "[parameters('deliveryRedisStorageType')]" } }, { - "apiVersion": "2016-04-01", - "name": "[variables('deliveryRedisName')]", + "apiVersion": "2018-03-01", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisName]", "type": "Microsoft.Cache/Redis", "location": "[resourceGroup().location]", + "tags": { + "displayName": "Redis Cache for inflight deliveries", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" + }, "dependsOn": [ - "[variables('deliveryRedisStorageId')]" + "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]" ], "properties": { "redisEnableNonSslPort": "false", @@ -287,20 +371,19 @@ "capacity": "[parameters('deliveryRedisCacheCapacity')]", "family": "[parameters('deliveryRedisCacheFamily')]", "name": "[parameters('deliveryRedisCacheSKU')]" - }, - "vm-size": "P4" + } }, "resources": [ { "apiVersion": "2017-05-01-preview", "type": "Microsoft.Cache/redis/providers/diagnosticsettings", - "name": "[concat(variables('deliveryRedisName'), '/Microsoft.Insights/', variables('deliveryRedisName'))]", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].deliveryRedisName, '/Microsoft.Insights/', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]" ], "properties": { - "storageAccountId": "[variables('deliveryRedisStorageId')]", + "storageAccountId": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]", "logs": [], "metrics": [ { @@ -318,11 +401,16 @@ }, { "type": "Microsoft.DocumentDB/databaseAccounts", - "name": "[variables('deliveryCosmosDbName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "apiVersion": "2016-03-31", "location": "[resourceGroup().location]", + "tags": { + "displayName": "Delivery Cosmos Db", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" + }, "properties": { - "name": "[variables('deliveryCosmosDbName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "databaseAccountOfferType": "Standard", "locations": [ { @@ -336,29 +424,41 @@ "apiVersion": "2015-04-08", "type": "Microsoft.DocumentDB/databaseAccounts", "kind": "MongoDB", - "name": "[variables('packageMongoDbName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "location": "[resourceGroup().location]", + "tags": { + "displayName": "Package Cosmos Db", + "app": "fabrikam-package", + "environment": "[parameters('environmentName')]" + }, "properties": { "databaseAccountOfferType": "Standard", - "name": "[variables('packageMongoDbName')]" + "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]" } }, { "type": "Microsoft.ServiceBus/namespaces", - "name": "[variables('ingestionSBNamespace')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace]", "apiVersion": "2017-04-01", "location": "[resourceGroup().location]", "sku": { "name": "Standard", "tier": "Standard" }, + "tags": { + "displayName": "Ingestion and Workflow Service Bus", + "app": "fabrikam-ingestion", + "app-producer": "fabrikam-ingestion", + "app-consumer": "fabrikam-workflow", + "environment": "[parameters('environmentName')]" + }, "resources": [ { - "name": "[variables('ingestionSBName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBName]", "type": "queues", "apiVersion": "2017-04-01", "dependsOn": [ - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]" ], "properties": { "lockDuration": "PT5M", @@ -367,7 +467,7 @@ } }, { - "name": "[variables('ingestionServiceAccessKey')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].ingestionServiceAccessKey]", "type": "AuthorizationRules", "apiVersion": "2017-04-01", "properties": { @@ -376,11 +476,11 @@ ] }, "dependsOn": [ - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]" ] }, { - "name": "[variables('workflowServiceAccessKey')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].workflowServiceAccessKey]", "type": "AuthorizationRules", "apiVersion": "2017-04-01", "properties": { @@ -389,19 +489,24 @@ ] }, "dependsOn": [ - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]" ] } ] }, { "type": "Microsoft.KeyVault/vaults", - "name": "[variables('deliveryKeyVaultName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[variables('deliveryRedisStorageId')]" + "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]" ], + "tags": { + "displayName": "Delivery Key Vault", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" + }, "properties": { "sku": { "family": "A", @@ -437,11 +542,11 @@ "name": "CosmosDB-Endpoint", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName'))).documentEndpoint]" + "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName)).documentEndpoint]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]", + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]" ] }, { @@ -449,11 +554,11 @@ "name": "CosmosDB-Key", "apiVersion": "2015-06-01", "properties": { - "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('deliveryCosmosDbName')), '2016-03-31').primaryMasterKey]" + "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName), '2016-03-31').primaryMasterKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]", + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]" ] }, { @@ -461,11 +566,11 @@ "name": "Redis-Endpoint", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))).hostName]" + "value": "[reference(resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)).hostName]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]", + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]" ] }, { @@ -473,11 +578,11 @@ "name": "Redis-AccessKey", "apiVersion": "2015-06-01", "properties": { - "value": "[listKeys(resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName')), '2016-04-01').primaryKey]" + "value": "[listKeys(resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName), '2016-04-01').primaryKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.Cache/Redis', variables('deliveryRedisName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]", + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]" ] }, { @@ -485,20 +590,25 @@ "name": "ApplicationInsights--InstrumentationKey", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + "value": "[reference(resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName),'2015-05-01').InstrumentationKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]", - "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]", + "[resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName)]" ] } ] }, { "type": "Microsoft.KeyVault/vaults", - "name": "[variables('droneSchedulerKeyVaultName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", + "tags": { + "displayName": "DroneScheduler Key Vault", + "app": "fabrikam-dronescheduler", + "environment": "[parameters('environmentName')]" + }, "properties": { "sku": { "family": "A", @@ -524,20 +634,25 @@ "name": "ApplicationInsights--InstrumentationKey", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + "value": "[reference(resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName),'2015-05-01').InstrumentationKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]", - "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]", + "[resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName)]" ] } ] }, { "type": "Microsoft.KeyVault/vaults", - "name": "[variables('workflowKeyVaultName')]", + "name": "[variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", + "tags": { + "displayName": "Workflow Key Vault", + "app": "fabrikam-workflow", + "environment": "[parameters('environmentName')]" + }, "properties": { "sku": { "family": "A", @@ -563,10 +678,10 @@ "name": "QueueName", "apiVersion": "2015-06-01", "properties": { - "value": "[variables('ingestionSBName')]" + "value": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBName]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]" ] }, { @@ -574,11 +689,11 @@ "name": "QueueEndpoint", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))).serviceBusEndpoint]" + "value": "[reference(resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)).serviceBusEndpoint]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", - "[resourceId('Microsoft.ServiceBus/namespaces', variables('ingestionSBNamespace'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]", + "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]" ] }, { @@ -586,10 +701,10 @@ "name": "QueueAccessPolicyName", "apiVersion": "2015-06-01", "properties": { - "value": "[variables('workflowServiceAccessKey')]" + "value": "[variables('environmentSettings')[parameters('environmentName')].workflowServiceAccessKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]" ] }, { @@ -597,11 +712,11 @@ "name": "QueueAccessPolicyKey", "apiVersion": "2015-06-01", "properties": { - "value": "[listkeys(resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('ingestionSBNamespace'), variables('workflowServiceAccessKey')), '2017-04-01').primaryKey]" + "value": "[listkeys(resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace, variables('environmentSettings')[parameters('environmentName')].workflowServiceAccessKey), '2017-04-01').primaryKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", - "[resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('ingestionSBNamespace'), variables('workflowServiceAccessKey'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]", + "[resourceId('Microsoft.ServiceBus/namespaces/authorizationRules', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace, variables('environmentSettings')[parameters('environmentName')].workflowServiceAccessKey)]" ] }, { @@ -609,73 +724,124 @@ "name": "ApplicationInsights-InstrumentationKey", "apiVersion": "2015-06-01", "properties": { - "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')),'2015-05-01').InstrumentationKey]" + "value": "[reference(resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName),'2015-05-01').InstrumentationKey]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]", - "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]", + "[resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName)]" ] } ] }, { "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", - "name": "[concat(variables('deliveryKeyVaultName'), '/Microsoft.Authorization/', guid('delivery-kv', resourceGroup().id))]", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName,'/Microsoft.Authorization/',guid(concat('kv-delivery',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", + "tags": { + "displayName": "Delivery app RBAC Reader for Key Vault", + "what": "rbac", + "to": "pod", + "identity-type": "msi", + "access": "keyvault", + "reason": "aad-pod-identity", + "flex-vol": "no", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('readerRoleId')]", "principalId": "[parameters('deliveryPrincipalId')]", - "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]" + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)]" ] }, { "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", - "name": "[concat(variables('droneSchedulerKeyVaultName'), '/Microsoft.Authorization/', guid('dronescheduler-kv', resourceGroup().id))]", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName,'/Microsoft.Authorization/',guid(concat('kv-dronescheduler',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", + "tags": { + "displayName": "DroneScheduler app RBAC Reader for Key Vault", + "what": "rbac", + "to": "pod", + "identity-type": "msi", + "access": "keyvault", + "reason": "aad-pod-identity", + "flex-vol": "no", + "app": "fabrikam-dronescheduler", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('readerRoleId')]", "principalId": "[parameters('droneSchedulerPrincipalId')]", - "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]" + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" ] }, { "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", - "name": "[concat(variables('workflowKeyVaultName'), '/Microsoft.Authorization/', guid('workflow-kv', resourceGroup().id))]", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName,'/Microsoft.Authorization/',guid(concat('kv-workflow',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", + "tags": { + "displayName": "Workflow app RBAC Reader for Key Vault", + "what": "rbac", + "to": "pod", + "identity-type": "msi", + "access": "keyvault", + "reason": "aad-pod-identity", + "flex-vol": "yes", + "app": "fabrikam-workflow", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('readerRoleId')]", "principalId": "[parameters('workflowPrincipalId')]", - "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + "scope": "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('workflowKeyVaultName'))]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]" ] }, { "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", - "name": "[concat(variables('acrName'), '/Microsoft.Authorization/', guid('aks', resourceGroup().id))]", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].acrName,'/Microsoft.Authorization/',guid('aks', resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the ACR instance", + "tags": { + "displayName": "AKS SP RBAC Access to ACR", + "what": "rbac", + "to": "cluster", + "identity-type": "sp", + "access": "acr", + "reason": "pull-images" + }, "properties": { "roleDefinitionId": "[variables('readerRoleId')]", "principalId": "[parameters('servicePrincipalId')]", - "scope": "[resourceId('Microsoft.ContainerRegistry/registries', variables('acrName'))]" + "scope": "[resourceId('Microsoft.ContainerRegistry/registries', variables('environmentSettings')[parameters('environmentName')].acrName)]" }, "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('acrName'))]" + "[resourceId('Microsoft.ContainerRegistry/registries', variables('environmentSettings')[parameters('environmentName')].acrName)]" ] }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(parameters('deliveryIdName'), '/Microsoft.Authorization/', guid('msi-delivery', resourceGroup().id))]", + "name": "[concat(parameters('deliveryIdName'), '/Microsoft.Authorization/', guid(concat('msi-delivery',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the delivery managed id", + "tags": { + "displayName": "AKS SP RBAC Access for delivery managed identity", + "what": "rbac", + "to": "cluster", + "identity-type": "sp", + "access": "msi", + "reason": "aad-pod-identity", + "app": "fabrikam-delivery", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" @@ -683,9 +849,19 @@ }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(parameters('workflowIdName'), '/Microsoft.Authorization/', guid('msi-workflow', resourceGroup().id))]", + "name": "[concat(parameters('workflowIdName'), '/Microsoft.Authorization/', guid(concat('msi-workflow',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the workflow managed id", + "tags": { + "displayName": "AKS SP RBAC Access for workflow managed identity", + "what": "rbac", + "to": "cluster", + "identity-type": "sp", + "access": "msi", + "reason": "aad-pod-identity", + "app": "fabrikam-workflow", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" @@ -693,9 +869,19 @@ }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "name": "[concat(parameters('droneSchedulerIdName'), '/Microsoft.Authorization/', guid('msi-dronescheduler', resourceGroup().id))]", + "name": "[concat(parameters('droneSchedulerIdName'), '/Microsoft.Authorization/', guid(concat('msi-dronescheduler',parameters('environmentName')), resourceGroup().id))]", "apiVersion": "2017-05-01", "comments": "Grant the AKS cluster access to the drone scheduler managed id", + "tags": { + "displayName": "AKS SP RBAC Access for dronescheduler managed identity", + "what": "rbac", + "to": "cluster", + "identity-type": "sp", + "access": "msi", + "reason": "aad-pod-identity", + "app": "fabrikam-dronescheduler", + "environment": "[parameters('environmentName')]" + }, "properties": { "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" @@ -704,51 +890,51 @@ ], "outputs": { "acrName": { - "value": "[variables('acrName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].acrName]", "type": "string" }, "acrLoginServer": { - "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('acrName')),'2016-06-27-preview').loginServer]", + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('environmentSettings')[parameters('environmentName')].acrName),'2016-06-27-preview').loginServer]", "type": "string" }, "aksClusterName": { - "value": "[variables('aksClusterName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "string" }, "appInsightsName": { - "value": "[variables('appInsightsName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].appInsightsName]", "type": "string" }, "deliveryKeyVaultUri": { - "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('deliveryKeyVaultName'))).vaultUri]", + "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].deliveryKeyVaultName)).vaultUri]", "type": "string" }, "deliveryCosmosDbName": { - "value": "[variables('deliveryCosmosDbName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "type": "string" }, "droneSchedulerKeyVaultUri": { - "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('droneSchedulerKeyVaultName'))).vaultUri]", + "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)).vaultUri]", "type": "string" }, "packageMongoDbName": { - "value": "[variables('packageMongoDbName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "type": "string" }, "workflowKeyVaultName": { - "value": "[variables('workflowKeyVaultName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName]", "type": "string" }, "ingestionQueueNamespace": { - "value": "[variables('ingestionSBNamespace')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace]", "type": "string" }, "ingestionQueueName": { - "value": "[variables('ingestionSBName')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBName]", "type": "string" }, "ingestionServiceAccessKeyName": { - "value": "[variables('ingestionServiceAccessKey')]", + "value": "[variables('environmentSettings')[parameters('environmentName')].ingestionServiceAccessKey]", "type": "string" } } diff --git a/deployment.md b/deployment.md index 518003ee..312d280f 100644 --- a/deployment.md +++ b/deployment.md @@ -58,30 +58,39 @@ export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) ``` -Deployment +## Optional: Set up automated CI/CD for dev, test, qa and production with Azure DevOps + +Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). + +> Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application for development as follows. + +## Manual deployment for dev > Note: this deployment might take up to 20 minutes +Infrastructure + ```bash # Deploy the managed identities # These are deployed first in a separate template to avoid propagation delays with AAD -az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities --template-file ${PROJECT_ROOT}/azuredeploy-identities.json -export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryIdName.value -o tsv) -export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalId.value -o tsv) -export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerIdName.value -o tsv) -export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) -export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowIdName.value -o tsv) -export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalId.value -o tsv) +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities-dev --template-file ${PROJECT_ROOT}/azuredeploy-identities.json +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalId.value -o tsv) +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalId.value -o tsv) # Wait for AAD propagation until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +# Export the kubernetes cluster version +export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) + # Deploy all other resources -# The version of kubernetes must be supported in the target region -export KUBERNETES_VERSION='1.12.6' -az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file ${PROJECT_ROOT}/azuredeploy.json \ +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template-file ${PROJECT_ROOT}/azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ servicePrincipalId=${SP_OBJECT_ID} \ @@ -98,9 +107,9 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy --template-file Get outputs from Azure Deploy ```bash # Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.acrLoginServer.value -o tsv) && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.aksClusterName.value -o tsv) +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrLoginServer.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) ``` Enable Azure Monitoring for the AKS cluster @@ -117,8 +126,7 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces -kubectl create namespace backend && \ -kubectl create namespace frontend +kubectl create namespace backend ``` Setup Helm in the container @@ -132,7 +140,7 @@ Integrate Application Insights instance ```bash # Acquire Instrumentation Key -export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.appInsightsName.value -o tsv) +export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appInsightsName.value -o tsv) export AI_IKEY=$(az resource show \ -g $RESOURCE_GROUP \ -n $AI_NAME \ @@ -181,15 +189,16 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ingestion-ingress-tls.key \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` + ## Deploy the Delivery service Extract resource details from deployment ```bash -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ -export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) +export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryKeyVaultUri.value -o tsv) ``` Build the Delivery service @@ -213,8 +222,8 @@ Deploy the Delivery service: ```bash # Extract pod identity outputs from deployment -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalClientId.value -o tsv) export DELIVERY_INGRESS_TLS_SECRET_NAME=delivery-ingress-tls # Deploy the service @@ -235,10 +244,10 @@ helm install $HELM_CHARTS/delivery/ \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ --set reason="Initial deployment" \ - --set tags.prod=true \ + --set tags.dev=true \ --set current=true \ - --namespace backend \ - --name delivery-v0.1.0 + --namespace backend-dev \ + --name delivery-v0.1.0-dev # Verify the pod is created helm status delivery-v0.1.0 @@ -249,7 +258,7 @@ helm status delivery-v0.1.0 Extract resource details from deployment ```bash -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.packageMongoDbName.value -o tsv) ``` Build the Package service @@ -282,10 +291,10 @@ helm install $HELM_CHARTS/package/ \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ - --set tags.prod=true \ + --set tags.dev=true \ --set current=true \ - --namespace backend \ - --name package-v0.1.0 + --namespace backend-dev \ + --name package-v0.1.0-dev # Verify the pod is created helm status package-v0.1.0 @@ -296,7 +305,7 @@ helm status package-v0.1.0 Extract resource details from deployment ```bash -export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) +export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.workflowKeyVaultName.value -o tsv) ``` Build the workflow service @@ -316,8 +325,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalClientId.value -o tsv) ``` Deploy the Workflow service: @@ -335,9 +344,9 @@ helm install $HELM_CHARTS/workflow/ \ --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ --set keyvault.tenantid=$TENANT_ID \ --set reason="Initial deployment" \ - --set tags.prod=true \ - --namespace backend \ - --name workflow-v0.1.0 + --set tags.dev=true \ + --namespace backend-dev \ + --name workflow-v0.1.0-dev # Verify the pod is created helm status workflow-v0.1.0 @@ -348,9 +357,9 @@ helm status workflow-v0.1.0 Extract resource details from deployment ```bash -export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ -export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) -export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) +export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ +export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionQueueName.value -o tsv) +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) ``` @@ -391,10 +400,10 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ --set reason="Initial deployment" \ - --set tags.prod=true \ + --set tags.dev=true \ --set current=true \ - --namespace backend \ - --name ingestion-v0.1.0 + --namespace backend-dev \ + --name ingestion-v0.1.0-dev # Verify the pod is created helm status ingestion-v0.1.0 @@ -405,7 +414,7 @@ helm status ingestion-v0.1.0 Extract resource details from deployment ```bash -export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) ``` Build the dronescheduler services @@ -418,8 +427,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) ``` Build and publish the container image @@ -444,10 +453,10 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --set reason="Initial deployment" \ - --set tags.prod=true \ + --set tags.dev=true \ --set current=true \ - --namespace backend \ - --name dronescheduler-v0.1.0 + --namespace backend-dev \ + --name dronescheduler-v0.1.0-dev # Verify the pod is created helm status dronescheduler-v0.1.0 diff --git a/deploymentCICD.md b/deploymentCICD.md index 7d72d041..61b2f398 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -8,6 +8,103 @@ - [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) - [Values from deployment instructions](./deployment.md) +## Infrastructure for dev, test, staging and production + +```bash +# Export the kubernetes cluster version and deploy +export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) && \ +for env in dev qa staging prod; do +ENV=${env^^} +az group deployment create \ + -g $RESOURCE_GROUP \ + --name azuredeploy-identities-${env} \ + --template-file ${PROJECT_ROOT}/azuredeploy-identities.json \ + --parameters environmentName=${env} + +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalId.value -o tsv) +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalId.value -o tsv) + +# Wait for AAD propagation +until az ad sp show --id $DELIVERY_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id $DRONESCHEDULER_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id $WORKFLOW_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done + +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --template-file ${PROJECT_ROOT}/azuredeploy.json \ + --parameters servicePrincipalClientId=${SP_APP_ID} \ + servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ + servicePrincipalId=${SP_OBJECT_ID} \ + kubernetesVersion=${KUBERNETES_VERSION} \ + sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \ + deliveryIdName="$DELIVERY_ID_NAME" \ + deliveryPrincipalId=$DELIVERY_ID_PRINCIPAL_ID \ + droneSchedulerIdName=$DRONESCHEDULER_ID_NAME \ + droneSchedulerPrincipalId=$DRONESCHEDULER_ID_PRINCIPAL_ID \ + workflowIdName=$WORKFLOW_ID_NAME \ + workflowPrincipalId=$WORKFLOW_ID_PRINCIPAL_ID \ + environmentName=${env} + +export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) +export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) + +done +``` + +Get outputs from Azure Deploy +```bash +# Shared +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrLoginServer.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) && \ +``` + +Enable Azure Monitoring for Containers in the AKS cluster +```bash +az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME +``` + +Download kubectl and create a k8s namespace +```bash +# Install kubectl +sudo az aks install-cli + +# Get the Kubernetes cluster credentials +az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME + +# Create namespaces +kubectl create namespace backend +``` + +Setup Helm in the container + +```bash +kubectl apply -f $K8S/tiller-rbac.yaml +helm init --service-account tiller +``` + +## Integrate Application Insights instance + +```bash +# add RBAC for AppInsights +kubectl apply -f $K8S/k8s-rbac-ai.yaml +``` + +## Setup AAD pod identity and key vault flexvol infrastructure + +Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol + +Note: the tested nmi version was 1.4. It enables namespaced pod identity. + +```bash +# setup AAD pod identity +kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml + +kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml +``` + ## Setup Azure DevOps ``` @@ -151,12 +248,14 @@ az pipelines create \ # query build definition details and resources export AZURE_DEVOPS_DELIVERY_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='delivery-ci'].id" -o tsv) && \ export AZURE_DEVOPS_DELIVERY_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='delivery-ci'].queue.id" -o tsv) && \ -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ -export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ -export COLLECTION_NAME="${DATABASE_NAME}-col" && \ -export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +for env in dev qa staging prod;do +ENV=${env^^} +export ${ENV}_DATABASE_NAME="deliveries-db" +export ${ENV}_COLLECTION_NAME="deliveries-col" +export ${ENV}_DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.deliveryKeyVaultUri.value -o tsv) +export ${ENV}_DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) +export ${ENV}_DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +done # add relese definition cat $DELIVERY_PATH/azure-pipelines-cd.json | \ @@ -171,29 +270,29 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources - sed "s#DEV_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#DEV_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#DEV_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ - sed "s#DEV_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ - sed "s#DEV_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + sed "s#DEV_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_DATABASE_NAME_VAR_VAL#$DEV_DATABASE_NAME#g" | \ + sed "s#DEV_COLLECTION_NAME_VAR_VAL#$DEV_COLLECTION_NAME#g" | \ + sed "s#DEV_DELIVERY_KEYVAULT_URI_VAR_VAL#$DEV_DELIVERY_KEYVAULT_URI#g" | \ # qa resources - sed "s#QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#QA_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ - sed "s#QA_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ - sed "s#QA_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + sed "s#QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_DATABASE_NAME_VAR_VAL#$QA_DATABASE_NAME#g" | \ + sed "s#QA_COLLECTION_NAME_VAR_VAL#$QA_COLLECTION_NAME#g" | \ + sed "s#QA_DELIVERY_KEYVAULT_URI_VAR_VAL#$QA_DELIVERY_KEYVAULT_URI#g" | \ # staging resources - sed "s#STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#STAGING_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ - sed "s#STAGING_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ - sed "s#STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" | \ + sed "s#STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_DATABASE_NAME_VAR_VAL#$STAGING_DATABASE_NAME#g" | \ + sed "s#STAGING_COLLECTION_NAME_VAR_VAL#$STAGING_COLLECTION_NAME#g" | \ + sed "s#STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL#$STAGING_DELIVERY_KEYVAULT_URI#g" | \ # production resources - sed "s#PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#PROD_DATABASE_NAME_VAR_VAL#$DATABASE_NAME#g" | \ - sed "s#PROD_COLLECTION_NAME_VAR_VAL#$COLLECTION_NAME#g" | \ - sed "s#PROD_DELIVERY_KEYVAULT_URI_VAR_VAL#$DELIVERY_KEYVAULT_URI#g" \ + sed "s#PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_DATABASE_NAME_VAR_VAL#$PROD_DATABASE_NAME#g" | \ + sed "s#PROD_COLLECTION_NAME_VAR_VAL#$PROD_COLLECTION_NAME#g" | \ + sed "s#PROD_DELIVERY_KEYVAULT_URI_VAR_VAL#$PROD_DELIVERY_KEYVAULT_URI#g" \ > $DELIVERY_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -234,9 +333,12 @@ az pipelines create \ # query build definition details and resources export AZURE_DEVOPS_PACKAGE_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='package-ci'].id" -o tsv) && \ export AZURE_DEVOPS_PACKAGE_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='package-ci'].queue.id" -o tsv) && \ -export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.packageMongoDbName.value -o tsv) && \ -export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ -export COSMOSDB_COL_NAME=packages +for env in dev qa staging prod;do +ENV=${env^^} +export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.packageMongoDbName.value -o tsv) +export ${ENV}_COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') +export ${ENV}_COSMOSDB_COL_NAME=packages +done # add relese definition cat $PACKAGE_PATH/azure-pipelines-cd.json | \ @@ -250,19 +352,22 @@ cat $PACKAGE_PATH/azure-pipelines-cd.json | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ # development resources - sed "s#DEV_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ - sed "s#DEV_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#DEV_AI_IKEY_VAR_VAL#$DEV_AI_IKEY#g" | \ + sed "s#DEV_COSMOSDB_COL_NAME_VAR_VAL#$DEV_COSMOSDB_COL_NAME#g" | \ + sed "s#DEV_COSMOSDB_CONNECTION_VAR_VAL#${DEV_COSMOSDB_CONNECTION//&/\\&}#g" | \ # qa resources - sed "s#QA_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ - sed "s#QA_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ + sed "s#QA_COSMOSDB_COL_NAME_VAR_VAL#$QA_COSMOSDB_COL_NAME#g" | \ + sed "s#QA_COSMOSDB_CONNECTION_VAR_VAL#${QA_COSMOSDB_CONNECTION//&/\\&}#g" | \ # staging resources - sed "s#STAGING_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ - sed "s#STAGING_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ + sed "s#STAGING_COSMOSDB_COL_NAME_VAR_VAL#$STAGING_COSMOSDB_COL_NAME#g" | \ + sed "s#STAGING_COSMOSDB_CONNECTION_VAR_VAL#${STAGING_COSMOSDB_CONNECTION//&/\\&}#g" | \ # production resources - sed "s#PROD_COSMOSDB_COL_NAME_VAR_VAL#$COSMOSDB_COL_NAME#g" | \ - sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${COSMOSDB_CONNECTION//&/\\&}#g" \ + sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ + sed "s#PROD_COSMOSDB_COL_NAME_VAR_VAL#$PROD_COSMOSDB_COL_NAME#g" | \ + sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${PROD_COSMOSDB_CONNECTION//&/\\&}#g" \ > $PACKAGE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -302,9 +407,12 @@ az pipelines create \ # query build definition details and resources export AZURE_DEVOPS_WORKFLOW_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='workflow-ci'].id" -o tsv) && \ export AZURE_DEVOPS_WORKFLOW_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='workflow-ci'].queue.id" -o tsv) && \ -export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.workflowKeyVaultName.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities --query properties.outputs.workflowPrincipalClientId.value -o tsv) +for env in dev qa staging prod;do +ENV=${env^^} +export ${ENV}_WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.workflowKeyVaultName.value -o tsv) +export ${ENV}_WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalResourceId.value -o tsv) +export ${ENV}_WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalClientId.value -o tsv) +done # add relese definition cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ @@ -320,30 +428,30 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources sed "s#DEV_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#DEV_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#DEV_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#DEV_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$DEV_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#DEV_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#DEV_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # qa resources sed "s#QA_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#QA_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#QA_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#QA_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$QA_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#QA_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#QA_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # staging resources sed "s#STAGING_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#STAGING_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$STAGING_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#STAGING_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#STAGING_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # production resources sed "s#PROD_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#PROD_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#PROD_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$WORKFLOW_KEYVAULT_NAME#g" | \ + sed "s#PROD_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$PROD_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#PROD_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#PROD_TENANT_ID_VAR_VAL#$TENANT_ID#g" \ > $WORKFLOW_PATH/azure-pipelines-cd-0.json @@ -388,10 +496,13 @@ az pipelines create \ # query build definition details and resources export AZURE_DEVOPS_INGESTION_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='ingestion-ci'].id" -o tsv) && \ export AZURE_DEVOPS_INGESTION_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='ingestion-ci'].queue.id" -o tsv) && \ -export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ -export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionQueueName.value -o tsv) && \ -export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) && \ -export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) && \ +for env in dev qa staging prod;do +ENV=${env^^} +export {${ENV}_INGESTION_QUEUE_NAMESPACE,INGESTION_QUEUE_NAMESPACE}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.ingestionQueueNamespace.value -o tsv) +export ${ENV}_INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.ingestionQueueName.value -o tsv) +export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv) +export ${ENV}_INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv) +done && \ export INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # add relese definition @@ -406,43 +517,46 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ - sed "s#AI_IKEY_VAR_VAL#$AI_IKEY#g" | \ # development resources - sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ - sed "s#DEV_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ - sed "s#DEV_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#DEV_AI_IKEY_VAR_VAL#$DEV_AI_IKEY#g" | \ sed "s#DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$DEV_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$DEV_EXTERNAL_INGEST_FQDN#g" | \ - sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$DEV_INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#DEV_INGESTION_QUEUE_NAME_VAR_VAL#$DEV_INGESTION_QUEUE_NAME#g" | \ + sed "s#DEV_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$DEV_INGESTION_ACCESS_KEY_VALUE#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$DEV_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # qa resources - sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ - sed "s#QA_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ - sed "s#QA_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ sed "s#QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$QA_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$QA_EXTERNAL_INGEST_FQDN#g" | \ - sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$QA_INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#QA_INGESTION_QUEUE_NAME_VAR_VAL#$QA_INGESTION_QUEUE_NAME#g" | \ + sed "s#QA_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$QA_INGESTION_ACCESS_KEY_VALUE#g" | \ sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$QA_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # staging resources - sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ - sed "s#STAGING_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ - sed "s#STAGING_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ sed "s#STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$STAGING_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$STAGING_EXTERNAL_INGEST_FQDN#g" | \ - sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$STAGING_INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#STAGING_INGESTION_QUEUE_NAME_VAR_VAL#$STAGING_INGESTION_QUEUE_NAME#g" | \ + sed "s#STAGING_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$STAGING_INGESTION_ACCESS_KEY_VALUE#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # production resources - sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$INGESTION_QUEUE_NAMESPACE#g" | \ - sed "s#PROD_INGESTION_QUEUE_NAME_VAR_VAL#$INGESTION_QUEUE_NAME#g" | \ - sed "s#PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$INGESTION_ACCESS_KEY_VALUE#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ sed "s#PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$PROD_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" \ + sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$PROD_INGESTION_QUEUE_NAMESPACE#g" | \ + sed "s#PROD_INGESTION_QUEUE_NAME_VAR_VAL#$PROD_INGESTION_QUEUE_NAME#g" | \ + sed "s#PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$PROD_INGESTION_ACCESS_KEY_VALUE#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" \ > $INGESTION_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -483,9 +597,12 @@ az pipelines create \ # query build definition details and resources export AZURE_DEVOPS_DRONE_BUILD_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='dronescheduler-ci'].id" -o tsv) && \ export AZURE_DEVOPS_DRONE_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='dronescheduler-ci'].queue.id" -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) && \ -export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +for env in dev qa staging prod;do +ENV=${env^^} +export ${ENV}_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) +export ${ENV}_DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +export ${ENV}_DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +done # add relese definition cat $DRONE_PATH/azure-pipelines-cd.json | \ @@ -500,21 +617,21 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources - sed "s#DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DEV_DRONESCHEDULER_KEYVAULT_URI#g" | \ # qa resources - sed "s#QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$QA_DRONESCHEDULER_KEYVAULT_URI#g" | \ # staging resources - sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$STAGING_DRONESCHEDULER_KEYVAULT_URI#g" | \ # production resources - sed "s#PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ - sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DRONESCHEDULER_KEYVAULT_URI#g" \ + sed "s#PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ + sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ + sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$PROD_DRONESCHEDULER_KEYVAULT_URI#g" \ > $DRONE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index e6252f9d..89ef1387 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -10,10 +10,6 @@ "ACR_SERVER": { "value": "ACR_SERVER_VAR_VAL" }, - "AI_IKEY": { - "value": "AI_IKEY_VAR_VAL", - "isSecret": true - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -35,6 +31,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "DEV_AI_IKEY_VAR_VAL", + "isSecret": true + }, "INGRESS_LOAD_BALANCER_IP": { "value": "DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL" }, @@ -411,10 +411,13 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "QA_AI_IKEY_VAR_VAL", + "isSecret": true + }, "INGRESS_LOAD_BALANCER_IP": { "value": "QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL" }, - "EXTERNAL_INGEST_FQDN": { "value": "QA_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -788,10 +791,13 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "STAGING_AI_IKEY_VAR_VAL", + "isSecret": true + }, "INGRESS_LOAD_BALANCER_IP": { "value": "STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL" }, - "EXTERNAL_INGEST_FQDN": { "value": "STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -1165,6 +1171,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "PROD_AI_IKEY_VAR_VAL", + "isSecret": true + }, "INGRESS_LOAD_BALANCER_IP": { "value": "PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL" }, diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 2adb897a..696a0eba 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -4,10 +4,6 @@ "description": null, "isDeleted": false, "variables": { - "AI_IKEY": { - "value": "AI_IKEY_VAR_VAL", - "isSecret": true - }, "ACR_SERVER": { "value": "ACR_SERVER_VAR_VAL" }, @@ -32,6 +28,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "DEV_AI_IKEY_VAR_VAL", + "isSecret": true + }, "COSMOSDB_COL_NAME": { "value": "DEV_COSMOSDB_COL_NAME_VAR_VAL" }, @@ -401,6 +401,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "QA_AI_IKEY_VAR_VAL", + "isSecret": true + }, "COSMOSDB_COL_NAME": { "value": "QA_COSMOSDB_COL_NAME_VAR_VAL" }, @@ -766,6 +770,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "STAGING_AI_IKEY_VAR_VAL", + "isSecret": true + }, "COSMOSDB_COL_NAME": { "value": "STAGING_COSMOSDB_COL_NAME_VAR_VAL" }, @@ -1130,6 +1138,10 @@ "id": "AZURE_DEVOPS_USER_ID_VAR_VAL" }, "variables": { + "AI_IKEY": { + "value": "PROD_AI_IKEY_VAR_VAL", + "isSecret": true + }, "COSMOSDB_COL_NAME": { "value": "PROD_COSMOSDB_COL_NAME_VAR_VAL" }, From 11e9c4e7d9004b52a335148a12cbb9fa7899ec94 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 5 Jun 2019 13:09:39 +0000 Subject: [PATCH 171/246] Merged PR 788: drive redis tier by env rightsizing redis tier environment-based - dev/test: basic c0 - staging/prod: standard c1 | | dev | qa | staging | production | |-------------------------|:---------:|:---------:|:---------:|:----------:| | aks_std_d2v2_x3 | shared | shared | shared | shared | | acr_basic | shared | shared | shared | shared | | redis | ded_ba_c0 | ded_ba_c0 | ded_std_c1 | ded_std_c1 | | cosmosdb_1k_ru | dedicated | dedicated | dedicated | dedicated | | servicebus_std_1gig | dedicated | dedicated | dedicated | dedicated | | mongo_2.6k_ru | dedicated | dedicated | dedicated | dedicated | | stoaccount_std_lrs_v1 | dedicated | dedicated | dedicated | dedicated | | appinsights | dedicated | dedicated | dedicated | dedicated | | keyvault_std_x3 | dedicated | dedicated | dedicated | dedicated | solved: #9199 Related work items: #9199 --- azuredeploy.json | 55 +++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 41613548..f6662d2c 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -136,43 +136,6 @@ "description": "Type of the storage account that will store Redis Cache." } }, - "deliveryRedisCacheSKU": { - "type": "string", - "allowedValues": [ - "Basic", - "Standard", - "Premium" - ], - "defaultValue": "Basic", - "metadata": { - "description": "The pricing tier of the new Azure Redis Cache." - } - }, - "deliveryRedisCacheFamily": { - "type": "string", - "defaultValue": "C", - "metadata": { - "description": "The family for the sku." - }, - "allowedValues": [ - "C", - "P" - ] - }, - "deliveryRedisCacheCapacity": { - "type": "int", - "allowedValues": [ - 0, - 1, - 2, - 3, - 4 - ], - "defaultValue": 0, - "metadata": { - "description": "The size of the new Azure Redis Cache instance. " - } - }, "deliveryRedisDiagnosticsEnabled": { "type": "bool", "allowedValues": [ @@ -201,6 +164,9 @@ "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryRedisCacheSKU": "Basic", + "deliveryRedisCacheFamily": "C", + "deliveryRedisCacheCapacity": 0, "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", @@ -218,6 +184,9 @@ "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryRedisCacheSKU": "Basic", + "deliveryRedisCacheFamily": "C", + "deliveryRedisCacheCapacity": 0, "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", @@ -235,6 +204,9 @@ "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryRedisCacheSKU": "Standard", + "deliveryRedisCacheFamily": "C", + "deliveryRedisCacheCapacity": 1, "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", @@ -252,6 +224,9 @@ "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", + "deliveryRedisCacheSKU": "Standard", + "deliveryRedisCacheFamily": "C", + "deliveryRedisCacheCapacity": 1, "deliveryCosmosDbName": "[concat(parameters('environmentName'),'-d-', uniqueString(resourceGroup().id))]", "deliveryRedisName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", @@ -368,9 +343,9 @@ "properties": { "redisEnableNonSslPort": "false", "sku": { - "capacity": "[parameters('deliveryRedisCacheCapacity')]", - "family": "[parameters('deliveryRedisCacheFamily')]", - "name": "[parameters('deliveryRedisCacheSKU')]" + "capacity": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisCacheCapacity]", + "family": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisCacheFamily]", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisCacheSKU]" } }, "resources": [ From 0b0eee961bc31d3398d4ea585b9894ef4066d6b7 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 7 Jun 2019 21:42:55 +0000 Subject: [PATCH 172/246] Merged PR 800: folder structure change: loadtests folder structure change: loadtests - move src/shipping/ingress to src/loadtests - rename Fabrikam.Ingress.Ingestion.LoadTests -> Fabrikam.Shipping.Loadtests --- deployment.md | 2 +- .../Fabrikam.Shipping.LoadTests.sln} | 2 +- .../ContextParamLoadTestPlugin.cs | 2 +- .../DeliveryRequestLoadTest.loadtest | 4 ++-- .../DeliveryRequestWebTest.cs | 12 ++++++------ .../Fabrikam.Shipping.LoadTests.csproj} | 8 ++++---- .../Properties/AssemblyInfo.cs | 0 .../Fabrikam.Shipping.LoadTests}/packages.config | 0 ...ion-CreateDynamicDeliveryRequests.testsettings | 0 src/{shipping/ingress => loadtests}/readme.md | 15 ++++++++------- 10 files changed, 23 insertions(+), 22 deletions(-) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln => loadtests/Fabrikam.Shipping.LoadTests.sln} (85%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests => loadtests/Fabrikam.Shipping.LoadTests}/ContextParamLoadTestPlugin.cs (94%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests => loadtests/Fabrikam.Shipping.LoadTests}/DeliveryRequestLoadTest.loadtest (99%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests => loadtests/Fabrikam.Shipping.LoadTests}/DeliveryRequestWebTest.cs (95%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj => loadtests/Fabrikam.Shipping.LoadTests/Fabrikam.Shipping.LoadTests.csproj} (96%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests => loadtests/Fabrikam.Shipping.LoadTests}/Properties/AssemblyInfo.cs (100%) rename src/{shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests => loadtests/Fabrikam.Shipping.LoadTests}/packages.config (100%) rename src/{shipping/ingress => loadtests}/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings (100%) rename src/{shipping/ingress => loadtests}/readme.md (80%) diff --git a/deployment.md b/deployment.md index 312d280f..a82f836c 100644 --- a/deployment.md +++ b/deployment.md @@ -499,7 +499,7 @@ curl "https://$EXTERNAL_INGEST_FQDN/api/deliveries/mydelivery" --header 'Accept: ### Load Test the application -To run load testing against the solution, follow the steps listed [here](./src/shipping/ingress/readme.md). +To run load testing against the solution, follow the steps listed [here](./src/loadtests/readme.md). ### Fluentd and Elastic Search diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln b/src/loadtests/Fabrikam.Shipping.LoadTests.sln similarity index 85% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln rename to src/loadtests/Fabrikam.Shipping.LoadTests.sln index 149c43a9..5d5b4438 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests.sln +++ b/src/loadtests/Fabrikam.Shipping.LoadTests.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28714.193 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Ingress.Ingestion.LoadTests", "Fabrikam.Ingress.Ingestion.LoadTests\Fabrikam.Ingress.Ingestion.LoadTests.csproj", "{4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabrikam.Shipping.LoadTests", "Fabrikam.Shipping.LoadTests\Fabrikam.Shipping.LoadTests.csproj", "{4BD986A7-3F52-46FA-AA35-CD5D8AC532E0}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{AD4DC75F-D0D4-4ACA-B852-508F78588C4B}" ProjectSection(SolutionItems) = preProject diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs b/src/loadtests/Fabrikam.Shipping.LoadTests/ContextParamLoadTestPlugin.cs similarity index 94% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs rename to src/loadtests/Fabrikam.Shipping.LoadTests/ContextParamLoadTestPlugin.cs index 6899bf20..60c1247b 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/ContextParamLoadTestPlugin.cs +++ b/src/loadtests/Fabrikam.Shipping.LoadTests/ContextParamLoadTestPlugin.cs @@ -5,7 +5,7 @@ using Microsoft.VisualStudio.TestTools.LoadTesting; -namespace Fabrikam.Ingress.Ingestion.LoadTests +namespace Fabrikam.Shipping.LoadTests { public class ContextParameterLoadTestPlugin : ILoadTestPlugin { diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest b/src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestLoadTest.loadtest similarity index 99% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest rename to src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestLoadTest.loadtest index ad50786e..8999387f 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestLoadTest.loadtest +++ b/src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestLoadTest.loadtest @@ -468,6 +468,6 @@ - + - \ No newline at end of file + diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs b/src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestWebTest.cs similarity index 95% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs rename to src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestWebTest.cs index ce10d0c2..2f9236ca 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/DeliveryRequestWebTest.cs +++ b/src/loadtests/Fabrikam.Shipping.LoadTests/DeliveryRequestWebTest.cs @@ -11,7 +11,7 @@ using Microsoft.VisualStudio.TestTools.WebTesting; using Newtonsoft.Json; -namespace Fabrikam.Ingress.Ingestion.LoadTests +namespace Fabrikam.Shipping.LoadTests { public class DeliveryRequestWebTest : WebTest { @@ -29,9 +29,9 @@ public DeliveryRequestWebTest() } public static bool RemoteCertificateValidationCallBack( - object sender, - X509Certificate certificate, - X509Chain chain, + object sender, + X509Certificate certificate, + X509Chain chain, SslPolicyErrors sslPolicyErrors) => true; public override IEnumerator GetRequestEnumerator() @@ -78,7 +78,7 @@ private static StringHttpBody CreateRandomHttpBodyString() { Guid randomTag = Guid.NewGuid(); - var httpBodyRequestWithRandomTag = + var httpBodyRequestWithRandomTag = new StringHttpBody { ContentType = "application/json", @@ -106,4 +106,4 @@ private static StringHttpBody CreateRandomHttpBodyString() return httpBodyRequestWithRandomTag; } } -} \ No newline at end of file +} diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj b/src/loadtests/Fabrikam.Shipping.LoadTests/Fabrikam.Shipping.LoadTests.csproj similarity index 96% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj rename to src/loadtests/Fabrikam.Shipping.LoadTests/Fabrikam.Shipping.LoadTests.csproj index a7e24517..5c8aeda9 100644 --- a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Fabrikam.Ingress.Ingestion.LoadTests.csproj +++ b/src/loadtests/Fabrikam.Shipping.LoadTests/Fabrikam.Shipping.LoadTests.csproj @@ -9,8 +9,8 @@ {4BD986A7-3F52-46FA-AA35-CD5D8AC532E0} Library Properties - Fabrikam.Ingress.Ingestion.LoadTests - Fabrikam.Ingress.Ingestion.LoadTests + Fabrikam.Shipping.LoadTests + Fabrikam.Shipping.LoadTests v4.7.2 512 {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} @@ -84,11 +84,11 @@ - - \ No newline at end of file + diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs b/src/loadtests/Fabrikam.Shipping.LoadTests/Properties/AssemblyInfo.cs similarity index 100% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/Properties/AssemblyInfo.cs rename to src/loadtests/Fabrikam.Shipping.LoadTests/Properties/AssemblyInfo.cs diff --git a/src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config b/src/loadtests/Fabrikam.Shipping.LoadTests/packages.config similarity index 100% rename from src/shipping/ingress/Fabrikam.Ingress.Ingestion.LoadTests/packages.config rename to src/loadtests/Fabrikam.Shipping.LoadTests/packages.config diff --git a/src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings b/src/loadtests/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings similarity index 100% rename from src/shipping/ingress/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings rename to src/loadtests/FabrikamIngestion-CreateDynamicDeliveryRequests.testsettings diff --git a/src/shipping/ingress/readme.md b/src/loadtests/readme.md similarity index 80% rename from src/shipping/ingress/readme.md rename to src/loadtests/readme.md index 9d0c154d..8a6b9145 100644 --- a/src/shipping/ingress/readme.md +++ b/src/loadtests/readme.md @@ -1,7 +1,7 @@ ## Delivery Request Load Testing ### Prerequisites -a. Microsoft Visual Studio 2017 or later Enterprise Edition +a. Microsoft Visual Studio 2017 or later Enterprise Edition b. Web performance and load testing tools must be installed. For more information, please take a look at [Install the Load testing component](https://docs.microsoft.com/en-us/visualstudio/test/quickstart-create-a-load-test-project?view=vs-2017#install-the-load-testing-component) c. Azure DevOps subscription d. ```EXTERNAL_INGEST_FQDN``` value from [deployment instructions](../../../deployment.md) @@ -9,16 +9,17 @@ d. ```EXTERNAL_INGEST_FQDN``` value from [deployment instructions](../../../depl ### Run load testing in Azure DevOps 1. open Microsoft Visual Studio 2017 or later -2. navigate to Team Explorer and connect to your Azure DevOps subscription -3. double click on DeliveryRequestLoadTest.loadtest +2. open the solution ./Fabrikam.Shipping.LoadTests.sln +3. navigate to Team Explorer and connect to your Azure DevOps subscription +4. double click on DeliveryRequestLoadTest.loadtest > if the xml is open instead of the load test designer. Right click on .loadtest file from the Solution Explorer, and then click ```Open With``` and select Load Test Editor -4. double click ```INGEST_URL``` Context Parameter -5. from Properties panel, set its value using the environment variable ```EXTERNAL_INGEST_FQDN``` as shown below: +5. double click ```INGEST_URL``` Context Parameter +6. from Properties panel, set its value using the environment variable ```EXTERNAL_INGEST_FQDN``` as shown below: > https://```$EXTERNAL_INGEST_FQDN``` - + > Important: the url should look like, https://```.```.cloudapp.azure.com -6. click Run Load Test +7. click Run Load Test > if the ```EXTERNAL_INGEST_FQDN``` was not configured properly it is noticed from Test Results ```Graph``` because Total Requests metric is 0. Please check from ```Details``` looking for errors and ensure the check FQDN is well formed. From 0387584253d68d176cd01791cbcde210ef1bfce4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Jun 2019 16:19:27 +0000 Subject: [PATCH 173/246] Merged PR 793: CI/CD bug fixing For more information, please let's refer to commit messages where detailed information has been included. - [CI] add missing subscription name export - [CI] naming convention for repo var and remove missing vars - [CD] [package] remove fixed acr - [CD] [Blue-Green] [all] ignore errs on first time swapping and init helm (cd blue-green) - [CD] [Blue-Green] [nginx] stop removing nginx while doing blue-green (cd) - [CD] contemplate delivery ingress in CD pipeline changes: - [CD] [all] deploy default services in dev, qa and staging by default - [all] move subcharts for environments from charts to envs folder - [nginx] ensure nginx chart version is higher than 1.6.8 solved: #9201 Related work items: #9149, #9156, #9166, #9213 --- .../{charts => envs}/delivery-dev/Chart.yaml | 0 .../{charts => envs}/delivery-dev/values.yaml | 1 + .../{charts => envs}/delivery-prod/Chart.yaml | 0 .../templates/delivery-service.yaml | 5 + .../delivery-prod/values.yaml | 0 .../{charts => envs}/delivery-qa/Chart.yaml | 0 .../{charts => envs}/delivery-qa/values.yaml | 1 + .../delivery-staging/Chart.yaml | 0 .../delivery-staging/values.yaml | 1 + charts/delivery/requirements.yaml | 8 +- .../delivery/templates/delivery-service.yaml | 3 + .../dronescheduler-dev/Chart.yaml | 0 .../dronescheduler-dev/values.yaml | 1 + .../dronescheduler-prod/Chart.yaml | 0 .../templates/dronescheduler-service.yaml | 5 + .../dronescheduler-prod/values.yaml | 0 .../dronescheduler-qa/Chart.yaml | 0 .../dronescheduler-qa/values.yaml | 1 + .../dronescheduler-staging/Chart.yaml | 0 .../dronescheduler-staging/values.yaml | 1 + charts/dronescheduler/requirements.yaml | 8 +- .../templates/dronescheduler-service.yaml | 3 + .../{charts => envs}/ingestion-dev/Chart.yaml | 0 .../ingestion-dev/values.yaml | 1 + .../ingestion-prod/Chart.yaml | 0 .../templates/ingestion-service.yaml | 5 + .../ingestion-prod/values.yaml | 0 .../{charts => envs}/ingestion-qa/Chart.yaml | 0 .../{charts => envs}/ingestion-qa/values.yaml | 1 + .../ingestion-staging/Chart.yaml | 0 .../ingestion-staging/values.yaml | 1 + charts/ingestion/requirements.lock | 18 +- charts/ingestion/requirements.yaml | 10 +- .../templates/ingestion-service.yaml | 3 + .../{charts => envs}/package-dev/Chart.yaml | 0 .../{charts => envs}/package-dev/values.yaml | 1 + .../{charts => envs}/package-prod/Chart.yaml | 0 .../templates/package-service.yaml | 5 + .../{charts => envs}/package-prod/values.yaml | 0 .../{charts => envs}/package-qa/Chart.yaml | 0 .../{charts => envs}/package-qa/values.yaml | 1 + .../package-staging/Chart.yaml | 0 .../package-staging/values.yaml | 1 + charts/package/requirements.yaml | 8 +- charts/package/templates/package-service.yaml | 3 + .../{charts => envs}/workflow-dev/Chart.yaml | 0 .../{charts => envs}/workflow-dev/values.yaml | 0 .../{charts => envs}/workflow-prod/Chart.yaml | 0 .../workflow-prod/values.yaml | 0 .../{charts => envs}/workflow-qa/Chart.yaml | 0 .../{charts => envs}/workflow-qa/values.yaml | 0 .../workflow-staging/Chart.yaml | 0 .../workflow-staging/values.yaml | 0 charts/workflow/requirements.yaml | 8 +- deployment.md | 20 ++- deploymentCICD.md | 36 ++-- src/shipping/delivery/azure-pipelines-cd.json | 167 ++++++++++++++++-- src/shipping/delivery/azure-pipelines.yml | 20 +-- .../dronescheduler/azure-pipelines-cd.json | 96 +++++++++- .../dronescheduler/azure-pipelines.yml | 20 +-- .../ingestion/azure-pipelines-cd.json | 98 +++++++++- src/shipping/ingestion/azure-pipelines.yml | 20 +-- src/shipping/package/azure-pipelines-cd.json | 103 ++++++++++- src/shipping/package/azure-pipelines.yml | 22 +-- src/shipping/workflow/azure-pipelines.yml | 20 +-- 65 files changed, 564 insertions(+), 162 deletions(-) rename charts/delivery/{charts => envs}/delivery-dev/Chart.yaml (100%) rename charts/delivery/{charts => envs}/delivery-dev/values.yaml (91%) rename charts/delivery/{charts => envs}/delivery-prod/Chart.yaml (100%) rename charts/delivery/{charts => envs}/delivery-prod/templates/delivery-service.yaml (88%) rename charts/delivery/{charts => envs}/delivery-prod/values.yaml (100%) rename charts/delivery/{charts => envs}/delivery-qa/Chart.yaml (100%) rename charts/delivery/{charts => envs}/delivery-qa/values.yaml (91%) rename charts/delivery/{charts => envs}/delivery-staging/Chart.yaml (100%) rename charts/delivery/{charts => envs}/delivery-staging/values.yaml (91%) rename charts/dronescheduler/{charts => envs}/dronescheduler-dev/Chart.yaml (100%) rename charts/dronescheduler/{charts => envs}/dronescheduler-dev/values.yaml (91%) rename charts/dronescheduler/{charts => envs}/dronescheduler-prod/Chart.yaml (100%) rename charts/dronescheduler/{charts => envs}/dronescheduler-prod/templates/dronescheduler-service.yaml (88%) rename charts/dronescheduler/{charts => envs}/dronescheduler-prod/values.yaml (100%) rename charts/dronescheduler/{charts => envs}/dronescheduler-qa/Chart.yaml (100%) rename charts/dronescheduler/{charts => envs}/dronescheduler-qa/values.yaml (91%) rename charts/dronescheduler/{charts => envs}/dronescheduler-staging/Chart.yaml (100%) rename charts/dronescheduler/{charts => envs}/dronescheduler-staging/values.yaml (92%) rename charts/ingestion/{charts => envs}/ingestion-dev/Chart.yaml (100%) rename charts/ingestion/{charts => envs}/ingestion-dev/values.yaml (91%) rename charts/ingestion/{charts => envs}/ingestion-prod/Chart.yaml (100%) rename charts/ingestion/{charts => envs}/ingestion-prod/templates/ingestion-service.yaml (88%) rename charts/ingestion/{charts => envs}/ingestion-prod/values.yaml (100%) rename charts/ingestion/{charts => envs}/ingestion-qa/Chart.yaml (100%) rename charts/ingestion/{charts => envs}/ingestion-qa/values.yaml (91%) rename charts/ingestion/{charts => envs}/ingestion-staging/Chart.yaml (100%) rename charts/ingestion/{charts => envs}/ingestion-staging/values.yaml (93%) rename charts/package/{charts => envs}/package-dev/Chart.yaml (100%) rename charts/package/{charts => envs}/package-dev/values.yaml (90%) rename charts/package/{charts => envs}/package-prod/Chart.yaml (100%) rename charts/package/{charts => envs}/package-prod/templates/package-service.yaml (88%) rename charts/package/{charts => envs}/package-prod/values.yaml (100%) rename charts/package/{charts => envs}/package-qa/Chart.yaml (100%) rename charts/package/{charts => envs}/package-qa/values.yaml (90%) rename charts/package/{charts => envs}/package-staging/Chart.yaml (100%) rename charts/package/{charts => envs}/package-staging/values.yaml (91%) rename charts/workflow/{charts => envs}/workflow-dev/Chart.yaml (100%) rename charts/workflow/{charts => envs}/workflow-dev/values.yaml (100%) rename charts/workflow/{charts => envs}/workflow-prod/Chart.yaml (100%) rename charts/workflow/{charts => envs}/workflow-prod/values.yaml (100%) rename charts/workflow/{charts => envs}/workflow-qa/Chart.yaml (100%) rename charts/workflow/{charts => envs}/workflow-qa/values.yaml (100%) rename charts/workflow/{charts => envs}/workflow-staging/Chart.yaml (100%) rename charts/workflow/{charts => envs}/workflow-staging/values.yaml (100%) diff --git a/charts/delivery/charts/delivery-dev/Chart.yaml b/charts/delivery/envs/delivery-dev/Chart.yaml similarity index 100% rename from charts/delivery/charts/delivery-dev/Chart.yaml rename to charts/delivery/envs/delivery-dev/Chart.yaml diff --git a/charts/delivery/charts/delivery-dev/values.yaml b/charts/delivery/envs/delivery-dev/values.yaml similarity index 91% rename from charts/delivery/charts/delivery-dev/values.yaml rename to charts/delivery/envs/delivery-dev/values.yaml index 0ccafb8f..7935b27b 100644 --- a/charts/delivery/charts/delivery-dev/values.yaml +++ b/charts/delivery/envs/delivery-dev/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new dev deploy" + current: true diff --git a/charts/delivery/charts/delivery-prod/Chart.yaml b/charts/delivery/envs/delivery-prod/Chart.yaml similarity index 100% rename from charts/delivery/charts/delivery-prod/Chart.yaml rename to charts/delivery/envs/delivery-prod/Chart.yaml diff --git a/charts/delivery/charts/delivery-prod/templates/delivery-service.yaml b/charts/delivery/envs/delivery-prod/templates/delivery-service.yaml similarity index 88% rename from charts/delivery/charts/delivery-prod/templates/delivery-service.yaml rename to charts/delivery/envs/delivery-prod/templates/delivery-service.yaml index 46595d00..7e015d4f 100644 --- a/charts/delivery/charts/delivery-prod/templates/delivery-service.yaml +++ b/charts/delivery/envs/delivery-prod/templates/delivery-service.yaml @@ -6,6 +6,7 @@ ################################################################################################### # Delivery service ################################################################################################### +{{ if .Values.experimental }} {{- $appname := include "delivery.name" . -}} {{- $chart := include "delivery.chart" . -}} {{- $instancename := .Release.Name }} @@ -15,6 +16,9 @@ apiVersion: v1 kind: Service metadata: name: delivery-experimental + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -29,3 +33,4 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} +{{ end }} diff --git a/charts/delivery/charts/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml similarity index 100% rename from charts/delivery/charts/delivery-prod/values.yaml rename to charts/delivery/envs/delivery-prod/values.yaml diff --git a/charts/delivery/charts/delivery-qa/Chart.yaml b/charts/delivery/envs/delivery-qa/Chart.yaml similarity index 100% rename from charts/delivery/charts/delivery-qa/Chart.yaml rename to charts/delivery/envs/delivery-qa/Chart.yaml diff --git a/charts/delivery/charts/delivery-qa/values.yaml b/charts/delivery/envs/delivery-qa/values.yaml similarity index 91% rename from charts/delivery/charts/delivery-qa/values.yaml rename to charts/delivery/envs/delivery-qa/values.yaml index 433bc215..6cb3d57b 100644 --- a/charts/delivery/charts/delivery-qa/values.yaml +++ b/charts/delivery/envs/delivery-qa/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new qa deploy" + current: true diff --git a/charts/delivery/charts/delivery-staging/Chart.yaml b/charts/delivery/envs/delivery-staging/Chart.yaml similarity index 100% rename from charts/delivery/charts/delivery-staging/Chart.yaml rename to charts/delivery/envs/delivery-staging/Chart.yaml diff --git a/charts/delivery/charts/delivery-staging/values.yaml b/charts/delivery/envs/delivery-staging/values.yaml similarity index 91% rename from charts/delivery/charts/delivery-staging/values.yaml rename to charts/delivery/envs/delivery-staging/values.yaml index 0f4427e7..74532ab6 100644 --- a/charts/delivery/charts/delivery-staging/values.yaml +++ b/charts/delivery/envs/delivery-staging/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new staging deploy" + current: true diff --git a/charts/delivery/requirements.yaml b/charts/delivery/requirements.yaml index 9fd8c214..c73bd374 100644 --- a/charts/delivery/requirements.yaml +++ b/charts/delivery/requirements.yaml @@ -1,6 +1,6 @@ dependencies: - name: delivery-dev - repository: "file://charts/delivery-dev" + repository: "file://envs/delivery-dev" version: ">= 0.0.1" tags: - dev @@ -8,7 +8,7 @@ dependencies: - data - name: delivery-prod - repository: "file://charts/delivery-prod" + repository: "file://envs/delivery-prod" version: ">= 0.0.1" tags: - prod @@ -16,7 +16,7 @@ dependencies: - data - name: delivery-qa - repository: "file://charts/delivery-qa" + repository: "file://envs/delivery-qa" version: ">= 0.0.1" tags: - qa @@ -24,7 +24,7 @@ dependencies: - data - name: delivery-staging - repository: "file://charts/delivery-staging" + repository: "file://envs/delivery-staging" version: ">= 0.0.1" tags: - staging diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 4dd39004..6d1d7519 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -35,6 +35,9 @@ apiVersion: v1 kind: Service metadata: name: delivery + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops diff --git a/charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml similarity index 100% rename from charts/dronescheduler/charts/dronescheduler-dev/Chart.yaml rename to charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-dev/values.yaml b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml similarity index 91% rename from charts/dronescheduler/charts/dronescheduler-dev/values.yaml rename to charts/dronescheduler/envs/dronescheduler-dev/values.yaml index 0b4c2579..2f8bff92 100644 --- a/charts/dronescheduler/charts/dronescheduler-dev/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new dev deploy" + current: true diff --git a/charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml similarity index 100% rename from charts/dronescheduler/charts/dronescheduler-prod/Chart.yaml rename to charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml b/charts/dronescheduler/envs/dronescheduler-prod/templates/dronescheduler-service.yaml similarity index 88% rename from charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml rename to charts/dronescheduler/envs/dronescheduler-prod/templates/dronescheduler-service.yaml index 48c10e93..a933866b 100644 --- a/charts/dronescheduler/charts/dronescheduler-prod/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/templates/dronescheduler-service.yaml @@ -6,6 +6,7 @@ ################################################################################################### # Dronescheduler service experimental ################################################################################################### +{{ if .Values.experimental }} {{- $appname := include "dronescheduler.name" . -}} {{- $chart := include "dronescheduler.chart" . -}} {{- $instancename := .Release.Name }} @@ -15,6 +16,9 @@ apiVersion: v1 kind: Service metadata: name: dronescheduler-experimental + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -29,3 +33,4 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} +{{ end }} diff --git a/charts/dronescheduler/charts/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml similarity index 100% rename from charts/dronescheduler/charts/dronescheduler-prod/values.yaml rename to charts/dronescheduler/envs/dronescheduler-prod/values.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml similarity index 100% rename from charts/dronescheduler/charts/dronescheduler-qa/Chart.yaml rename to charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-qa/values.yaml b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml similarity index 91% rename from charts/dronescheduler/charts/dronescheduler-qa/values.yaml rename to charts/dronescheduler/envs/dronescheduler-qa/values.yaml index 83db9d26..80584f16 100644 --- a/charts/dronescheduler/charts/dronescheduler-qa/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new qa deploy" + current: true diff --git a/charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml similarity index 100% rename from charts/dronescheduler/charts/dronescheduler-staging/Chart.yaml rename to charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml diff --git a/charts/dronescheduler/charts/dronescheduler-staging/values.yaml b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml similarity index 92% rename from charts/dronescheduler/charts/dronescheduler-staging/values.yaml rename to charts/dronescheduler/envs/dronescheduler-staging/values.yaml index d6deacb9..aa43bb50 100644 --- a/charts/dronescheduler/charts/dronescheduler-staging/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "Information" reason: "new staging deploy" + current: true diff --git a/charts/dronescheduler/requirements.yaml b/charts/dronescheduler/requirements.yaml index 9fdc5a0b..dc951129 100644 --- a/charts/dronescheduler/requirements.yaml +++ b/charts/dronescheduler/requirements.yaml @@ -1,6 +1,6 @@ dependencies: - name: dronescheduler-dev - repository: "file://charts/dronescheduler-dev" + repository: "file://envs/dronescheduler-dev" version: ">= 0.0.1" tags: - dev @@ -8,7 +8,7 @@ dependencies: - data - name: dronescheduler-prod - repository: "file://charts/dronescheduler-prod" + repository: "file://envs/dronescheduler-prod" version: ">= 0.0.1" tags: - prod @@ -16,7 +16,7 @@ dependencies: - data - name: dronescheduler-qa - repository: "file://charts/dronescheduler-qa" + repository: "file://envs/dronescheduler-qa" version: ">= 0.0.1" tags: - qa @@ -24,7 +24,7 @@ dependencies: - data - name: dronescheduler-staging - repository: "file://charts/dronescheduler-staging" + repository: "file://envs/dronescheduler-staging" version: ">= 0.0.1" tags: - staging diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index bb8c149b..f347d1bb 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -35,6 +35,9 @@ apiVersion: v1 kind: Service metadata: name: dronescheduler + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops diff --git a/charts/ingestion/charts/ingestion-dev/Chart.yaml b/charts/ingestion/envs/ingestion-dev/Chart.yaml similarity index 100% rename from charts/ingestion/charts/ingestion-dev/Chart.yaml rename to charts/ingestion/envs/ingestion-dev/Chart.yaml diff --git a/charts/ingestion/charts/ingestion-dev/values.yaml b/charts/ingestion/envs/ingestion-dev/values.yaml similarity index 91% rename from charts/ingestion/charts/ingestion-dev/values.yaml rename to charts/ingestion/envs/ingestion-dev/values.yaml index 8eb41d5d..53ea4f1d 100644 --- a/charts/ingestion/charts/ingestion-dev/values.yaml +++ b/charts/ingestion/envs/ingestion-dev/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "info" reason: "new dev deploy" + current: true diff --git a/charts/ingestion/charts/ingestion-prod/Chart.yaml b/charts/ingestion/envs/ingestion-prod/Chart.yaml similarity index 100% rename from charts/ingestion/charts/ingestion-prod/Chart.yaml rename to charts/ingestion/envs/ingestion-prod/Chart.yaml diff --git a/charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml b/charts/ingestion/envs/ingestion-prod/templates/ingestion-service.yaml similarity index 88% rename from charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml rename to charts/ingestion/envs/ingestion-prod/templates/ingestion-service.yaml index e03157e8..edfa720d 100644 --- a/charts/ingestion/charts/ingestion-prod/templates/ingestion-service.yaml +++ b/charts/ingestion/envs/ingestion-prod/templates/ingestion-service.yaml @@ -6,6 +6,7 @@ ################################################################################################### # ingestion service experimental ################################################################################################### +{{ if .Values.experimental }} {{- $appname := include "ingestion.name" . -}} {{- $chart := include "ingestion.chart" . -}} {{- $instancename := .Release.Name }} @@ -15,6 +16,9 @@ apiVersion: v1 kind: Service metadata: name: ingestion-experimental + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -29,3 +33,4 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} +{{ end }} diff --git a/charts/ingestion/charts/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml similarity index 100% rename from charts/ingestion/charts/ingestion-prod/values.yaml rename to charts/ingestion/envs/ingestion-prod/values.yaml diff --git a/charts/ingestion/charts/ingestion-qa/Chart.yaml b/charts/ingestion/envs/ingestion-qa/Chart.yaml similarity index 100% rename from charts/ingestion/charts/ingestion-qa/Chart.yaml rename to charts/ingestion/envs/ingestion-qa/Chart.yaml diff --git a/charts/ingestion/charts/ingestion-qa/values.yaml b/charts/ingestion/envs/ingestion-qa/values.yaml similarity index 91% rename from charts/ingestion/charts/ingestion-qa/values.yaml rename to charts/ingestion/envs/ingestion-qa/values.yaml index bdcc0af6..105b36e4 100644 --- a/charts/ingestion/charts/ingestion-qa/values.yaml +++ b/charts/ingestion/envs/ingestion-qa/values.yaml @@ -8,3 +8,4 @@ exports: telemetry: level: "info" reason: "new qa deploy" + current: true diff --git a/charts/ingestion/charts/ingestion-staging/Chart.yaml b/charts/ingestion/envs/ingestion-staging/Chart.yaml similarity index 100% rename from charts/ingestion/charts/ingestion-staging/Chart.yaml rename to charts/ingestion/envs/ingestion-staging/Chart.yaml diff --git a/charts/ingestion/charts/ingestion-staging/values.yaml b/charts/ingestion/envs/ingestion-staging/values.yaml similarity index 93% rename from charts/ingestion/charts/ingestion-staging/values.yaml rename to charts/ingestion/envs/ingestion-staging/values.yaml index c42d1d88..e43ac83d 100644 --- a/charts/ingestion/charts/ingestion-staging/values.yaml +++ b/charts/ingestion/envs/ingestion-staging/values.yaml @@ -11,3 +11,4 @@ exports: nginx-ingress: controller: replicaCount: 2 + current: true diff --git a/charts/ingestion/requirements.lock b/charts/ingestion/requirements.lock index 4de12479..c10856a9 100644 --- a/charts/ingestion/requirements.lock +++ b/charts/ingestion/requirements.lock @@ -1,6 +1,18 @@ dependencies: - name: nginx-ingress repository: https://kubernetes-charts.storage.googleapis.com - version: 1.6.5 -digest: sha256:ac4894123e00ec6c55fa3cd2a659a66b13882fbe825706afb4613a2a8f507bcf -generated: 2019-05-17T12:50:32.882555938-03:00 + version: 1.6.15 +- name: ingestion-dev + repository: file://envs/ingestion-dev + version: v0.1.0 +- name: ingestion-prod + repository: file://envs/ingestion-prod + version: v0.1.0 +- name: ingestion-qa + repository: file://envs/ingestion-qa + version: v0.1.0 +- name: ingestion-staging + repository: file://envs/ingestion-staging + version: v0.1.0 +digest: sha256:c2cdc67015f90e1ff33c49603a6ece3f2a99dd8e28e88e8756ef99041820b47a +generated: 2019-05-29T10:41:19.509573491-03:00 diff --git a/charts/ingestion/requirements.yaml b/charts/ingestion/requirements.yaml index 2b7b48a2..ad03c9b6 100644 --- a/charts/ingestion/requirements.yaml +++ b/charts/ingestion/requirements.yaml @@ -1,11 +1,11 @@ dependencies: - name: nginx-ingress repository: https://kubernetes-charts.storage.googleapis.com - version: ">=1.6.0" + version: ">=1.6.9" condition: nginx-ingress.enabled - name: ingestion-dev - repository: "file://charts/ingestion-dev" + repository: "file://envs/ingestion-dev" version: ">= 0.0.1" tags: - dev @@ -13,7 +13,7 @@ dependencies: - data - name: ingestion-prod - repository: "file://charts/ingestion-prod" + repository: "file://envs/ingestion-prod" version: ">= 0.0.1" tags: - prod @@ -21,7 +21,7 @@ dependencies: - data - name: ingestion-qa - repository: "file://charts/ingestion-qa" + repository: "file://envs/ingestion-qa" version: ">= 0.0.1" tags: - qa @@ -29,7 +29,7 @@ dependencies: - data - name: ingestion-staging - repository: "file://charts/ingestion-staging" + repository: "file://envs/ingestion-staging" version: ">= 0.0.1" tags: - staging diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index e0e7f32b..740aba5f 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -35,6 +35,9 @@ apiVersion: v1 kind: Service metadata: name: ingestion + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops diff --git a/charts/package/charts/package-dev/Chart.yaml b/charts/package/envs/package-dev/Chart.yaml similarity index 100% rename from charts/package/charts/package-dev/Chart.yaml rename to charts/package/envs/package-dev/Chart.yaml diff --git a/charts/package/charts/package-dev/values.yaml b/charts/package/envs/package-dev/values.yaml similarity index 90% rename from charts/package/charts/package-dev/values.yaml rename to charts/package/envs/package-dev/values.yaml index d9d1e84c..9cf0ef2c 100644 --- a/charts/package/charts/package-dev/values.yaml +++ b/charts/package/envs/package-dev/values.yaml @@ -8,3 +8,4 @@ exports: log: level: "info" reason: "new dev deploy" + current: true diff --git a/charts/package/charts/package-prod/Chart.yaml b/charts/package/envs/package-prod/Chart.yaml similarity index 100% rename from charts/package/charts/package-prod/Chart.yaml rename to charts/package/envs/package-prod/Chart.yaml diff --git a/charts/package/charts/package-prod/templates/package-service.yaml b/charts/package/envs/package-prod/templates/package-service.yaml similarity index 88% rename from charts/package/charts/package-prod/templates/package-service.yaml rename to charts/package/envs/package-prod/templates/package-service.yaml index e2f8ee4d..8dbaabc2 100644 --- a/charts/package/charts/package-prod/templates/package-service.yaml +++ b/charts/package/envs/package-prod/templates/package-service.yaml @@ -6,6 +6,7 @@ ################################################################################################### # package service experimental ################################################################################################### +{{ if .Values.experimental }} {{- $appname := include "package.name" . -}} {{- $chart := include "package.chart" . -}} {{- $instancename := .Release.Name }} @@ -15,6 +16,9 @@ apiVersion: v1 kind: Service metadata: name: package-experimental + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops @@ -29,3 +33,4 @@ spec: selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} +{{ end }} diff --git a/charts/package/charts/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml similarity index 100% rename from charts/package/charts/package-prod/values.yaml rename to charts/package/envs/package-prod/values.yaml diff --git a/charts/package/charts/package-qa/Chart.yaml b/charts/package/envs/package-qa/Chart.yaml similarity index 100% rename from charts/package/charts/package-qa/Chart.yaml rename to charts/package/envs/package-qa/Chart.yaml diff --git a/charts/package/charts/package-qa/values.yaml b/charts/package/envs/package-qa/values.yaml similarity index 90% rename from charts/package/charts/package-qa/values.yaml rename to charts/package/envs/package-qa/values.yaml index 5f8b06e7..04ec88c0 100644 --- a/charts/package/charts/package-qa/values.yaml +++ b/charts/package/envs/package-qa/values.yaml @@ -8,3 +8,4 @@ exports: log: level: "info" reason: "new qa deploy" + current: true diff --git a/charts/package/charts/package-staging/Chart.yaml b/charts/package/envs/package-staging/Chart.yaml similarity index 100% rename from charts/package/charts/package-staging/Chart.yaml rename to charts/package/envs/package-staging/Chart.yaml diff --git a/charts/package/charts/package-staging/values.yaml b/charts/package/envs/package-staging/values.yaml similarity index 91% rename from charts/package/charts/package-staging/values.yaml rename to charts/package/envs/package-staging/values.yaml index 5b7777a8..1cb8b548 100644 --- a/charts/package/charts/package-staging/values.yaml +++ b/charts/package/envs/package-staging/values.yaml @@ -8,3 +8,4 @@ exports: log: level: "info" reason: "new staging deploy" + current: true diff --git a/charts/package/requirements.yaml b/charts/package/requirements.yaml index debe028e..fbee7424 100644 --- a/charts/package/requirements.yaml +++ b/charts/package/requirements.yaml @@ -1,6 +1,6 @@ dependencies: - name: package-dev - repository: "file://charts/package-dev" + repository: "file://envs/package-dev" version: ">= 0.0.1" tags: - dev @@ -8,7 +8,7 @@ dependencies: - data - name: package-prod - repository: "file://charts/package-prod" + repository: "file://envs/package-prod" version: ">= 0.0.1" tags: - prod @@ -16,7 +16,7 @@ dependencies: - data - name: package-qa - repository: "file://charts/package-qa" + repository: "file://envs/package-qa" version: ">= 0.0.1" tags: - qa @@ -24,7 +24,7 @@ dependencies: - data - name: package-staging - repository: "file://charts/package-staging" + repository: "file://envs/package-staging" version: ">= 0.0.1" tags: - staging diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index 998b58c7..7be0cde0 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -35,6 +35,9 @@ apiVersion: v1 kind: Service metadata: name: package + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation labels: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/managed-by: azuredeveops diff --git a/charts/workflow/charts/workflow-dev/Chart.yaml b/charts/workflow/envs/workflow-dev/Chart.yaml similarity index 100% rename from charts/workflow/charts/workflow-dev/Chart.yaml rename to charts/workflow/envs/workflow-dev/Chart.yaml diff --git a/charts/workflow/charts/workflow-dev/values.yaml b/charts/workflow/envs/workflow-dev/values.yaml similarity index 100% rename from charts/workflow/charts/workflow-dev/values.yaml rename to charts/workflow/envs/workflow-dev/values.yaml diff --git a/charts/workflow/charts/workflow-prod/Chart.yaml b/charts/workflow/envs/workflow-prod/Chart.yaml similarity index 100% rename from charts/workflow/charts/workflow-prod/Chart.yaml rename to charts/workflow/envs/workflow-prod/Chart.yaml diff --git a/charts/workflow/charts/workflow-prod/values.yaml b/charts/workflow/envs/workflow-prod/values.yaml similarity index 100% rename from charts/workflow/charts/workflow-prod/values.yaml rename to charts/workflow/envs/workflow-prod/values.yaml diff --git a/charts/workflow/charts/workflow-qa/Chart.yaml b/charts/workflow/envs/workflow-qa/Chart.yaml similarity index 100% rename from charts/workflow/charts/workflow-qa/Chart.yaml rename to charts/workflow/envs/workflow-qa/Chart.yaml diff --git a/charts/workflow/charts/workflow-qa/values.yaml b/charts/workflow/envs/workflow-qa/values.yaml similarity index 100% rename from charts/workflow/charts/workflow-qa/values.yaml rename to charts/workflow/envs/workflow-qa/values.yaml diff --git a/charts/workflow/charts/workflow-staging/Chart.yaml b/charts/workflow/envs/workflow-staging/Chart.yaml similarity index 100% rename from charts/workflow/charts/workflow-staging/Chart.yaml rename to charts/workflow/envs/workflow-staging/Chart.yaml diff --git a/charts/workflow/charts/workflow-staging/values.yaml b/charts/workflow/envs/workflow-staging/values.yaml similarity index 100% rename from charts/workflow/charts/workflow-staging/values.yaml rename to charts/workflow/envs/workflow-staging/values.yaml diff --git a/charts/workflow/requirements.yaml b/charts/workflow/requirements.yaml index 10232128..5971dee1 100644 --- a/charts/workflow/requirements.yaml +++ b/charts/workflow/requirements.yaml @@ -1,6 +1,6 @@ dependencies: - name: workflow-dev - repository: "file://charts/workflow-dev" + repository: "file://envs/workflow-dev" version: ">= 0.0.1" tags: - dev @@ -8,7 +8,7 @@ dependencies: - data - name: workflow-prod - repository: "file://charts/workflow-prod" + repository: "file://envs/workflow-prod" version: ">= 0.0.1" tags: - prod @@ -16,7 +16,7 @@ dependencies: - data - name: workflow-qa - repository: "file://charts/workflow-qa" + repository: "file://envs/workflow-qa" version: ">= 0.0.1" tags: - qa @@ -24,7 +24,7 @@ dependencies: - data - name: workflow-staging - repository: "file://charts/workflow-staging" + repository: "file://envs/workflow-staging" version: ">= 0.0.1" tags: - staging diff --git a/deployment.md b/deployment.md index a82f836c..80c60da0 100644 --- a/deployment.md +++ b/deployment.md @@ -37,6 +37,7 @@ export LOCATION=[YOUR_LOCATION_HERE] export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE] export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export SUBSCRIPTION_NAME=$(az account show --query name --output tsv) export TENANT_ID=$(az account show --query tenantId --output tsv) export PROJECT_ROOT=./microservices-reference-implementation @@ -245,9 +246,9 @@ helm install $HELM_CHARTS/delivery/ \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ --set reason="Initial deployment" \ --set tags.dev=true \ - --set current=true \ --namespace backend-dev \ - --name delivery-v0.1.0-dev + --name delivery-v0.1.0-dev \ + --dep-up # Verify the pod is created helm status delivery-v0.1.0 @@ -292,9 +293,9 @@ helm install $HELM_CHARTS/package/ \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ --set tags.dev=true \ - --set current=true \ --namespace backend-dev \ - --name package-v0.1.0-dev + --name package-v0.1.0-dev \ + --dep-up # Verify the pod is created helm status package-v0.1.0 @@ -346,7 +347,8 @@ helm install $HELM_CHARTS/workflow/ \ --set reason="Initial deployment" \ --set tags.dev=true \ --namespace backend-dev \ - --name workflow-v0.1.0-dev + --name workflow-v0.1.0-dev \ + --dep-up # Verify the pod is created helm status workflow-v0.1.0 @@ -401,9 +403,9 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ --set reason="Initial deployment" \ --set tags.dev=true \ - --set current=true \ --namespace backend-dev \ - --name ingestion-v0.1.0-dev + --name ingestion-v0.1.0-dev \ + --dep-up # Verify the pod is created helm status ingestion-v0.1.0 @@ -454,9 +456,9 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --set reason="Initial deployment" \ --set tags.dev=true \ - --set current=true \ --namespace backend-dev \ - --name dronescheduler-v0.1.0-dev + --name dronescheduler-v0.1.0-dev \ + --dep-up # Verify the pod is created helm status dronescheduler-v0.1.0 diff --git a/deploymentCICD.md b/deploymentCICD.md index 61b2f398..ac6370ad 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -116,7 +116,7 @@ AZURE_DEVOPS_ORG_NAME= AZURE_DEVOPS_ORG=https://dev.azure.com/$AZURE_DEVOPS_ORG_NAME AZURE_DEVOPS_VSRM_ORG=https://vsrm.dev.azure.com/$AZURE_DEVOPS_ORG_NAME AZURE_DEVOPS_PROJECT_NAME= -AZURE_REPOS_NAME= +AZURE_DEVOPS_REPOS_NAME= AZURE_PIPELINES_SERVICE_CONN_NAME=default_cicd_service-connection # create project or skip this step if you are using an existent Azure DevOps project @@ -126,7 +126,7 @@ az devops project create \ # create repo az repos create \ - --name $AZURE_REPOS_NAME \ + --name $AZURE_DEVOPS_REPOS_NAME \ --organization $AZURE_DEVOPS_ORG \ --project $AZURE_DEVOPS_PROJECT_NAME @@ -138,16 +138,15 @@ az devops service-endpoint create \ --service-endpoint-type azurerm \ --name $AZURE_PIPELINES_SERVICE_CONN_NAME \ --authorization-scheme ServicePrincipal \ - --azure-rm-service-principal-id $SERVICE_PRINCIPAL_ID \ - --azure-rm-service-prinicipal-key $SERVICE_PRINCIPAL_KEY \ --azure-rm-tenant-id $TENANT_ID \ --azure-rm-subscription-id $SUBSCRIPTION_ID \ --azure-rm-subscription-name "$SUBSCRIPTION_NAME" \ --organization $AZURE_DEVOPS_ORG \ - --project $AZURE_DEVOPS_PROJECT_NAME + --project $AZURE_DEVOPS_PROJECT_NAME \ + --azure-rm-service-principal-id --azure-rm-service-principal-key # navigate to the repo and add ssh following links below or just skip this step for https -open $AZURE_DEVOPS_ORG/$AZURE_DEVOPS_PROJECT_NAME/_git/$AZURE_REPOS_NAME +open $AZURE_DEVOPS_ORG/$AZURE_DEVOPS_PROJECT_NAME/_git/$AZURE_DEVOPS_REPOS_NAME ``` > Follow instructions at [Use SSH Key authentication](https://docs.microsoft.com/en-us/azure/devops/repos/git/use-ssh-keys-to-authenticate?view=azure-devops) to add ssh. @@ -158,7 +157,7 @@ For more information on the different authentication types, please take a look a ## Add new remote for the new Azure Repo ``` # get the ssh url. For https just replace sshUrl with remoteUrl below -export NEW_REMOTE=$(az repos show -r $AZURE_REPOS_NAME --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query sshUrl -o tsv) +export NEW_REMOTE=$(az repos show -r $AZURE_DEVOPS_REPOS_NAME --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query sshUrl -o tsv) # push master from cloned repo to the new remote cd && \ @@ -215,8 +214,6 @@ export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler # configure build YAML definitions for pipelinePath in $DELIVERY_PATH $PACKAGE_PATH $WORKFLOW_PATH $INGESTION_PATH $DRONE_PATH; do sed -i \ - -e "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" \ - -e "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" \ -e "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" \ -e "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" \ -e "s#AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL#$AZURE_PIPELINES_SERVICE_CONN_NAME#g" \ @@ -255,7 +252,8 @@ export ${ENV}_COLLECTION_NAME="deliveries-col" export ${ENV}_DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.deliveryKeyVaultUri.value -o tsv) export ${ENV}_DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) export ${ENV}_DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalClientId.value -o tsv) -done +done && \ +export INGRESS_TLS_SECRET_NAME=delivery-ingress-tls # add relese definition cat $DELIVERY_PATH/azure-pipelines-cd.json | \ @@ -275,24 +273,40 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#DEV_DATABASE_NAME_VAR_VAL#$DEV_DATABASE_NAME#g" | \ sed "s#DEV_COLLECTION_NAME_VAR_VAL#$DEV_COLLECTION_NAME#g" | \ sed "s#DEV_DELIVERY_KEYVAULT_URI_VAR_VAL#$DEV_DELIVERY_KEYVAULT_URI#g" | \ + sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$DEV_EXTERNAL_INGEST_FQDN#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$DEV_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # qa resources sed "s#QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#QA_DATABASE_NAME_VAR_VAL#$QA_DATABASE_NAME#g" | \ sed "s#QA_COLLECTION_NAME_VAR_VAL#$QA_COLLECTION_NAME#g" | \ sed "s#QA_DELIVERY_KEYVAULT_URI_VAR_VAL#$QA_DELIVERY_KEYVAULT_URI#g" | \ + sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$QA_EXTERNAL_INGEST_FQDN#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$QA_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # staging resources sed "s#STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#STAGING_DATABASE_NAME_VAR_VAL#$STAGING_DATABASE_NAME#g" | \ sed "s#STAGING_COLLECTION_NAME_VAR_VAL#$STAGING_COLLECTION_NAME#g" | \ sed "s#STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL#$STAGING_DELIVERY_KEYVAULT_URI#g" | \ + sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$STAGING_EXTERNAL_INGEST_FQDN#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # production resources sed "s#PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#PROD_DATABASE_NAME_VAR_VAL#$PROD_DATABASE_NAME#g" | \ sed "s#PROD_COLLECTION_NAME_VAR_VAL#$PROD_COLLECTION_NAME#g" | \ - sed "s#PROD_DELIVERY_KEYVAULT_URI_VAR_VAL#$PROD_DELIVERY_KEYVAULT_URI#g" \ + sed "s#PROD_DELIVERY_KEYVAULT_URI_VAR_VAL#$PROD_DELIVERY_KEYVAULT_URI#g" | \ + sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ + sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" \ > $DELIVERY_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index b16d88a2..c9a62049 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -16,6 +16,9 @@ }, "ACR_NAME": { "value": "ACR_NAME_VAR_VAL" + }, + "SERVICE_NAME": { + "value": "delivery" } }, "variableGroups": [], @@ -42,7 +45,20 @@ }, "DELIVERY_KEYVAULT_URI": { "value": "DEV_DELIVERY_KEYVAULT_URI_VAR_VAL" - } + }, + "EXTERNAL_INGEST_FQDN": { + "value": "DEV_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -232,7 +248,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -420,7 +436,20 @@ }, "DELIVERY_KEYVAULT_URI": { "value": "QA_DELIVERY_KEYVAULT_URI_VAR_VAL" - } + }, + "EXTERNAL_INGEST_FQDN": { + "value": "QA_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "QA_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "QA_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "QA_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -610,7 +639,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -793,7 +822,20 @@ }, "DELIVERY_KEYVAULT_URI": { "value": "STAGING_DELIVERY_KEYVAULT_URI_VAR_VAL" - } + }, + "EXTERNAL_INGEST_FQDN": { + "value": "STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -983,7 +1025,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1165,7 +1207,20 @@ }, "DELIVERY_KEYVAULT_URI": { "value": "PROD_DELIVERY_KEYVAULT_URI_VAR_VAL" - } + }, + "EXTERNAL_INGEST_FQDN": { + "value": "PROD_EXTERNAL_INGEST_FQDN_VAR_VAL" + }, + "INGRESS_TLS_SECRET_CERT": { + "value": "PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL" + }, + "INGRESS_TLS_SECRET_KEY": { + "value": "PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL", + "isSecret": true + }, + "INGRESS_TLS_SECRET_NAME": { + "value": "PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -1499,7 +1554,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,delivery-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1612,7 +1667,7 @@ "configurationType": "configuration", "configuration": "", "inline": "", - "arguments": "-n backend svc/delivery -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "arguments": "-n backend svc/delivery -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\" --ignore-not-found=true", "secretType": "dockerRegistry", "secretArguments": "", "containerRegistryType": "Azure Container Registry", @@ -1634,6 +1689,98 @@ "outputFormat": "" } }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, { "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", @@ -1708,7 +1855,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml index d0a9be12..fad33c12 100644 --- a/src/shipping/delivery/azure-pipelines.yml +++ b/src/shipping/delivery/azure-pipelines.yml @@ -6,8 +6,6 @@ variables: azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL azureContainerRegistry: ACR_SERVER_VAR_VAL azureContainerRegistryName: ACR_NAME_VAR_VAL - azureKubernetesCluster: CLUSTER_NAME_VAR_VAL - resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) @@ -128,22 +126,6 @@ jobs: checkLatestKubectl: false - - task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(azureSubscription) - - azureResourceGroup: $(resourceGroup) - - kubernetesCluster: $(azureKubernetesCluster) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - - task: HelmDeploy@0 condition: eq(variables['fullCI'],True) displayName: 'helm package' @@ -154,7 +136,7 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName)' + arguments: '--app-version $(Build.SourceBranchName) --dependency-update' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index db331a07..511ca99d 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -1468,7 +1468,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,dronescheduler-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1581,7 +1581,7 @@ "configurationType": "configuration", "configuration": "", "inline": "", - "arguments": "-n backend svc/dronescheduler -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "arguments": "-n backend svc/dronescheduler -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\" --ignore-not-found=true", "secretType": "dockerRegistry", "secretArguments": "", "containerRegistryType": "Azure Container Registry", @@ -1603,6 +1603,98 @@ "outputFormat": "" } }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, { "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", diff --git a/src/shipping/dronescheduler/azure-pipelines.yml b/src/shipping/dronescheduler/azure-pipelines.yml index 17260999..a3619bb9 100644 --- a/src/shipping/dronescheduler/azure-pipelines.yml +++ b/src/shipping/dronescheduler/azure-pipelines.yml @@ -7,8 +7,6 @@ variables: azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL azureContainerRegistry: ACR_SERVER_VAR_VAL azureContainerRegistryName: ACR_NAME_VAR_VAL - azureKubernetesCluster: CLUSTER_NAME_VAR_VAL - resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) @@ -92,22 +90,6 @@ jobs: checkLatestKubectl: false - - task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(azureSubscription) - - azureResourceGroup: $(ResourceGroup) - - kubernetesCluster: $(azureKubernetesCluster) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - - task: HelmDeploy@0 condition: eq(variables['fullCI'],True) displayName: 'helm package' @@ -118,7 +100,7 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName)' + arguments: '--app-version $(Build.SourceBranchName) --dependency-update' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 89ef1387..837752e5 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -1527,7 +1527,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1633,7 +1633,7 @@ "configurationType": "configuration", "configuration": "", "inline": "", - "arguments": "-n backend svc/ingestion -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "arguments": "-n backend svc/ingestion -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\" --ignore-not-found=true", "secretType": "dockerRegistry", "secretArguments": "", "containerRegistryType": "Azure Container Registry", @@ -1655,6 +1655,98 @@ "outputFormat": "" } }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, { "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", @@ -1729,7 +1821,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/ingestion/azure-pipelines.yml b/src/shipping/ingestion/azure-pipelines.yml index f7010ae4..7a4040cc 100644 --- a/src/shipping/ingestion/azure-pipelines.yml +++ b/src/shipping/ingestion/azure-pipelines.yml @@ -6,8 +6,6 @@ variables: azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL azureContainerRegistry: ACR_SERVER_VAR_VAL azureContainerRegistryName: ACR_NAME_VAR_VAL - azureKubernetesCluster: CLUSTER_NAME_VAR_VAL - resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) @@ -128,22 +126,6 @@ jobs: checkLatestKubectl: false - - task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(azureSubscription) - - azureresourceGroup: $(resourceGroup) - - kubernetesCluster: $(azureKubernetesCluster) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - - task: HelmDeploy@0 condition: eq(variables['fullCI'],True) displayName: 'helm package' @@ -154,7 +136,7 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName)' + arguments: '--app-version $(Build.SourceBranchName) --dependency-update' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 696a0eba..7fa6205e 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -413,7 +413,6 @@ "isSecret": true } }, - "variables": {}, "variableGroups": [], "preDeployApprovals": { "approvals": [ @@ -1241,7 +1240,7 @@ "containerregistrytype": "Azure Container Registry", "dockerRegistryEndpoint": "", "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", "command": "pull", "dockerFile": "**/Dockerfile", "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", @@ -1288,7 +1287,7 @@ "containerregistrytype": "Azure Container Registry", "dockerRegistryEndpoint": "", "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", "command": "Tag image", "dockerFile": "**/Dockerfile", "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", @@ -1335,7 +1334,7 @@ "containerregistrytype": "Azure Container Registry", "dockerRegistryEndpoint": "", "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "pnpaksraacr.azurecr.io", + "azureContainerRegistry": "ACR_SERVER_VAR_VAL", "command": "Push an image", "dockerFile": "**/Dockerfile", "arguments": "", @@ -1483,7 +1482,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true,package-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1596,7 +1595,7 @@ "configurationType": "configuration", "configuration": "", "inline": "", - "arguments": "-n backend svc/package -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\"", + "arguments": "-n backend svc/package -o \"jsonpath={.spec.selector['app\\.kubernetes\\.io\\/instance']}\" --ignore-not-found=true", "secretType": "dockerRegistry", "secretArguments": "", "containerRegistryType": "Azure Container Registry", @@ -1618,6 +1617,98 @@ "outputFormat": "" } }, + { + "environment": {}, + "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", + "version": "0.*", + "name": "Install Helm 2.12.3", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "helmVersion": "2.12.3", + "checkLatestHelmVersion": "false", + "installKubeCtl": "true", + "kubectlVersion": "1.12.4", + "checkLatestKubeCtl": "false" + } + }, + { + "environment": {}, + "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", + "version": "0.*", + "name": "helm init", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": null, + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectionType": "$(Parameters.connectionType)", + "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", + "azureResourceGroup": "$(Parameters.azureResourceGroup)", + "kubernetesCluster": "$(Parameters.kubernetesCluster)", + "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", + "namespace": "", + "command": "init", + "chartType": "Name", + "chartName": "", + "chartPath": "", + "version": "", + "releaseName": "", + "overrideValues": "", + "valueFile": "", + "destination": "$(Build.ArtifactStagingDirectory)", + "canaryimage": "false", + "upgradetiller": "true", + "updatedependency": "false", + "save": "true", + "install": "true", + "recreate": "false", + "resetValues": "false", + "force": "false", + "waitForExecution": "true", + "arguments": "--service-account tiller", + "enableTls": "false", + "caCert": "", + "certificate": "", + "privatekey": "", + "tillernamespace": "" + } + }, + { + "environment": {}, + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", + "version": "1.*", + "name": "Azure CLI ", + "refName": "", + "enabled": true, + "alwaysRun": false, + "continueOnError": false, + "timeoutInMinutes": 0, + "definitionType": "task", + "overrideInputs": {}, + "condition": "succeeded()", + "inputs": { + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" + } + }, { "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", diff --git a/src/shipping/package/azure-pipelines.yml b/src/shipping/package/azure-pipelines.yml index 39558349..12f5809b 100644 --- a/src/shipping/package/azure-pipelines.yml +++ b/src/shipping/package/azure-pipelines.yml @@ -6,8 +6,6 @@ variables: azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL azureContainerRegistry: ACR_SERVER_VAR_VAL azureContainerRegistryName: ACR_NAME_VAR_VAL - azureKubernetesCluster: CLUSTER_NAME_VAR_VAL - resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) @@ -87,24 +85,6 @@ jobs: checkLatestKubectl: false - - task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(azureSubscription) - - azureresourceGroup: $(resourceGroup) - - kubernetesCluster: $(azureKubernetesCluster) - - namespace: prod - - command: init - - upgradeTiller: false - - arguments: '--client-only' - - task: HelmDeploy@0 condition: eq(variables['fullCI'],True) displayName: 'helm package' @@ -115,7 +95,7 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName)' + arguments: '--app-version $(Build.SourceBranchName) --dependency-update' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) diff --git a/src/shipping/workflow/azure-pipelines.yml b/src/shipping/workflow/azure-pipelines.yml index 2a9d77c9..ebfc3e27 100644 --- a/src/shipping/workflow/azure-pipelines.yml +++ b/src/shipping/workflow/azure-pipelines.yml @@ -6,8 +6,6 @@ variables: azureSubscription: AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL azureContainerRegistry: ACR_SERVER_VAR_VAL azureContainerRegistryName: ACR_NAME_VAR_VAL - azureKubernetesCluster: CLUSTER_NAME_VAR_VAL - resourceGroup: RESOURCE_GROUP_VAR_VAL name: $(build.sourceBranch)-$(Date:yyyyMMdd)$(Rev:.rr) @@ -128,22 +126,6 @@ jobs: checkLatestKubectl: false - - task: HelmDeploy@0 - condition: eq(variables['fullCI'],True) - displayName: 'helm init --client-only' - inputs: - azureSubscription: $(azureSubscription) - - azureresourceGroup: $(resourceGroup) - - kubernetesCluster: $(azureKubernetesCluster) - - command: init - - upgradeTiller: false - - arguments: '--client-only' - - task: HelmDeploy@0 condition: eq(variables['fullCI'],True) displayName: 'helm package' @@ -154,7 +136,7 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName)' + arguments: '--app-version $(Build.SourceBranchName) --dependency-update' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From bd9bc9ef7841af42a92e51b1eb7b864ddcbfff56 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Jun 2019 15:42:04 +0000 Subject: [PATCH 174/246] Merged PR 794: implement common Docker image promotion pattern using multiple acr(s) implement common Docker image promotion pattern using multiple acr(s) - deploy dedicated acr for prod - deploy shared acr for dev, test, and staging - update cd to use acr import command for more information about what this change is about: - promotion pattern, https://github.com/Azure/acr/blob/master/docs/acr-roadmap.md#image-promotion - acr namespaces, https://docs.microsoft.com/en-us/azure/container-registry/container-registry-best-practices#repository-namespaces - the acr import command, https://docs.microsoft.com/en-us/azure/container-registry/container-registry-import-images | | ACR Namespaces | ACR Import | |-----------------|-------------------------|---------------------| | Isolation | logical | physical | | Resources | single instance | multi-instance | | Guidance | organizational purposes | common pattern | | Geo-replication | dev-test and prod | replicate just prod | | Security | under development | rbac | | | dev | qa | staging | production | |-------------------------|:---------:|:---------:|:---------:|:----------:| | aks_std_d2v2_x3 | shared | shared | shared | shared | | acr_basic | shared | shared | shared | dedicated | | redis | ded_ba_c0 | ded_ba_c0 | ded_std_c1 | ded_std_c1 | | cosmosdb_1k_ru | dedicated | dedicated | dedicated | dedicated | | servicebus_std_1gig | dedicated | dedicated | dedicated | dedicated | | mongo_2.6k_ru | dedicated | dedicated | dedicated | dedicated | | stoaccount_std_lrs_v1 | dedicated | dedicated | dedicated | dedicated | | appinsights | dedicated | dedicated | dedicated | dedicated | | keyvault_std_x3 | dedicated | dedicated | dedicated | dedicated | solved: #9212 Related work items: #9212 --- azuredeploy.json | 9 +- .../delivery/envs/delivery-prod/values.yaml | 1 - .../envs/dronescheduler-prod/values.yaml | 1 - .../ingestion/envs/ingestion-prod/values.yaml | 1 - charts/package/envs/package-prod/values.yaml | 1 - .../workflow/envs/workflow-prod/values.yaml | 1 - deploymentCICD.md | 64 ++++- src/shipping/delivery/azure-pipelines-cd.json | 224 ++++++------------ .../dronescheduler/azure-pipelines-cd.json | 216 +++++------------ .../ingestion/azure-pipelines-cd.json | 216 +++++------------ src/shipping/package/azure-pipelines-cd.json | 216 +++++------------ src/shipping/workflow/azure-pipelines-cd.json | 202 +++++----------- 12 files changed, 364 insertions(+), 788 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index f6662d2c..0759baf4 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -152,6 +152,7 @@ "clusterNamePrefix": "aks", "acrNamePrefix": "acr", "aiNamePrefix": "ai", + "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", @@ -160,7 +161,7 @@ "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", - "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "acrName": "[variables('acrName')]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -180,7 +181,7 @@ }, "qa": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", - "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "acrName": "[variables('acrName')]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -200,7 +201,7 @@ }, "staging": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", - "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "acrName": "[variables('acrName')]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -220,7 +221,7 @@ }, "prod": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", - "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "acrName": "[concat(parameters('environmentName'),variables('acrName'))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", diff --git a/charts/delivery/envs/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml index a02382e0..3a1ba13f 100644 --- a/charts/delivery/envs/delivery-prod/values.yaml +++ b/charts/delivery/envs/delivery-prod/values.yaml @@ -3,7 +3,6 @@ nameOverride: delivery exports: data: replicaCount: 3 - dockerregistrynamespace: "/prod" image: pullPolicy: IfNotPresent telemetry: diff --git a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml index 51f75da5..81788552 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml @@ -3,7 +3,6 @@ nameOverride: dronescheduler exports: data: replicaCount: 3 - dockerregistrynamespace: "/prod" image: pullPolicy: IfNotPresent telemetry: diff --git a/charts/ingestion/envs/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml index 431bb060..51f25b01 100644 --- a/charts/ingestion/envs/ingestion-prod/values.yaml +++ b/charts/ingestion/envs/ingestion-prod/values.yaml @@ -3,7 +3,6 @@ nameOverride: ingestion exports: data: replicaCount: 1 - dockerregistrynamespace: "/prod" image: pullPolicy: IfNotPresent telemetry: diff --git a/charts/package/envs/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml index 5f897e06..aecca82e 100644 --- a/charts/package/envs/package-prod/values.yaml +++ b/charts/package/envs/package-prod/values.yaml @@ -3,7 +3,6 @@ nameOverride: package exports: data: replicaCount: 3 - dockerregistrynamespace: "/prod" image: pullPolicy: IfNotPresent log: diff --git a/charts/workflow/envs/workflow-prod/values.yaml b/charts/workflow/envs/workflow-prod/values.yaml index 9d027222..22edfc42 100644 --- a/charts/workflow/envs/workflow-prod/values.yaml +++ b/charts/workflow/envs/workflow-prod/values.yaml @@ -3,7 +3,6 @@ nameOverride: workflow exports: data: replicaCount: 3 - dockerregistrynamespace: "/prod" image: pullPolicy: IfNotPresent telemetry: diff --git a/deploymentCICD.md b/deploymentCICD.md index ac6370ad..a6d9868b 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -49,6 +49,8 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) +export {${ENV}_ACR_NAME,ACR_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.acrName.value -o tsv) +export ${ENV}_ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) done ``` @@ -56,8 +58,6 @@ done Get outputs from Azure Deploy ```bash # Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrLoginServer.value -o tsv) && \ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) && \ ``` @@ -265,9 +265,9 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources + sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ + sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#DEV_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#DEV_DATABASE_NAME_VAR_VAL#$DEV_DATABASE_NAME#g" | \ @@ -278,6 +278,8 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # qa resources + sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ + sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#QA_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#QA_DATABASE_NAME_VAR_VAL#$QA_DATABASE_NAME#g" | \ @@ -288,6 +290,8 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # staging resources + sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#STAGING_DATABASE_NAME_VAR_VAL#$STAGING_DATABASE_NAME#g" | \ @@ -298,6 +302,10 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # production resources + sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ + sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ + sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_DELIVERY_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_CLIENT_ID#g" | \ sed "s#PROD_DELIVERY_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DELIVERY_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#PROD_DATABASE_NAME_VAR_VAL#$PROD_DATABASE_NAME#g" | \ @@ -364,22 +372,30 @@ cat $PACKAGE_PATH/azure-pipelines-cd.json | \ sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources sed "s#DEV_AI_IKEY_VAR_VAL#$DEV_AI_IKEY#g" | \ + sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ + sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_COSMOSDB_COL_NAME_VAR_VAL#$DEV_COSMOSDB_COL_NAME#g" | \ sed "s#DEV_COSMOSDB_CONNECTION_VAR_VAL#${DEV_COSMOSDB_CONNECTION//&/\\&}#g" | \ # qa resources sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ + sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ + sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_COSMOSDB_COL_NAME_VAR_VAL#$QA_COSMOSDB_COL_NAME#g" | \ sed "s#QA_COSMOSDB_CONNECTION_VAR_VAL#${QA_COSMOSDB_CONNECTION//&/\\&}#g" | \ # staging resources sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ + sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_COSMOSDB_COL_NAME_VAR_VAL#$STAGING_COSMOSDB_COL_NAME#g" | \ sed "s#STAGING_COSMOSDB_CONNECTION_VAR_VAL#${STAGING_COSMOSDB_CONNECTION//&/\\&}#g" | \ # production resources sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ + sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ + sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ + sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_COSMOSDB_COL_NAME_VAR_VAL#$PROD_COSMOSDB_COL_NAME#g" | \ sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${PROD_COSMOSDB_CONNECTION//&/\\&}#g" \ > $PACKAGE_PATH/azure-pipelines-cd-0.json @@ -438,9 +454,9 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ sed "s#CLUSTER_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources + sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ + sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#DEV_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ sed "s#DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ @@ -448,6 +464,8 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#DEV_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#DEV_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # qa resources + sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ + sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#QA_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ sed "s#QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ @@ -455,6 +473,8 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#QA_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#QA_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # staging resources + sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ sed "s#STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ @@ -462,6 +482,10 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#STAGING_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ # production resources + sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ + sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ + sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_WORKFLOW_KEYVAULT_RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ sed "s#PROD_WORKFLOW_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_WORKFLOW_PRINCIPAL_CLIENT_ID#g" | \ sed "s#PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ @@ -529,10 +553,10 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources sed "s#DEV_AI_IKEY_VAR_VAL#$DEV_AI_IKEY#g" | \ + sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ + sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$DEV_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$DEV_EXTERNAL_INGEST_FQDN#g" | \ sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$DEV_INGESTION_QUEUE_NAMESPACE#g" | \ @@ -543,6 +567,8 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # qa resources sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ + sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ + sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$QA_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$QA_EXTERNAL_INGEST_FQDN#g" | \ sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$QA_INGESTION_QUEUE_NAMESPACE#g" | \ @@ -553,6 +579,8 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # staging resources sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ + sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$STAGING_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$STAGING_EXTERNAL_INGEST_FQDN#g" | \ sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$STAGING_INGESTION_QUEUE_NAMESPACE#g" | \ @@ -563,6 +591,10 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ # production resources sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ + sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ + sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ + sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$PROD_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$PROD_INGESTION_QUEUE_NAMESPACE#g" | \ @@ -628,21 +660,29 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#AZURE_DEVOPS_USER_ID_VAR_VAL#$AZURE_DEVOPS_USER_ID#g" | \ sed "s#CLUSTER_NAME_VAR_VAL#$CLUSTER_NAME#g" | \ sed "s#RESOURCE_GROUP_VAR_VAL#$RESOURCE_GROUP#g" | \ - sed "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" | \ - sed "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" | \ # development resources + sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ + sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DEV_DRONESCHEDULER_KEYVAULT_URI#g" | \ # qa resources + sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ + sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$QA_DRONESCHEDULER_KEYVAULT_URI#g" | \ # staging resources + sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$STAGING_DRONESCHEDULER_KEYVAULT_URI#g" | \ # production resources + sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ + sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ + sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ + sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$PROD_DRONESCHEDULER_KEYVAULT_URI#g" \ diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index c9a62049..8a5505e9 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -11,12 +11,6 @@ "value": "Azure DevOps CD Pipeline", "allowOverride": true }, - "ACR_SERVER": { - "value": "ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "ACR_NAME_VAR_VAL" - }, "SERVICE_NAME": { "value": "delivery" } @@ -58,7 +52,13 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL" - } + }, + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -134,7 +134,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -201,7 +201,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -226,7 +226,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade delivery dev", "refName": "", "enabled": true, "alwaysRun": false, @@ -449,7 +449,13 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "QA_INGRESS_TLS_SECRET_NAME_VAR_VAL" - } + }, + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -525,7 +531,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -592,7 +598,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -617,7 +623,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade delivery qa", "refName": "", "enabled": true, "alwaysRun": false, @@ -835,7 +841,13 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL" - } + }, + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -911,7 +923,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -978,7 +990,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1003,7 +1015,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade delivery staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -1220,7 +1232,19 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL" - } + }, + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -1297,103 +1321,9 @@ "workflowTasks": [ { "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Pull a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "pull", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(Build.Repository.Name):$(Build.BuildId)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Tag a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Tag image", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "false", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Push a docker image", + "name": "promote delivery image to production", "refName": "", "enabled": true, "alwaysRun": false, @@ -1403,44 +1333,22 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Push an image", - "dockerFile": "**/Dockerfile", - "arguments": "", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr import --name $(ACR_NAME) --source $(SOURCE_ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName) --force", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" } }, { "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1507,7 +1415,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1520,7 +1428,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1532,7 +1440,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "install new delivery version in the green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1550,7 +1458,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", @@ -1645,7 +1553,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl get", + "name": "get current delivery blue slot version", "refName": "BlueVersion", "enabled": true, "alwaysRun": false, @@ -1693,7 +1601,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1760,7 +1668,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1773,7 +1681,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1785,7 +1693,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl set green", + "name": "swap blue delivery version to green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1833,7 +1741,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade (mark as current blue)", + "name": "install new delivery version in the blue slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1851,7 +1759,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index 511ca99d..29418217 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -4,12 +4,6 @@ "description": null, "isDeleted": false, "variables": { - "ACR_NAME": { - "value": "ACR_NAME_VAR_VAL" - }, - "ACR_SERVER": { - "value": "ACR_SERVER_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -36,6 +30,12 @@ }, "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { "value": "DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -112,7 +112,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -179,7 +179,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -204,7 +204,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade dronescheduler dev", "refName": "", "enabled": true, "alwaysRun": false, @@ -402,6 +402,12 @@ }, "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { "value": "QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -478,7 +484,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -545,7 +551,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -570,7 +576,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade dronescheduler qa", "refName": "", "enabled": true, "alwaysRun": false, @@ -768,6 +774,12 @@ }, "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { "value": "STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -844,7 +856,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -911,7 +923,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -936,7 +948,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrate dronescheduler staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -1134,6 +1146,18 @@ }, "DRONESCHEDULER_PRINCIPAL_RESOURCE_ID": { "value": "PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -1211,103 +1235,9 @@ "workflowTasks": [ { "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Pull a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "pull", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(Build.Repository.Name):$(Build.BuildId)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Tag a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Tag image", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "false", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Push a docker image", + "name": "promote dronescheduler image to production", "refName": "", "enabled": true, "alwaysRun": false, @@ -1317,44 +1247,22 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Push an image", - "dockerFile": "**/Dockerfile", - "arguments": "", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr import --name $(ACR_NAME) --source $(SOURCE_ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName) --force", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" } }, { "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1421,7 +1329,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1434,7 +1342,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1446,7 +1354,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "install new dronescheduler version in the green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1464,7 +1372,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", @@ -1559,7 +1467,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl get", + "name": "get current dronescheduler blue slot version", "refName": "BlueVersion", "enabled": true, "alwaysRun": false, @@ -1607,7 +1515,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1674,7 +1582,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1687,7 +1595,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1699,7 +1607,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl set green", + "name": "swap blue dronescheduler version to green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1747,7 +1655,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade (mark as current blue)", + "name": "install new dronescheduler version in the blue slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1765,7 +1673,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 837752e5..8b222409 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -4,12 +4,6 @@ "description": null, "isDeleted": false, "variables": { - "ACR_NAME": { - "value": "ACR_NAME_VAR_VAL" - }, - "ACR_SERVER": { - "value": "ACR_SERVER_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -60,6 +54,12 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -129,7 +129,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -196,7 +196,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -221,7 +221,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade ingestion dev", "refName": "", "enabled": true, "alwaysRun": false, @@ -440,6 +440,12 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "QA_INGRESS_TLS_SECRET_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -509,7 +515,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -576,7 +582,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -601,7 +607,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade ingestion qa", "refName": "", "enabled": true, "alwaysRun": false, @@ -820,6 +826,12 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -889,7 +901,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -956,7 +968,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -981,7 +993,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade ingestion staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -1200,6 +1212,18 @@ }, "INGRESS_TLS_SECRET_NAME": { "value": "PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL" + }, + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -1270,103 +1294,9 @@ "workflowTasks": [ { "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Pull a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "pull", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(Build.Repository.Name):$(Build.BuildId)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Tag a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Tag image", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "false", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Push a docker image", + "name": "promote ingestion image to production", "refName": "", "enabled": true, "alwaysRun": false, @@ -1376,44 +1306,22 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Push an image", - "dockerFile": "**/Dockerfile", - "arguments": "", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr import --name $(ACR_NAME) --source $(SOURCE_ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName) --force", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" } }, { "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1480,7 +1388,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1493,7 +1401,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1505,7 +1413,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "install new ingestion version in the green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1523,7 +1431,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", @@ -1611,7 +1519,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl get", + "name": "get current ingestion blue slot version", "refName": "BlueVersion", "enabled": true, "alwaysRun": false, @@ -1659,7 +1567,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1726,7 +1634,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1739,7 +1647,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1751,7 +1659,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl set green", + "name": "swap blue ingestion version to green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1799,7 +1707,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade (mark as current blue)", + "name": "install new ingestion version in the blue slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1817,7 +1725,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 7fa6205e..66b74068 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -4,12 +4,6 @@ "description": null, "isDeleted": false, "variables": { - "ACR_SERVER": { - "value": "ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "ACR_NAME_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -38,6 +32,12 @@ "COSMOSDB_CONNECTION": { "value": "DEV_COSMOSDB_CONNECTION_VAR_VAL", "isSecret": true + }, + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -114,7 +114,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -181,7 +181,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -206,7 +206,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade package dev", "refName": "", "enabled": true, "alwaysRun": false, @@ -411,6 +411,12 @@ "COSMOSDB_CONNECTION": { "value": "QA_COSMOSDB_CONNECTION_VAR_VAL", "isSecret": true + }, + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -487,7 +493,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -554,7 +560,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -579,7 +585,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade package staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -779,6 +785,12 @@ "COSMOSDB_CONNECTION": { "value": "STAGING_COSMOSDB_CONNECTION_VAR_VAL", "isSecret": true + }, + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -855,7 +867,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -922,7 +934,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -947,7 +959,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade package staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -1147,6 +1159,18 @@ "COSMOSDB_CONNECTION": { "value": "PROD_COSMOSDB_CONNECTION_VAR_VAL", "isSecret": true + }, + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -1225,103 +1249,9 @@ "workflowTasks": [ { "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Pull a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "pull", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(Build.Repository.Name):$(Build.BuildId)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Tag a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Tag image", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "false", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Push a docker image", + "name": "promote package image to production", "refName": "", "enabled": true, "alwaysRun": false, @@ -1331,44 +1261,22 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Push an image", - "dockerFile": "**/Dockerfile", - "arguments": "", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr import --name $(ACR_NAME) --source $(SOURCE_ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName) --force", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" } }, { "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1435,7 +1343,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1448,7 +1356,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1460,7 +1368,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "install new package version in the green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1478,7 +1386,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", @@ -1573,7 +1481,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl get", + "name": "get current package blue slot version", "refName": "BlueVersion", "enabled": true, "alwaysRun": false, @@ -1621,7 +1529,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1688,7 +1596,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1701,7 +1609,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1713,7 +1621,7 @@ "environment": {}, "taskId": "cbc316a2-586f-4def-be79-488a1f503564", "version": "1.*", - "name": "kubectl set green", + "name": "swap blue package version to green slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1761,7 +1669,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade (set current as blue)", + "name": "install new package version in the blue slot", "refName": "", "enabled": true, "alwaysRun": false, @@ -1779,7 +1687,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json index 847d68b3..69827538 100644 --- a/src/shipping/workflow/azure-pipelines-cd.json +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -4,12 +4,6 @@ "description": null, "isDeleted": false, "variables": { - "ACR_NAME": { - "value": "ACR_NAME_VAR_VAL" - }, - "ACR_SERVER": { - "value": "ACR_SERVER_VAR_VAL" - }, "REASON": { "value": "Azure DevOps CD Pipeline", "allowOverride": true @@ -45,6 +39,12 @@ }, "WORKFLOW_PRINCIPAL_RESOURCE_ID": { "value": "DEV_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -114,7 +114,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -181,7 +181,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -206,7 +206,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade workflow dev", "refName": "", "enabled": true, "alwaysRun": false, @@ -413,6 +413,12 @@ }, "WORKFLOW_PRINCIPAL_RESOURCE_ID": { "value": "QA_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -482,7 +488,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -549,7 +555,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -574,7 +580,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade workflow qa", "refName": "", "enabled": true, "alwaysRun": false, @@ -781,6 +787,12 @@ }, "WORKFLOW_PRINCIPAL_RESOURCE_ID": { "value": "STAGING_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -850,7 +862,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -917,7 +929,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -942,7 +954,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade workflow staging", "refName": "", "enabled": true, "alwaysRun": false, @@ -1149,6 +1161,18 @@ }, "WORKFLOW_PRINCIPAL_RESOURCE_ID": { "value": "PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL" + }, + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" } }, "variableGroups": [], @@ -1227,103 +1251,9 @@ "workflowTasks": [ { "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Pull a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "pull", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(Build.Repository.Name):$(Build.BuildId)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", - "version": "1.*", - "name": "Tag a docker image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Tag image", - "dockerFile": "**/Dockerfile", - "arguments": "$(ACR_SERVER)/prod/$(REPO_NAME):$(Build.SourceBranchName)", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "$(ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "false", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" - } - }, - { - "environment": {}, - "taskId": "e28912f1-0114-4464-802a-a3a35437fd16", + "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Push a docker image", + "name": "promote workflow image to production", "refName": "", "enabled": true, "alwaysRun": false, @@ -1333,44 +1263,22 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "containerregistrytype": "Azure Container Registry", - "dockerRegistryEndpoint": "", - "azureSubscriptionEndpoint": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", - "azureContainerRegistry": "ACR_SERVER_VAR_VAL", - "command": "Push an image", - "dockerFile": "**/Dockerfile", - "arguments": "", - "pushMultipleImages": "false", - "tagMultipleImages": "false", - "imageName": "prod/$(REPO_NAME):$(Build.SourceBranchName)", - "imageNamesPath": "", - "qualifyImageName": "true", - "includeSourceTags": "false", - "includeLatestTag": "false", - "addDefaultLabels": "true", - "useDefaultContext": "true", - "buildContext": "", - "imageDigestFile": "", - "containerName": "", - "ports": "", - "volumes": "", - "envVars": "", - "workingDirectory": "", - "entrypointOverride": "", - "containerCommand": "", - "runInBackground": "true", - "restartPolicy": "no", - "maxRestartRetries": "", - "dockerHostEndpoint": "", - "enforceDockerNamingConvention": "true", - "memoryLimit": "" + "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", + "scriptLocation": "inlineScript", + "scriptPath": "", + "inlineScript": "az acr import --name $(ACR_NAME) --source $(SOURCE_ACR_SERVER)/$(REPO_NAME):$(Build.SourceBranchName) --force", + "args": "", + "addSpnToEnvironment": "false", + "useGlobalConfig": "false", + "cwd": "", + "failOnStandardError": "false" } }, { "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "Install Helm 2.12.3", + "name": "install helm 2.12.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1437,7 +1345,7 @@ "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", "version": "1.*", - "name": "Azure CLI ", + "name": "add helm repo", "refName": "", "enabled": true, "alwaysRun": false, @@ -1450,7 +1358,7 @@ "connectedServiceNameARM": "AZURE_DEVOPS_SERVICE_CONN_ID_VAR_VAL", "scriptLocation": "inlineScript", "scriptPath": "", - "inlineScript": "az acr helm repo add --name $(ACR_NAME)", + "inlineScript": "az acr helm repo add --name $(SOURCE_ACR_NAME)", "args": "", "addSpnToEnvironment": "false", "useGlobalConfig": "false", @@ -1462,7 +1370,7 @@ "environment": {}, "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", "version": "0.*", - "name": "helm upgrade", + "name": "helm upgrade workflow prod", "refName": "", "enabled": true, "alwaysRun": false, @@ -1480,7 +1388,7 @@ "namespace": "backend", "command": "upgrade", "chartType": "Name", - "chartName": "$(ACR_NAME)/$(REPO_NAME)", + "chartName": "$(SOURCE_ACR_NAME)/$(REPO_NAME)", "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", From dba457e6e200a3e84af3492711bdf3afbc77f94d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Jun 2019 20:45:56 +0000 Subject: [PATCH 175/246] Merged PR 795: implement dedicated resource group acr best practice implement dedicated resource group acr best practice for more information about this and other best pracrices, please take a look at https://docs.microsoft.com/en-us/azure/container-registry/container-registry-best-practices#dedicated-resource-group based on the imminent explosure of resource groups when working with acr and multiple environments, resource groups creation is now part of the arm pre-requesites deployment template and multiple acr instances are being created within them - resource groups creation template - acr creation in diff resource groups env-based - streamline manual/ci-cd instructions - remove outputs using reference func provided it is a nested resource. it is because, that function is not supported when using nested templates. For more information, please take a at https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-linked-templates#nested-template solved: #9211 Related work items: #9211 --- azuredeploy-identities.json | 131 ---------------------------- azuredeploy-prereqs.json | 166 ++++++++++++++++++++++++++++++++++++ azuredeploy.json | 116 +++++++++++++++---------- deployment.md | 45 ++++++---- deploymentCICD.md | 48 +++++++---- 5 files changed, 295 insertions(+), 211 deletions(-) delete mode 100644 azuredeploy-identities.json create mode 100644 azuredeploy-prereqs.json diff --git a/azuredeploy-identities.json b/azuredeploy-identities.json deleted file mode 100644 index 1b606431..00000000 --- a/azuredeploy-identities.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "type": "string", - "defaultValue": "dev", - "allowedValues": [ - "dev", - "qa", - "staging", - "prod" - ] - } - }, - "variables": { - "environmentSettings": { - "dev": { - "deliveryIdName": "dev-d", - "workflowIdName": "dev-wf", - "droneSchedulerIdName": "dev-ds" - }, - "qa": { - "deliveryIdName": "qa-d", - "workflowIdName": "qa-wf", - "droneSchedulerIdName": "qa-ds" - }, - "staging": { - "deliveryIdName": "staging-d", - "workflowIdName": "staging-wf", - "droneSchedulerIdName": "staging-ds" - }, - "prod": { - "deliveryIdName": "prod-d", - "workflowIdName": "prod-wf", - "droneSchedulerIdName": "prod-ds" - } - } - }, - "resources": [ - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]", - "tags": { - "displayName": "workflow managed identity", - "what": "rbac", - "reason": "aad-pod-identity", - "app": "fabrikam-workflow", - "tier": "[parameters('environmentName')]" - } - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]", - "tags": { - "displayName": "delivery managed identity", - "what": "rbac", - "reason": "aad-pod-identity", - "app": "fabrikam-delivery", - "tier": "[parameters('environmentName')]" - } - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", - "apiVersion": "2015-08-31-preview", - "location": "[resourceGroup().location]", - "tags": { - "displayName": "dronescheduler managed identity", - "what": "rbac", - "reason": "aad-pod-identity", - "app": "fabrikam-dronescheduler", - "tier": "[parameters('environmentName')]" - } - } - ], - "outputs": { - "deliveryIdName": { - "value": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", - "type": "string" - }, - "deliveryPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)).principalId]", - "type": "string" - }, - "deliveryPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)]", - "type": "string" - }, - "deliveryPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].deliveryIdName)).clientId]", - "type": "string" - }, - "droneSchedulerIdName": { - "value": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", - "type": "string" - }, - "droneSchedulerPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)).principalId]", - "type": "string" - }, - "droneSchedulerPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)]", - "type": "string" - }, - "droneSchedulerPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)).clientId]", - "type": "string" - }, - "workflowIdName": { - "value": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", - "type": "string" - }, - "workflowPrincipalId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)).principalId]", - "type": "string" - }, - "workflowPrincipalResourceId": { - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)]", - "type": "string" - }, - "workflowPrincipalClientId": { - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('environmentSettings')[parameters('environmentName')].workflowIdName)).clientId]", - "type": "string" - } - } -} diff --git a/azuredeploy-prereqs.json b/azuredeploy-prereqs.json new file mode 100644 index 00000000..65d57a0d --- /dev/null +++ b/azuredeploy-prereqs.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "type": "string", + "defaultValue": "dev", + "allowedValues": [ + "dev", + "qa", + "staging", + "prod" + ] + }, + "resourceGroupName": { + "type": "string" + }, + "resourceGroupLocation": { + "type": "string", + "defaultValue": "eastus" + } + }, + "variables": { + "acrResourceGroupNamePrefix": "[concat(parameters('resourceGroupName'),'-acr')]", + "nestedIdDeploymentName": "[concat(deployment().name,'-identities')]", + "environmentSettings": { + "dev": { + "deliveryIdName": "dev-d", + "workflowIdName": "dev-wf", + "droneSchedulerIdName": "dev-ds", + "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" + }, + "qa": { + "deliveryIdName": "qa-d", + "workflowIdName": "qa-wf", + "droneSchedulerIdName": "qa-ds", + "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" + }, + "staging": { + "deliveryIdName": "staging-d", + "workflowIdName": "staging-wf", + "droneSchedulerIdName": "staging-ds", + "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" + }, + "prod": { + "deliveryIdName": "prod-d", + "workflowIdName": "prod-wf", + "droneSchedulerIdName": "prod-ds", + "acrResourceGroupName": "[concat(variables('acrResourceGroupNamePrefix'),'-',parameters('environmentName'))]" + } + } + }, + "resources": [ + { + "name": "[parameters('resourceGroupName')]", + "location": "[parameters('resourceGroupLocation')]", + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2018-05-01", + "tags": { + "displayName": "Resource Group for general purpose" + } + }, + { + "name": "[variables('environmentSettings')[parameters('environmentName')].acrResourceGroupName]", + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2018-05-01", + "location": "[parameters('resourceGroupLocation')]", + "tags": { + "displayName": "Container Registry Resource Group" + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-05-01", + "name": "[variables('nestedIdDeploymentName')]", + "resourceGroup": "[parameters('resourceGroupName')]", + "dependsOn": [ + "[resourceId('Microsoft.Resources/resourceGroups/', parameters('resourceGroupName'))]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", + "apiVersion": "2015-08-31-preview", + "location": "[parameters('resourceGroupLocation')]", + "tags": { + "displayName": "workflow managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-workflow", + "[parameters('environmentName')]": true + } + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", + "apiVersion": "2015-08-31-preview", + "location": "[parameters('resourceGroupLocation')]", + "tags": { + "displayName": "delivery managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-delivery", + "[parameters('environmentName')]": true + } + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", + "apiVersion": "2015-08-31-preview", + "location": "[parameters('resourceGroupLocation')]", + "tags": { + "displayName": "dronescheduler managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "app": "fabrikam-dronescheduler", + "[parameters('environmentName')]": true + } + } + ], + "outputs": { + "deliveryIdName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].deliveryIdName]", + "type": "string" + }, + "deliveryPrincipalResourceId": { + "value": "[concat(subscription().id, '/resourceGroups/',parameters('resourceGroupName'),'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',variables('environmentSettings')[parameters('environmentName')].deliveryIdName)]", + "type": "string" + }, + "droneSchedulerIdName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName]", + "type": "string" + }, + "droneSchedulerPrincipalResourceId": { + "value": "[concat(subscription().id, '/resourceGroups/',parameters('resourceGroupName'),'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',variables('environmentSettings')[parameters('environmentName')].droneSchedulerIdName)]", + "type": "string" + }, + "workflowIdName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].workflowIdName]", + "type": "string" + }, + "workflowPrincipalResourceId": { + "value": "[concat(subscription().id, '/resourceGroups/',parameters('resourceGroupName'),'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',variables('environmentSettings')[parameters('environmentName')].workflowIdName)]", + "type": "string" + }, + "acrResourceGroupName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].acrResourceGroupName]", + "type": "string" + } + } + } + } + } + ], + "outputs": { + "identitiesDeploymentName": { + "value": "[variables('nestedIdDeploymentName')]", + "type": "string" + } + } +} diff --git a/azuredeploy.json b/azuredeploy.json index 0759baf4..cacf0735 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -12,6 +12,12 @@ "prod" ] }, + "acrResourceGroupName": { + "type": "string" + }, + "acrResourceGroupLocation": { + "type": "string" + }, "deliveryIdName": { "metadata": { "description": "Name of the delivery managed identity" @@ -158,6 +164,7 @@ "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]", "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", + "nestedACRDeploymentName": "[concat('azuredeploy-acr-',parameters('acrResourceGroupName'),parameters('environmentName'))]", "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -242,6 +249,67 @@ } }, "resources": [ + { + "name": "[variables('nestedACRDeploymentName')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-05-01", + "resourceGroup": "[parameters('acrResourceGroupName')]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "name": "[variables('environmentSettings')[parameters('environmentName')].acrName]", + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2017-10-01", + "sku": { + "name": "Basic", + "tier": "Basic" + }, + "location": "[resourceGroup().location]", + "tags": { + "displayName": "Container Registry", + "container.registry": "[variables('environmentSettings')[parameters('environmentName')].acrName]", + "clusterName": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]" + }, + "properties": { + "adminUserEnabled": false + } + }, + { + "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].acrName,'/Microsoft.Authorization/',guid(concat('aks',variables('environmentSettings')[parameters('environmentName')].acrName), resourceGroup().id))]", + "apiVersion": "2017-05-01", + "comments": "Grant the AKS cluster access to the ACR instance", + "tags": { + "displayName": "AKS SP RBAC Access to ACR", + "what": "rbac", + "to": "cluster", + "identity-type": "sp", + "access": "acr", + "reason": "pull-images" + }, + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[parameters('servicePrincipalId')]", + "scope": "[concat(subscription().id, '/resourceGroups/',parameters('acrResourceGroupName'),'/providers/Microsoft.ContainerRegistry/registries/',variables('environmentSettings')[parameters('environmentName')].acrName)]" + }, + "dependsOn": [ + "[concat(subscription().id, '/resourceGroups/',parameters('acrResourceGroupName'),'/providers/Microsoft.ContainerRegistry/registries/',variables('environmentSettings')[parameters('environmentName')].acrName)]" + ] + } + ], + "outputs": { + "acrId": { + "value": "[resourceId('Microsoft.ContainerRegistry/registries', variables('environmentSettings')[parameters('environmentName')].acrName)]", + "type": "string" + } + } + } + } + }, { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", @@ -280,24 +348,6 @@ "enableRBAC": true } }, - { - "name": "[variables('environmentSettings')[parameters('environmentName')].acrName]", - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2017-10-01", - "sku": { - "name": "Basic", - "tier": "Basic" - }, - "location": "[resourceGroup().location]", - "tags": { - "displayName": "Shared Container Registry for dev, qa, staging and production", - "container.registry": "[variables('environmentSettings')[parameters('environmentName')].acrName]", - "clusterName": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]" - }, - "properties": { - "adminUserEnabled": false - } - }, { "name": "[variables('environmentSettings')[parameters('environmentName')].appInsightsName]", "type": "Microsoft.Insights/components", @@ -781,28 +831,6 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName)]" ] }, - { - "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", - "name": "[concat(variables('environmentSettings')[parameters('environmentName')].acrName,'/Microsoft.Authorization/',guid('aks', resourceGroup().id))]", - "apiVersion": "2017-05-01", - "comments": "Grant the AKS cluster access to the ACR instance", - "tags": { - "displayName": "AKS SP RBAC Access to ACR", - "what": "rbac", - "to": "cluster", - "identity-type": "sp", - "access": "acr", - "reason": "pull-images" - }, - "properties": { - "roleDefinitionId": "[variables('readerRoleId')]", - "principalId": "[parameters('servicePrincipalId')]", - "scope": "[resourceId('Microsoft.ContainerRegistry/registries', variables('environmentSettings')[parameters('environmentName')].acrName)]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('environmentSettings')[parameters('environmentName')].acrName)]" - ] - }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", "name": "[concat(parameters('deliveryIdName'), '/Microsoft.Authorization/', guid(concat('msi-delivery',parameters('environmentName')), resourceGroup().id))]", @@ -869,10 +897,6 @@ "value": "[variables('environmentSettings')[parameters('environmentName')].acrName]", "type": "string" }, - "acrLoginServer": { - "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries',variables('environmentSettings')[parameters('environmentName')].acrName),'2016-06-27-preview').loginServer]", - "type": "string" - }, "aksClusterName": { "value": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "string" @@ -912,6 +936,10 @@ "ingestionServiceAccessKeyName": { "value": "[variables('environmentSettings')[parameters('environmentName')].ingestionServiceAccessKey]", "type": "string" + }, + "acrDeploymentName": { + "value": "[variables('nestedACRDeploymentName')]", + "type": "string" } } } diff --git a/deployment.md b/deployment.md index 80c60da0..9d12f663 100644 --- a/deployment.md +++ b/deployment.md @@ -51,8 +51,7 @@ Infrastructure Prerequisites # Log in to Azure az login -# Create a resource group and service principal for AKS -az group create --name $RESOURCE_GROUP --location $LOCATION && \ +# Create service principal for AKS export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ @@ -72,15 +71,23 @@ Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.m Infrastructure ```bash -# Deploy the managed identities +# Deploy the resource groups and managed identities # These are deployed first in a separate template to avoid propagation delays with AAD -az group deployment create -g $RESOURCE_GROUP --name azuredeploy-identities-dev --template-file ${PROJECT_ROOT}/azuredeploy-identities.json -export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryIdName.value -o tsv) -export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalId.value -o tsv) -export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerIdName.value -o tsv) -export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) -export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowIdName.value -o tsv) -export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalId.value -o tsv) +az deployment create \ + --name azuredeploy-prereqs-dev \ + --location $LOCATION \ + --template-file ${PROJECT_ROOT}/azuredeploy-prereqs.json \ + --parameters resourceGroupName=$RESOURCE_GROUP \ + resourceGroupLocation=$LOCATION + +export IDENTITIES_DEPLOYMENT_NAME=$(az deployment show -n azuredeploy-prereqs-dev --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ +export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryIdName.value -o tsv) && \ +export DELIVERY_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query principalId -o tsv) && \ +export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerIdName.value -o tsv) && \ +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) && \ +export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowIdName.value -o tsv) && \ +export WORKFLOW_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) && \ +export RESOURCE_GROUP_ACR=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.acrResourceGroupName.value -o tsv) # Wait for AAD propagation until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done @@ -102,14 +109,16 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template- droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \ droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ workflowIdName=${WORKFLOW_ID_NAME} \ - workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} + workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} \ + acrResourceGroupName=${RESOURCE_GROUP_ACR} \ + acrResourceGroupLocation=$LOCATION ``` Get outputs from Azure Deploy ```bash # Shared export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrLoginServer.value -o tsv) && \ +export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) ``` @@ -223,8 +232,8 @@ Deploy the Delivery service: ```bash # Extract pod identity outputs from deployment -export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ -export DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \ +export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query clientId -o tsv) export DELIVERY_INGRESS_TLS_SECRET_NAME=delivery-ingress-tls # Deploy the service @@ -326,8 +335,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.workflowPrincipalClientId.value -o tsv) +export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) ``` Deploy the Workflow service: @@ -429,8 +438,8 @@ Create and set up pod identity ```bash # Extract outputs from deployment -export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-dev --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) ``` Build and publish the container image diff --git a/deploymentCICD.md b/deploymentCICD.md index a6d9868b..9d0570ec 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -15,18 +15,22 @@ export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) && \ for env in dev qa staging prod; do ENV=${env^^} -az group deployment create \ - -g $RESOURCE_GROUP \ - --name azuredeploy-identities-${env} \ - --template-file ${PROJECT_ROOT}/azuredeploy-identities.json \ - --parameters environmentName=${env} - -export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryIdName.value -o tsv) -export DELIVERY_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalId.value -o tsv) -export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerIdName.value -o tsv) -export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalId.value -o tsv) -export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowIdName.value -o tsv) -export WORKFLOW_ID_PRINCIPAL_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalId.value -o tsv) +az deployment create \ + --name azuredeploy-prereqs-${env} \ + --location $LOCATION \ + --template-file ${PROJECT_ROOT}/azuredeploy-prereqs.json \ + --parameters resourceGroupName=$RESOURCE_GROUP \ + resourceGroupLocation=$LOCATION \ + environmentName=${env} + +export {${ENV}_IDENTITIES_DEPLOYMENT_NAME,IDENTITIES_DEPLOYMENT_NAME}=$(az deployment show -n azuredeploy-prereqs-dev --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ +export {${ENV}_DELIVERY_ID_NAME,DELIVERY_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryIdName.value -o tsv) +export DELIVERY_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query principalId -o tsv) +export {${ENV}_DRONESCHEDULER_ID_NAME,DRONESCHEDULER_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerIdName.value -o tsv) +export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) +export {${ENV}_WORKFLOW_ID_NAME,WORKFLOW_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowIdName.value -o tsv) +export WORKFLOW_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) +export RESOURCE_GROUP_ACR=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.acrResourceGroupName.value -o tsv) # Wait for AAD propagation until az ad sp show --id $DELIVERY_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done @@ -45,6 +49,8 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa droneSchedulerPrincipalId=$DRONESCHEDULER_ID_PRINCIPAL_ID \ workflowIdName=$WORKFLOW_ID_NAME \ workflowPrincipalId=$WORKFLOW_ID_PRINCIPAL_ID \ + acrResourceGroupName=${RESOURCE_GROUP_ACR} \ + acrResourceGroupLocation=$LOCATION \ environmentName=${env} export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) @@ -249,9 +255,11 @@ for env in dev qa staging prod;do ENV=${env^^} export ${ENV}_DATABASE_NAME="deliveries-db" export ${ENV}_COLLECTION_NAME="deliveries-col" +envIdentitiesDeploymentName="${ENV}_IDENTITIES_DEPLOYMENT_NAME" export ${ENV}_DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.deliveryKeyVaultUri.value -o tsv) -export ${ENV}_DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) -export ${ENV}_DELIVERY_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.deliveryPrincipalClientId.value -o tsv) +export ${ENV}_DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n ${!envIdentitiesDeploymentName} --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) +envDeliveryIdName="${ENV}_DELIVERY_ID_NAME" +export ${ENV}_DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n ${!envDeliveryIdName} --query clientId -o tsv) done && \ export INGRESS_TLS_SECRET_NAME=delivery-ingress-tls @@ -439,9 +447,11 @@ export AZURE_DEVOPS_WORKFLOW_BUILD_ID=$(az pipelines build definition list --org export AZURE_DEVOPS_WORKFLOW_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='workflow-ci'].queue.id" -o tsv) && \ for env in dev qa staging prod;do ENV=${env^^} +envIdentitiesDeploymentName="${ENV}_IDENTITIES_DEPLOYMENT_NAME" export ${ENV}_WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.workflowKeyVaultName.value -o tsv) -export ${ENV}_WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalResourceId.value -o tsv) -export ${ENV}_WORKFLOW_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.workflowPrincipalClientId.value -o tsv) +export ${ENV}_WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n ${!envIdentitiesDeploymentName} --query properties.outputs.workflowPrincipalResourceId.value -o tsv) +envWorkflowIdName="${ENV}_WORKFLOW_ID_NAME" +export ${ENV}_WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n ${!envWorkflowIdName} --query clientId -o tsv) done # add relese definition @@ -645,8 +655,10 @@ export AZURE_DEVOPS_DRONE_BUILD_ID=$(az pipelines build definition list --organi export AZURE_DEVOPS_DRONE_QUEUE_ID=$(az pipelines build definition list --organization $AZURE_DEVOPS_ORG --project $AZURE_DEVOPS_PROJECT_NAME --query "[?name=='dronescheduler-ci'].queue.id" -o tsv) && \ for env in dev qa staging prod;do ENV=${env^^} -export ${ENV}_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) -export ${ENV}_DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-identities-${env} --query properties.outputs.droneSchedulerPrincipalClientId.value -o tsv) +envIdentitiesDeploymentName="${ENV}_IDENTITIES_DEPLOYMENT_NAME" +export ${ENV}_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n ${!envIdentitiesDeploymentName} --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) +envDroneSchedulerIdName="${ENV}_DRONESCHEDULER_ID_NAME" +export ${ENV}_DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n ${!envDroneSchedulerIdName} --query clientId -o tsv) export ${ENV}_DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) done From 2b398bb63b958a6db9bc926f616fd28b9fb51961 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 19 Jun 2019 15:05:51 -0300 Subject: [PATCH 176/246] bug fixing deployment - workflow and dronescheduler: change principalId with clientId - change verification steps to check for dev release instead - remove dup link to CI/CD path --- deployment.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/deployment.md b/deployment.md index 9d12f663..913fb0c1 100644 --- a/deployment.md +++ b/deployment.md @@ -175,12 +175,6 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` -## Optional: Set up CI/CD with Azure DevOps - -Add [CI/CD to Drone Delivery using Azure Pipelines with YAML](./deploymentCICD.md). - -> Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application as follows. - ## Deploy the ingress controller ```bash @@ -260,7 +254,7 @@ helm install $HELM_CHARTS/delivery/ \ --dep-up # Verify the pod is created -helm status delivery-v0.1.0 +helm status delivery-v0.1.0-dev ``` ## Deploy the Package service @@ -307,7 +301,7 @@ helm install $HELM_CHARTS/package/ \ --dep-up # Verify the pod is created -helm status package-v0.1.0 +helm status package-v0.1.0-dev ``` ## Deploy the Workflow service @@ -336,7 +330,7 @@ Create and set up pod identity ```bash # Extract outputs from deployment export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \ -export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) +export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query clientId -o tsv) ``` Deploy the Workflow service: @@ -360,7 +354,7 @@ helm install $HELM_CHARTS/workflow/ \ --dep-up # Verify the pod is created -helm status workflow-v0.1.0 +helm status workflow-v0.1.0-dev ``` ## Deploy the Ingestion service @@ -417,7 +411,7 @@ helm install $HELM_CHARTS/ingestion/ \ --dep-up # Verify the pod is created -helm status ingestion-v0.1.0 +helm status ingestion-v0.1.0-dev ``` ## Deploy DroneScheduler service @@ -439,7 +433,7 @@ Create and set up pod identity ```bash # Extract outputs from deployment export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) +export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query clientId -o tsv) ``` Build and publish the container image @@ -470,7 +464,7 @@ helm install $HELM_CHARTS/dronescheduler/ \ --dep-up # Verify the pod is created -helm status dronescheduler-v0.1.0 +helm status dronescheduler-v0.1.0-dev ``` ## Validate the application is running From 128cb777eed3402c0741574e39a98fbb675f5c64 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Jul 2019 12:57:21 +0000 Subject: [PATCH 177/246] Merged PR 816: Merged PR 790: Implement basic gateway aggregation with ingress Implement basic gateway aggregation with ingress Implement invoincing endpoints: * Adding new query endpoints for delivery, package, and scheduler * Adding new "internal" ingresses for the new endpoints Related work items: #9205 --- .../templates/delivery-internal-ingress.yaml | 41 ++++++++++++++++ .../dronescheduler-internal-ingress.yaml | 41 ++++++++++++++++ .../templates/package-internal-ingress.yaml | 41 ++++++++++++++++ .../DeliveriesControllerFixture.cs | 40 +++++++++++++-- .../Controllers/DeliveriesController.cs | 24 +++++++-- .../Models/DeliveriesSummary.cs | 14 ++++++ .../Services/DeliveryRepository.cs | 11 +++++ .../Services/IDeliveryRepository.cs | 2 + .../Controllers/DroneDeliveriesController.cs | 18 +++++++ .../Models/DroneUtilization.cs | 15 ++++++ src/shipping/package/app/api.json | 49 +++++++++++++++++++ .../app/controllers/package-controllers.ts | 20 ++++++++ src/shipping/package/app/models/api-models.ts | 11 +++++ 13 files changed, 317 insertions(+), 10 deletions(-) create mode 100644 charts/delivery/templates/delivery-internal-ingress.yaml create mode 100644 charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml create mode 100644 charts/package/templates/package-internal-ingress.yaml create mode 100644 src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveriesSummary.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs diff --git a/charts/delivery/templates/delivery-internal-ingress.yaml b/charts/delivery/templates/delivery-internal-ingress.yaml new file mode 100644 index 00000000..867221b6 --- /dev/null +++ b/charts/delivery/templates/delivery-internal-ingress.yaml @@ -0,0 +1,41 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingress +################################################################################################### +{{- $svcversion := .Chart.AppVersion | replace "." "" }} +{{- $appversion := .Chart.AppVersion }} +{{- $defaultversionedpath := printf "/%s/" $appversion }} +{{- $relname := .Release.Name }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $relname }}-internal-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries$1 + nginx.ingress.kubernetes.io/configuration-snippet: | + internal; +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + {{- if .path }} + - path: {{ printf "%s/%s/" .path $appversion }}api/internal/deliveries(.*) + {{- else }} + - path: {{ $defaultversionedpath }}api/internal/deliveries(.*) + {{- end }} + backend: + serviceName: "{{ .serviceName }}-{{ $svcversion }}" + servicePort: http + {{- if (eq $appversion "v0.1.0") }} + - path: {{ default "/" .path }}api/internal/deliveries(.*) + backend: + serviceName: "{{ .serviceName }}" + servicePort: http + {{- end }} + {{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml b/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml new file mode 100644 index 00000000..375e2899 --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml @@ -0,0 +1,41 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingress +################################################################################################### +{{- $svcversion := .Chart.AppVersion | replace "." "" }} +{{- $appversion := .Chart.AppVersion }} +{{- $defaultversionedpath := printf "/%s/" $appversion }} +{{- $relname := .Release.Name }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $relname }}-internal-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /api/dronedeliveries$1 + nginx.ingress.kubernetes.io/configuration-snippet: | + internal; +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + {{- if .path }} + - path: {{ printf "%s/%s/" .path $appversion }}api/internal/dronedeliveries(.*) + {{- else }} + - path: {{ $defaultversionedpath }}api/internal/dronedeliveries(.*) + {{- end }} + backend: + serviceName: "{{ .serviceName }}-{{ $svcversion }}" + servicePort: http + {{- if (eq $appversion "v0.1.0") }} + - path: {{ default "/" .path }}api/internal/dronedeliveries(.*) + backend: + serviceName: "{{ .serviceName }}" + servicePort: http + {{- end }} + {{ end }} diff --git a/charts/package/templates/package-internal-ingress.yaml b/charts/package/templates/package-internal-ingress.yaml new file mode 100644 index 00000000..dbad0b09 --- /dev/null +++ b/charts/package/templates/package-internal-ingress.yaml @@ -0,0 +1,41 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# ingress +################################################################################################### +{{- $svcversion := .Chart.AppVersion | replace "." "" }} +{{- $appversion := .Chart.AppVersion }} +{{- $defaultversionedpath := printf "/%s/" $appversion }} +{{- $relname := .Release.Name }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $relname }}-internal-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /api/package$1 + nginx.ingress.kubernetes.io/configuration-snippet: | + internal; +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + {{- if .path }} + - path: {{ printf "%s/%s/" .path $appversion }}api/internal/package(.*) + {{- else }} + - path: {{ $defaultversionedpath }}api/internal/package(.*) + {{- end }} + backend: + serviceName: "{{ .serviceName }}-{{ $svcversion }}" + servicePort: http + {{- if (eq $appversion "v0.1.0") }} + - path: {{ default "/" .path }}api/internal/package(.*) + backend: + serviceName: "{{ .serviceName }}" + servicePort: http + {{- end }} + {{ end }} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs index f8ffc156..548e54af 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerFixture.cs @@ -3,17 +3,18 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; using Fabrikam.DroneDelivery.Common; using Fabrikam.DroneDelivery.DeliveryService.Controllers; -using Fabrikam.DroneDelivery.DeliveryService.Services; using Fabrikam.DroneDelivery.DeliveryService.Models; +using Fabrikam.DroneDelivery.DeliveryService.Services; +using Moq; namespace Fabrikam.DroneDelivery.DeliveryService.Tests { @@ -42,7 +43,7 @@ public async Task Get_Returns404_IfDeliveryIdNotValid() new Mock().Object, new Mock().Object, loggerFactory.Object); - + // Act var result = await target.Get("invaliddeliveryid") as NotFoundResult; @@ -88,7 +89,7 @@ public async Task GetOwner_ReturnsOwner() new Mock().Object, new Mock().Object, loggerFactory.Object); - + // Act var result = await target.GetOwner("deliveryid") as OkObjectResult; @@ -400,7 +401,7 @@ public async Task NotifyMe_AddsNotifyMeRequest() loggerFactory.Object); var notifyMeRequest = new NotifyMeRequest("email@test.com", "1234567"); - + // Act var result = await target.NotifyMe("deliveryid", notifyMeRequest) as NoContentResult; @@ -566,5 +567,34 @@ public async Task Confirm_AddsDeliveryCompletedEvent() Assert.AreEqual(DeliveryStage.Completed, completedDelivery.Stage); } + [TestMethod] + public async Task GetSummry_ReturnsSummary() + { + const int TestDeliveryCount = 5000; + + // Arrange + var deliveryRepository = new Mock(); + deliveryRepository.Setup(r => r.GetDeliveryCountAsync("owner", 2019, 01)) + .ReturnsAsync(TestDeliveryCount) + .Verifiable(); + + var loggerFactory = new Mock(); + loggerFactory.Setup(f => f.CreateLogger(It.IsAny())).Returns(new Mock().Object); + + var target = new DeliveriesController(deliveryRepository.Object, + new Mock().Object, + new Mock().Object, + new Mock().Object, + loggerFactory.Object); + + // Act + var result = await target.GetSummary("owner", 2019, 01) as OkObjectResult; + + // Assert + Assert.IsNotNull(result); + Assert.IsInstanceOfType(result.Value, typeof(DeliveriesSummary)); + Assert.AreEqual(TestDeliveryCount, ((DeliveriesSummary)result.Value).Count); + deliveryRepository.VerifyAll(); + } } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs index 8f8e9d2e..13bd3f72 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Fabrikam.DroneDelivery.Common; @@ -40,7 +41,7 @@ public DeliveriesController(IDeliveryRepository deliveryRepository, // GET api/deliveries/5 [HttpGet("{id}", Name = "GetDelivery")] [HttpGet("public/{id}")] - [ProducesResponseType(typeof(Delivery), 200)] + [ProducesResponseType(typeof(Delivery), StatusCodes.Status200OK)] public async Task Get(string id) { logger.LogInformation("In Get action with id: {Id}", id); @@ -58,7 +59,7 @@ public async Task Get(string id) // GET api/deliveries/5/owner [HttpGet("{id}/owner")] - [ProducesResponseType(typeof(UserAccount), 200)] + [ProducesResponseType(typeof(UserAccount), StatusCodes.Status200OK)] public async Task GetOwner(string id) { logger.LogInformation("In GetOwner action with id: {Id}", id); @@ -75,7 +76,7 @@ public async Task GetOwner(string id) // GET api/deliveries/5/status [HttpGet("{id}/status")] - [ProducesResponseType(typeof(DeliveryStatus), 200)] + [ProducesResponseType(typeof(DeliveryStatus), StatusCodes.Status200OK)] public async Task GetStatus(string id) { logger.LogInformation("In GetStatus action with id: {Id}", id); @@ -93,8 +94,8 @@ public async Task GetStatus(string id) // PUT api/deliveries/5 [HttpPut("{id}")] - [ProducesResponseType(typeof(Delivery), 201)] - [ProducesResponseType(typeof(void), 204)] + [ProducesResponseType(typeof(Delivery), StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] public async Task Put([FromBody]Delivery delivery, string id) { logger.LogInformation("In Put action with delivery {Id}: {@DeliveryInfo}", id, delivery.ToLogInfo()); @@ -263,5 +264,18 @@ await deliveryTrackingRepository.AddAsync( return Ok(); } + + [HttpGet("summary")] + [ProducesResponseType(typeof(DeliveriesSummary), StatusCodes.Status200OK)] + public async Task GetSummary([FromQuery] string ownerId, [FromQuery] int year, [FromQuery] int month) + { + var deliveryCount = await deliveryRepository.GetDeliveryCountAsync(ownerId, year, month); + + return Ok( + new DeliveriesSummary + { + Count = deliveryCount + }); + } } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveriesSummary.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveriesSummary.cs new file mode 100644 index 00000000..304a3108 --- /dev/null +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/DeliveriesSummary.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace Fabrikam.DroneDelivery.DeliveryService.Models +{ + public class DeliveriesSummary + { + public int Count { get; internal set; } + } +} diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs index 3151657e..85c4bccc 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/DeliveryRepository.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System; using System.Threading.Tasks; using Fabrikam.DroneDelivery.DeliveryService.Models; @@ -29,5 +30,15 @@ public async Task DeleteAsync(string id, InternalDelivery delivery) { await RedisCache.DeleteItemAsync(id, delivery).ConfigureAwait(continueOnCapturedContext: false); } + + public Task GetDeliveryCountAsync(string ownerId, int year, int month) + { + const int MinDeliveries = 1000; + const int MaxDeliveries = 10000; + + var deliveryCount = new Random().Next(MinDeliveries, MaxDeliveries); + + return Task.FromResult(deliveryCount); + } } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs index 9c16bc53..a6fbed69 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Services/IDeliveryRepository.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System; using System.Threading.Tasks; using Fabrikam.DroneDelivery.DeliveryService.Models; @@ -14,5 +15,6 @@ public interface IDeliveryRepository Task CreateAsync(InternalDelivery delivery); Task UpdateAsync(string id, InternalDelivery updatedDelivery); Task DeleteAsync(string id, InternalDelivery delivery); + Task GetDeliveryCountAsync(string ownerId, int year, int month); } } \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs index 5920d2c4..c1ede0ce 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs @@ -36,5 +36,23 @@ public void Delete(string id) { logger.LogInformation("In Delete action with DeliveryId: {DeliveryId}", id); } + + // GET api/dronedeliveries/utilization + [HttpGet("utilization")] + public DroneUtilization GetDroneUtilization([FromQuery] string ownerId, [FromQuery] int year, [FromQuery] int month) + { + const double MinMiles = 300d; + const double MaxMiles = 5000d; + const double MinHours = 10d; + const double MaxHours = 300d; + + var random = new Random(); + + return new DroneUtilization + { + TraveledMiles = MinMiles + random.NextDouble() * (MaxMiles - MinMiles), + AssignedHours = MinHours + random.NextDouble() * (MaxHours - MinHours) + }; + } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs new file mode 100644 index 00000000..294cd8a3 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace MockDroneScheduler.Models +{ + public class DroneUtilization + { + public double TraveledMiles { get; internal set; } + public double AssignedHours { get; internal set; } + } +} diff --git a/src/shipping/package/app/api.json b/src/shipping/package/app/api.json index 54d268d1..8b5d1cbb 100644 --- a/src/shipping/package/app/api.json +++ b/src/shipping/package/app/api.json @@ -99,6 +99,47 @@ } } } + }, + "/packages/summary": { + "get": { + "summary": "Get summary information about packages from a user", + "description": "", + "operationId": "getSummary", + "parameters": [ + { + "name": "ownerId", + "in": "query", + "description": "ID of the owner of the packages", + "required": true, + "type": "string" + }, + { + "name": "year", + "in": "query", + "description": "Year of the summary requested", + "required": true, + "type": "integer" + }, + { + "name": "month", + "in": "query", + "description": "Month of the summary requested", + "required": true, + "type": "integer" + } + ], + "responses": { + "200": { + "description": "successful operation", + "schema": { + "$ref": "#/definitions/PackageUtilization" + } + }, + "400": { + "description": "Invalid ID, year, or month supplied" + } + } + } } }, "definitions": { @@ -124,6 +165,14 @@ } } }, + "PackageUtilization": { + "type": "object", + "properties": { + "totalWeight": { + "type": "number" + } + } + }, "Error": { "type": "object", "properties": { diff --git a/src/shipping/package/app/controllers/package-controllers.ts b/src/shipping/package/app/controllers/package-controllers.ts index bad68cb5..21212dd1 100644 --- a/src/shipping/package/app/controllers/package-controllers.ts +++ b/src/shipping/package/app/controllers/package-controllers.ts @@ -126,4 +126,24 @@ export class PackageControllers { } } } + + // Get summary information about packages from a user + async getSummary(ctx: any, next: any) { + + var logger : ILogger = ctx.state.logger; + var ownerId = ctx.params.ownerId; + var year = ctx.params.year; + var month = ctx.params.month; + logger.info('retrieve summary %s %d/%d', ownerId) + + await next(); + + let utilization = new apiModels.PackageUtilization(); + utilization.totalWeight = 400; + + ctx.body = utilization; + ctx.response.status = 200; + + return; + } } diff --git a/src/shipping/package/app/models/api-models.ts b/src/shipping/package/app/models/api-models.ts index e9563464..a6f2258c 100644 --- a/src/shipping/package/app/models/api-models.ts +++ b/src/shipping/package/app/models/api-models.ts @@ -8,3 +8,14 @@ export class Package tag:string } +export class PackageUtilization +{ + totalWeight:number +} + +export class Error +{ + code:number + message:string +} + From 211bf7343986498fbaf9ed0750817fbb4445bcb8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Jul 2019 14:57:18 +0000 Subject: [PATCH 178/246] Merged PR 817: Merged PR 804: add cosmos to droneschedduler add cosmos to droneschedduler add cosmos to droneschedduler - remove random fake data generation - implement GetDroneUtilization controller method (Ok, NotFound and BadRequest) - add aggregated invoicing data query - add instructions to deploy dronescheduler app using cosmos - add cosmos resource to arm - add solution and test project - rename app DroneScheduler to DroneScedulerService solved: #9219 Related work items: #9219 Related work items: #9219 --- azuredeploy.json | 48 ++++++ .../templates/dronescheduler-deploy.yaml | 4 + charts/dronescheduler/values.yaml | 3 + deployment.md | 6 + src/shipping/dronescheduler/Dockerfile | 14 +- .../Controllers/DroneDeliveriesController.cs | 58 ------- .../CosmosDBRespositoryTests.cs | 110 +++++++++++++ .../CustomWebApplicationFactory.cs | 30 ++++ ...neDeliveriesUtilizationIntegrationTests.cs | 145 ++++++++++++++++++ ...elivery.DroneSchedulerService.Tests.csproj | 41 +++++ .../InvoicingRepositoryTests.cs | 92 +++++++++++ .../Utils/DocumentClientMock.cs | 129 ++++++++++++++++ .../Utils/IFakeDocumentQuery.cs | 16 ++ .../appsettings.Test.json | 20 +++ .../appsettings.json | 0 ...am.DroneDelivery.DroneSchedulerService.sln | 48 ++++++ .../.dockerignore | 0 .../Controllers/DroneDeliveriesController.cs | 79 ++++++++++ .../CosmosDBExtensions.cs | 34 ++++ ...roneDelivery.DroneSchedulerService.csproj} | 6 +- .../Models/BaseDocument.cs | 18 +++ .../Models/DroneDelivery.cs | 2 +- .../Models/DroneUtilization.cs | 2 +- .../Models/InternalDroneUtilization.cs | 31 ++++ .../Models/PackageDetail.cs | 2 +- .../Models/PackageSize.cs | 2 +- .../Program.cs | 8 +- .../ConfigureCosmosDBRepositoryOptions.cs | 52 +++++++ .../Services/CosmosDBRepository.cs | 71 +++++++++ .../Services/CosmosDBRepositoryOptions.cs | 16 ++ .../Services/ICosmosDBRepository.cs | 20 +++ .../Services/IInvoicingRepository.cs | 25 +++ .../Services/InvoicingRepository.cs | 60 ++++++++ .../Startup.cs | 9 +- .../TracingExtensions.cs | 2 +- .../appsettings.Development.json | 0 .../appsettings.json | 18 +++ src/shipping/dronescheduler/scripts/run.sh | 2 +- 38 files changed, 1143 insertions(+), 80 deletions(-) delete mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService.Tests}/appsettings.json (100%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.sln rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/.dockerignore (100%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj => Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj} (78%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/BaseDocument.cs rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Models/DroneDelivery.cs (91%) rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Models/DroneUtilization.cs (88%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/InternalDroneUtilization.cs rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Models/PackageDetail.cs (89%) rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Models/PackageSize.cs (86%) rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Program.cs (86%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/Startup.cs (89%) rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/TracingExtensions.cs (93%) rename src/shipping/dronescheduler/{Fabrikam.DroneDelivery.DroneScheduler => Fabrikam.DroneDelivery.DroneSchedulerService}/appsettings.Development.json (100%) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json diff --git a/azuredeploy.json b/azuredeploy.json index cacf0735..4e3a7904 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -183,6 +183,7 @@ "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey" }, @@ -203,6 +204,7 @@ "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey" }, @@ -223,6 +225,7 @@ "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey" }, @@ -243,6 +246,7 @@ "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", + "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey" } @@ -462,6 +466,22 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]" } }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", + "apiVersion": "2015-04-08", + "location": "[resourceGroup().location]", + "properties": { + "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", + "databaseAccountOfferType": "Standard", + "locations": [ + { + "locationName": "[resourceGroup().location]", + "failoverPriority": 0 + } + ] + } + }, { "type": "Microsoft.ServiceBus/namespaces", "name": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace]", @@ -666,6 +686,30 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]", "[resourceId('Microsoft.Insights/components', variables('environmentSettings')[parameters('environmentName')].appInsightsName)]" ] + }, + { + "type": "secrets", + "name": "CosmosDB-Endpoint", + "apiVersion": "2015-06-01", + "properties": { + "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName)).documentEndpoint]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName)]" + ] + }, + { + "type": "secrets", + "name": "CosmosDB-Key", + "apiVersion": "2015-06-01", + "properties": { + "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName), '2016-03-31').primaryMasterKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName)]" + ] } ] }, @@ -917,6 +961,10 @@ "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)).vaultUri]", "type": "string" }, + "droneSchedulerCosmosDbName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", + "type": "string" + }, "packageMongoDbName": { "value": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "type": "string" diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index ccadbced..098e56fe 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -49,6 +49,10 @@ spec: env: - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} + - name: COSMOSDB_DATABASEID + value: {{ required .Values.cosmosdb.id }} + - name: COSMOSDB_COLLECTIONID + value: {{ required .Values.cosmosdb.collectionid }} - name: LOGGING__ApplicationInsights__LOGLEVEL__DEFAULT value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 02b3f73c..d014d0c3 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -11,6 +11,9 @@ image: pullPolicy: IfNotPresent keyvault: uri: +cosmosdb: + id: + collectionid: reason: unknown telemetry: level: "Error" diff --git a/deployment.md b/deployment.md index 913fb0c1..38eb8a0a 100644 --- a/deployment.md +++ b/deployment.md @@ -420,6 +420,10 @@ Extract resource details from deployment ```bash export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +export DRONESCHEDULER_COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerCosmosDbName.value -o tsv) && \ +export ENDPOINT_URL=$(az cosmosdb show -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query documentEndpoint -o tsv) && \ +export AUTH_KEY=$(az cosmosdb list-keys -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query primaryMasterKey -o tsv) && \ +export DATABASE_NAME="invoicing" ``` Build the dronescheduler services @@ -457,6 +461,8 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ + --set cosmosdb.id=$DATABASE_NAME \ + --set cosmosdb.collectionid=$COLLECTION_NAME \ --set reason="Initial deployment" \ --set tags.dev=true \ --namespace backend-dev \ diff --git a/src/shipping/dronescheduler/Dockerfile b/src/shipping/dronescheduler/Dockerfile index e11441d6..5ee44111 100644 --- a/src/shipping/dronescheduler/Dockerfile +++ b/src/shipping/dronescheduler/Dockerfile @@ -7,22 +7,22 @@ FROM microsoft/dotnet:2.2-sdk AS build MAINTAINER Fernando Antivero (https://github.com/ferantivero) WORKDIR /app -COPY delivery/Fabrikam.DroneDelivery.Common/*.csproj ./Fabrikam.DroneDelivery.Common/ -COPY dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/*.csproj ./Fabrikam.DroneDelivery.DroneScheduler/ +COPY delivery/Fabrikam.DroneDelivery.Common/*.csproj ./delivery/Fabrikam.DroneDelivery.Common/ +COPY dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/*.csproj ./dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ WORKDIR /app -RUN dotnet restore /app/Fabrikam.DroneDelivery.Common/ -RUN dotnet restore /app/Fabrikam.DroneDelivery.DroneScheduler/ +RUN dotnet restore /app/delivery/Fabrikam.DroneDelivery.Common/ +RUN dotnet restore /app/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ WORKDIR /app -COPY delivery/Fabrikam.DroneDelivery.Common/. ./Fabrikam.DroneDelivery.Common/ -COPY dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/. ./Fabrikam.DroneDelivery.DroneScheduler/ +COPY delivery/Fabrikam.DroneDelivery.Common/. ./delivery/Fabrikam.DroneDelivery.Common/ +COPY dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/. ./dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ FROM build AS publish MAINTAINER Fernando Antivero (https://github.com/ferantivero) WORKDIR /app -RUN dotnet publish /app/Fabrikam.DroneDelivery.DroneScheduler/ -c Release -o ../out +RUN dotnet publish /app/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ -c Release -o ../../out FROM base AS runtime diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs deleted file mode 100644 index c1ede0ce..00000000 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Controllers/DroneDeliveriesController.cs +++ /dev/null @@ -1,58 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using MockDroneScheduler.Models; - -namespace MockDroneScheduler.Controllers -{ - [Route("api/[controller]")] - public class DroneDeliveriesController : Controller - { - private readonly ILogger logger; - - public DroneDeliveriesController(ILoggerFactory loggerFactory) - { - this.logger = loggerFactory.CreateLogger(); - } - - // PUT api/dronedeliveries/5 - [HttpPut("{id}")] - public string Put([FromBody]DroneDelivery droneDelivery, string id) - { - logger.LogInformation("In Put action with DeliveryId: {DeliveryId}", id); - - var guid = Guid.NewGuid(); - return $"AssignedDroneId{guid}"; - } - - // DELETE api/dronedeliveries/5 - [HttpDelete("{id}")] - public void Delete(string id) - { - logger.LogInformation("In Delete action with DeliveryId: {DeliveryId}", id); - } - - // GET api/dronedeliveries/utilization - [HttpGet("utilization")] - public DroneUtilization GetDroneUtilization([FromQuery] string ownerId, [FromQuery] int year, [FromQuery] int month) - { - const double MinMiles = 300d; - const double MaxMiles = 5000d; - const double MinHours = 10d; - const double MaxHours = 300d; - - var random = new Random(); - - return new DroneUtilization - { - TraveledMiles = MinMiles + random.NextDouble() * (MaxMiles - MinMiles), - AssignedHours = MinHours + random.NextDouble() * (MaxHours - MinHours) - }; - } - } -} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs new file mode 100644 index 00000000..e725e45b --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using Xunit; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests +{ + public class CosmosDBRespositoryTests + { + private readonly ILogger> _loggerDebug; + + private readonly IDocumentClient _clientMockObject; + private readonly IOptions> _optionsMockObject; + + private readonly IQueryable _fakeResults; + + public CosmosDBRespositoryTests() + { + var servicesBuilder = new ServiceCollection(); + servicesBuilder.AddLogging(logging => logging.AddDebug()); + var services = servicesBuilder.BuildServiceProvider(); + + _loggerDebug = services.GetService< + ILogger< + CosmosRepository< + InternalDroneUtilization>>>(); + + _fakeResults = new List { + new InternalDroneUtilization { + Id = "d0001", + PartitionKey = "o00042", + OwnerId = "o00042", + Month = 6, + Year = 2019, + TraveledMiles =10d, + AssignedHours=1d, + DocumentType = typeof(InternalDroneUtilization).Name + }, + new InternalDroneUtilization { + Id = "d0002", + PartitionKey = "o00042", + OwnerId = "o00042", + Month = 6, + Year = 2019, + TraveledMiles=32d, + AssignedHours=2d, + DocumentType = typeof(InternalDroneUtilization).Name + } + }.AsQueryable(); + + _clientMockObject = DocumentClientMock. + CreateDocumentClientMockObject(_fakeResults); + + var fakeOptionsValue = + new CosmosDBRepositoryOptions + { + CollectionUri = UriFactory.CreateDocumentCollectionUri( + "fakeDb", + "fakeCol") + }; + + var optionsMock = new Mock< + IOptions< + CosmosDBRepositoryOptions< + InternalDroneUtilization>>>(); + optionsMock + .Setup(o => o.Value) + .Returns(fakeOptionsValue); + + _optionsMockObject = optionsMock.Object; + } + + [Fact] + public async Task WhenGetItemsAsync_ThenClientMakesAQuery() + { + // Arrange + string ownerId = "o00042"; + + var repo = new CosmosRepository( + _clientMockObject, + _optionsMockObject, + _loggerDebug); + + // Act + var res = await repo.GetItemsAsync( + p => true, + ownerId); + + // Assert + Assert.NotNull(res); + Assert.Equal(_fakeResults.Count(), res.Count()); + Assert.True(res.All(r => r.PartitionKey == ownerId)); + Assert.True(res.All(r => r.DocumentType == typeof(InternalDroneUtilization).Name)); + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs new file mode 100644 index 00000000..83cd6968 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests +{ + public class CustomWebApplicationFactory + : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder + .UseContentRoot(".") + .UseEnvironment("Test") + .ConfigureTestServices(s => + { + s.AddLogging(b => b.AddDebug()); + }); + + base.ConfigureWebHost(builder); + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs new file mode 100644 index 00000000..db9cb5f4 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests +{ + public class DroneDeliveriesUtilizationIntegrationTests : + IClassFixture, + IDisposable + { + private const string RequestUri = "localhost/api/dronedeliveries/utilization"; + private const string ExpectedContentType = "application/json; charset=utf-8"; + + private readonly HttpClient _client; + private readonly CustomWebApplicationFactory _factory; + + private readonly IConfigureOptions> _configOptMockObject; + + public DroneDeliveriesUtilizationIntegrationTests( + CustomWebApplicationFactory factory) + { + Uri fakeCollectionUri = UriFactory.CreateDocumentCollectionUri( + "fakeDb", + "fakeCol"); + + var configOptMock = new Mock>>(); + configOptMock + .Setup(c => c.Configure( + It.IsAny>())) + .Callback>( + o => o.CollectionUri = fakeCollectionUri); + + _configOptMockObject = configOptMock.Object; + + _factory = factory; + _client = factory.WithWebHostBuilder(b => + b.ConfigureTestServices(s => + { + s.ConfigureOptions(_configOptMockObject); + s.AddSingleton(DocumentClientMock + .CreateDocumentClientMockObject( + new List { + new InternalDroneUtilization { + Id = "d0001", + PartitionKey = "o00042", + OwnerId = "o00042", + Month = 6, + Year = 2019, + TraveledMiles =10d, + AssignedHours=1d, + DocumentType = typeof(InternalDroneUtilization).Name + } + }.AsQueryable())); + })) + .CreateClient(); + } + + [Fact] + public async Task GetInvoicingWithoutQueryParams_ThenResponseBadRequestStatusCode() + { + // Arrange + var droneUtilizationUriWithoutParams = new UriBuilder(RequestUri); + + // Act + var response = await _client.GetAsync(droneUtilizationUriWithoutParams.ToString()); + + // Assert + Assert.NotNull(response); + Assert.Equal(System.Net.HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task GetInvoicingMonthlyBasisWithValidCriteria_ThenResponseWithSuccessStatusCode() + { + // Arrange + string ownerId = "o00042"; + int year = 2019, month = 6; + + var droneUtilizationUriWithParams = new UriBuilder(RequestUri); + droneUtilizationUriWithParams.Query = + $"ownerId={ownerId}&year={year}&month={month}"; + + // Act + var response = await _client.GetAsync(droneUtilizationUriWithParams.ToString()); + + // Assert + Assert.NotNull(response); + response.EnsureSuccessStatusCode(); // Status Code 200-299 + Assert.Equal(ExpectedContentType, + response.Content.Headers.ContentType.ToString()); + } + + [Fact] + public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusCode() + { + // Arrange + HttpClient client = _factory.WithWebHostBuilder(b => + b.ConfigureTestServices(s => + { + s.ConfigureOptions(_configOptMockObject); + s.AddSingleton(DocumentClientMock + .CreateDocumentClientMockObject( + new List() + .AsQueryable())); + })) + .CreateClient(); + + string ownerId = "o00042"; + var minValidDateTime = DateTime.MinValue; + int year = minValidDateTime.Year, month = minValidDateTime.Month; + + var droneUtilizationUriWithParams = new UriBuilder(RequestUri); + droneUtilizationUriWithParams.Query = + $"ownerId={ownerId}&year={year}&month={month}"; + + // Act + var response = await client.GetAsync(droneUtilizationUriWithParams.ToString()); + + // Assert + Assert.NotNull(response); + Assert.Equal(System.Net.HttpStatusCode.NotFound, response.StatusCode); + } + + public void Dispose() + { + _factory.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj new file mode 100644 index 00000000..7f10d231 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj @@ -0,0 +1,41 @@ + + + + netcoreapp2.2 + false + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + + + + + + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs new file mode 100644 index 00000000..bb360a93 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs @@ -0,0 +1,92 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Moq; +using Xunit; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests +{ + public class InvoicingRepositoryTests + { + [Fact] + public async Task WhenGetAggregatedInvoincingData_ThenInvokesDroneUtilizationRepository() + { + // Arrange + string ownerId = "o00042"; + int year = 2019; + int month = 6; + var cosmosDbMock = new Mock>(); + var repo = new InvoicingRepository(cosmosDbMock.Object); + + // Act + var result = await repo.GetAggreatedInvoincingDataAsync(ownerId, year, month); + + // Assert + Assert.NotNull(result); + Assert.Equal(0d, result.Item1); + Assert.Equal(0d, result.Item2); + cosmosDbMock + .Verify(p => + p.GetItemsAsync( + It.IsAny>>(), + ownerId), + Times.Once); + } + + [Fact] + public async Task WhenGetAggregatedInvoincingDataForAValidPeriod_ThenRepoReturnsData() + { + // Arrange + string ownerId = "o00042"; + int year = 2019; + int month = 6; + var invoicingData = new List { + new InternalDroneUtilization{ + TraveledMiles=10d, + AssignedHours=1d + }, + new InternalDroneUtilization{ + TraveledMiles=32d, + AssignedHours=2d + } + }; + var cosmosDbMock = new Mock>(); + cosmosDbMock.Setup(r => + r.GetItemsAsync( + It.IsAny>>(), + ownerId)) + .ReturnsAsync(invoicingData.AsEnumerable()); + var repo = new InvoicingRepository(cosmosDbMock.Object); + + // Act + var (traveledMiles, assignedHours) = + await repo.GetAggreatedInvoincingDataAsync( + ownerId, + year, + month); + + // Assert + Assert.Equal( + invoicingData.Sum(d => d.TraveledMiles), + traveledMiles); + Assert.Equal( + invoicingData.Sum(d => d.AssignedHours), + assignedHours); + cosmosDbMock + .Verify(p => + p.GetItemsAsync( + It.IsAny>>(), + ownerId), + Times.Once); + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs new file mode 100644 index 00000000..aaceae45 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs @@ -0,0 +1,129 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Moq; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils +{ + public static class DocumentClientMock + { + public static IDocumentClient CreateDocumentClientMockObject( + IQueryable fakeResults) + { + // doc query + var fakeResponse = new FeedResponse(fakeResults); + + var mockDocumentQuery = new Mock>(); + + var docProvider = new Mock(); + docProvider + .Setup(p => p.CreateQuery( + It.IsAny())) + .Returns(mockDocumentQuery.Object); + + mockDocumentQuery + .SetupSequence(q => q.HasMoreResults) + .Returns(true) + .Returns(false); + + mockDocumentQuery + .Setup(q => + q.ExecuteNextAsync( + It.IsAny())) + .ReturnsAsync(fakeResponse); + + mockDocumentQuery + .Setup(q => q.Provider) + .Returns(docProvider.Object); + mockDocumentQuery + .Setup(q => q.ElementType) + .Returns(fakeResults.ElementType); + mockDocumentQuery + .Setup(q => q.Expression) + .Returns(fakeResults.Expression); + mockDocumentQuery + .Setup(q => q.GetEnumerator()) + .Returns(fakeResults.GetEnumerator()); + + // db query + var mockDatabaseQuery = new Mock>(); + var dbProvider = new Mock(); + dbProvider + .Setup(p => p.CreateQuery( + It.IsAny())) + .Returns(mockDatabaseQuery.Object); + + var fakeDbResutls = new List + { + Mock.Of() + }.AsQueryable(); + + mockDatabaseQuery + .Setup(q => q.Provider) + .Returns(dbProvider.Object); + mockDatabaseQuery + .Setup(q => q.ElementType) + .Returns(fakeDbResutls.ElementType); + mockDatabaseQuery + .Setup(q => q.Expression) + .Returns(fakeDbResutls.Expression); + mockDatabaseQuery + .Setup(q => q.GetEnumerator()) + .Returns(fakeDbResutls.GetEnumerator()); + + // collection query + var mockCollectionQuery = new Mock>(); + var colProvider = new Mock(); + colProvider + .Setup(p => p.CreateQuery( + It.IsAny())) + .Returns(mockCollectionQuery.Object); + + var fakeColResutls = new List + { + Mock.Of() + }.AsQueryable(); + + mockCollectionQuery + .Setup(q => q.Provider) + .Returns(colProvider.Object); + mockCollectionQuery + .Setup(q => q.ElementType) + .Returns(fakeColResutls.ElementType); + mockCollectionQuery + .Setup(q => q.Expression) + .Returns(fakeColResutls.Expression); + mockCollectionQuery + .Setup(q => q.GetEnumerator()) + .Returns(fakeColResutls.GetEnumerator()); + + // DocumentClient mock + var clientMock = new Mock(); + + clientMock + .Setup(q => q.CreateDocumentQuery( + It.IsAny(), + It.IsAny())) + .Returns(mockDocumentQuery.Object); + + clientMock + .Setup(q => q.CreateDatabaseQuery(null)) + .Returns(mockDatabaseQuery.Object); + + clientMock + .Setup(q => q.CreateDocumentCollectionQuery(It.IsAny(), null)) + .Returns(mockCollectionQuery.Object); + + return clientMock.Object; + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs new file mode 100644 index 00000000..f0962eca --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Linq; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Linq; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils +{ + public interface IFakeDocumentQuery : + IDocumentQuery, + IOrderedQueryable + { } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json new file mode 100644 index 00000000..eb76cf6f --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "Error" + } + }, + "LogLevel": { + "Default": "Information" + } + }, + "Serilog": { + "MinimumLevel": "Verbose", + "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], + "WriteTo": [] + }, + "CosmosDB-Endpoint": "https://fake-invoicing.documents.azure.com:443/", + "CosmosDB-Key": "ZmFrZXBhc3MK" +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.json similarity index 100% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.json rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.json diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.sln b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.sln new file mode 100644 index 00000000..7620e661 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28917.182 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.Common", "..\delivery\Fabrikam.DroneDelivery.Common\Fabrikam.DroneDelivery.Common.csproj", "{81A668CF-785F-4A63-A2C7-76C86F16F49A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.DroneSchedulerService.Tests", "Fabrikam.DroneDelivery.DroneSchedulerService.Tests\Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj", "{4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{44E83671-6389-4799-92EF-E3D4743925DD}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{ECB4611F-B203-4C60-BE5C-30ACF51839F1}" + ProjectSection(SolutionItems) = preProject + Dockerfile = Dockerfile + scripts\run.sh = scripts\run.sh + EndProjectSection +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fabrikam.DroneDelivery.DroneSchedulerService", "Fabrikam.DroneDelivery.DroneSchedulerService\Fabrikam.DroneDelivery.DroneSchedulerService.csproj", "{8EC22B29-F95C-4B0C-8274-3FCDB55C555C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {81A668CF-785F-4A63-A2C7-76C86F16F49A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {81A668CF-785F-4A63-A2C7-76C86F16F49A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {81A668CF-785F-4A63-A2C7-76C86F16F49A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {81A668CF-785F-4A63-A2C7-76C86F16F49A}.Release|Any CPU.Build.0 = Release|Any CPU + {4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD}.Release|Any CPU.Build.0 = Release|Any CPU + {8EC22B29-F95C-4B0C-8274-3FCDB55C555C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8EC22B29-F95C-4B0C-8274-3FCDB55C555C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8EC22B29-F95C-4B0C-8274-3FCDB55C555C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8EC22B29-F95C-4B0C-8274-3FCDB55C555C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {4D0F39C2-0653-4E01-B1F6-9E39CAB2D3DD} = {44E83671-6389-4799-92EF-E3D4743925DD} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C81CEA0C-455B-422D-BEB3-AB22F0F3F95B} + EndGlobalSection +EndGlobal diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/.dockerignore b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/.dockerignore similarity index 100% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/.dockerignore rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/.dockerignore diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs new file mode 100644 index 00000000..607ac985 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs @@ -0,0 +1,79 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Microsoft.AspNetCore.Http; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Controllers +{ + [Route("api/[controller]")] + public class DroneDeliveriesController : Controller + { + private readonly ILogger logger; + private readonly IInvoicingRepository _invoicingRepository; + + public DroneDeliveriesController( + IInvoicingRepository invoicingRepository, + ILoggerFactory loggerFactory) + { + this.logger = loggerFactory.CreateLogger(); + this._invoicingRepository = invoicingRepository; + } + + // PUT api/dronedeliveries/5 + [HttpPut("{id}")] + public string Put([FromBody]Models.DroneDelivery droneDelivery, string id) + { + logger.LogInformation("In Put action with DeliveryId: {DeliveryId}", id); + + var guid = Guid.NewGuid(); + return $"AssignedDroneId{guid}"; + } + + // DELETE api/dronedeliveries/5 + [HttpDelete("{id}")] + public void Delete(string id) + { + logger.LogInformation("In Delete action with DeliveryId: {DeliveryId}", id); + } + + // GET api/dronedeliveries/utilization + [HttpGet("utilization")] + [ProducesResponseType(StatusCodes.Status200OK, + Type = typeof(DroneUtilization))] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task GetDroneUtilization([FromQuery] string ownerId, [FromQuery] int year, [FromQuery] int month) + { + // TODO: improve binding model --> improve err msg. + if (string.IsNullOrEmpty(ownerId) + || year < 1 + || month < 1) + { + return BadRequest(); + } + + var (traveledMiles, assignedHours) = await _invoicingRepository. + GetAggreatedInvoincingDataAsync(ownerId, year, month); + + if (traveledMiles == 0d && + assignedHours == 0d) + { + return NotFound(); + } + + return Ok(new DroneUtilization + { + TraveledMiles = traveledMiles, + AssignedHours = assignedHours + }); + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs new file mode 100644 index 00000000..ac023b67 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Microsoft.Azure.Documents; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService +{ + public static class CosmosDBExtensions + { + public static IServiceCollection AddCosmosRepository( + this IServiceCollection services, + IConfiguration config) + where T : BaseDocument + { + var endpoint = config["CosmosDB-Endpoint"]; + var key = config["CosmosDB-Key"]; + + services.AddSingleton(s => new DocumentClient(new Uri(endpoint), key)); + services.ConfigureOptions>(); + + services.AddSingleton, CosmosRepository>(); + + return services; + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj similarity index 78% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj index af4ebbf9..b89e0d9f 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Fabrikam.DroneDelivery.DroneScheduler.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj @@ -3,6 +3,8 @@ netcoreapp2.2 ..\docker-compose.dcproj + Fabrikam.DroneDelivery.DroneSchedulerService + Fabrikam.DroneDelivery.DroneSchedulerService @@ -13,6 +15,7 @@ + @@ -23,7 +26,8 @@ - + + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/BaseDocument.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/BaseDocument.cs new file mode 100644 index 00000000..bea24eb9 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/BaseDocument.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Newtonsoft.Json; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models +{ + public class BaseDocument + { + [JsonProperty(PropertyName = "partitionKey")] + public string PartitionKey { get; internal set; } + + [JsonProperty(PropertyName = "type")] + public string DocumentType { get; internal set; } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneDelivery.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneDelivery.cs similarity index 91% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneDelivery.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneDelivery.cs index c683103f..e4fbe2c7 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneDelivery.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneDelivery.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using Fabrikam.DroneDelivery.Common; -namespace MockDroneScheduler.Models +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models { public class DroneDelivery { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneUtilization.cs similarity index 88% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneUtilization.cs index 294cd8a3..41791028 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/DroneUtilization.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/DroneUtilization.cs @@ -5,7 +5,7 @@ using System; -namespace MockDroneScheduler.Models +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models { public class DroneUtilization { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/InternalDroneUtilization.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/InternalDroneUtilization.cs new file mode 100644 index 00000000..fc66ba09 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/InternalDroneUtilization.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Newtonsoft.Json; + +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Fabrikam.DroneDelivery.DroneSchedulerService.Tests")] +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models +{ + public class InternalDroneUtilization: BaseDocument + { + [JsonProperty(PropertyName = "id")] + public string Id { get; internal set; } + + [JsonProperty(PropertyName = "year")] + public int Year { get; internal set; } + + [JsonProperty(PropertyName = "month")] + public int Month { get; internal set; } + + [JsonProperty(PropertyName = "ownerId")] + public string OwnerId { get; internal set; } + + [JsonProperty(PropertyName = "travelledMiles")] + public double TraveledMiles { get; internal set; } + + [JsonProperty(PropertyName = "assignedHours")] + public double AssignedHours { get; internal set; } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageDetail.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageDetail.cs similarity index 89% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageDetail.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageDetail.cs index 6c828d4a..4ab88ca4 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageDetail.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageDetail.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Threading.Tasks; -namespace MockDroneScheduler.Models +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models { public class PackageDetail { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageSize.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageSize.cs similarity index 86% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageSize.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageSize.cs index 554a005d..704a5808 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Models/PackageSize.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Models/PackageSize.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -namespace MockDroneScheduler.Models +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Models { public enum PackageSize { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs similarity index 86% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs index 749a53f7..f1d74e7f 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs @@ -3,19 +3,13 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.ApplicationInsights.Extensibility; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Serilog; -namespace MockDroneScheduler +namespace Fabrikam.DroneDelivery.DroneSchedulerService { public class Program { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs new file mode 100644 index 00000000..fb64fed5 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Linq; +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class ConfigureCosmosDBRepositoryOptions : IConfigureOptions> + where T : BaseDocument + { + private readonly IConfiguration _config; + private readonly IDocumentClient _client; + + public ConfigureCosmosDBRepositoryOptions( + IConfiguration config, + IDocumentClient client) + { + _config = config; + _client = client; + } + + public void Configure(CosmosDBRepositoryOptions options) + { + Database db = _client + .CreateDatabaseQuery() + .Where(d => d.Id == _config["COSMOSDB_DATABASEID"]) + .AsEnumerable() + .FirstOrDefault(); + + if (db != null) + { + DocumentCollection col = _client + .CreateDocumentCollectionQuery(db.SelfLink) + .Where(d => d.Id == _config["COSMOSDB_COLLECTIONID"]) + .AsEnumerable() + .FirstOrDefault(); + + if (Uri.TryCreate(col?.SelfLink, UriKind.Relative, out Uri uri)) + { + options.CollectionUri = uri; + } + } + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs new file mode 100644 index 00000000..0e98c8e0 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -0,0 +1,71 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class CosmosRepository: ICosmosRepository + where T : BaseDocument + { + private readonly IDocumentClient _client; + private readonly CosmosDBRepositoryOptions _options; + private readonly ILogger> _logger; + + public CosmosRepository( + IDocumentClient client, + IOptions> options, + ILogger> logger) + { + this._client = client; + this._options = options.Value; + this._logger = logger; + } + + public async Task> GetItemsAsync( + Expression> predicate, + string partitionKey) + { + using (_logger.BeginScope(nameof(GetItemsAsync))) + { + _logger.LogInformation( + "partitionKey: {PartitionKey}", + partitionKey); + + IDocumentQuery query = + _client.CreateDocumentQuery( + this._options.CollectionUri, + new FeedOptions { + MaxItemCount = -1, + PartitionKey = new PartitionKey(partitionKey) }) + .Where(predicate) + .Where(d => d.DocumentType == typeof(T).Name) + .AsDocumentQuery(); + + var results = new List(); + + _logger.LogInformation("Start: reading results from query"); + while (query.HasMoreResults) + { + results.AddRange(await query.ExecuteNextAsync()); + } + + _logger.LogInformation("End: reading results from query"); + + return results; + } + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs new file mode 100644 index 00000000..3b106927 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class CosmosDBRepositoryOptions + where T : BaseDocument + { + public Uri CollectionUri { get; set; } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs new file mode 100644 index 00000000..5b93de51 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public interface ICosmosRepository + { + Task> GetItemsAsync( + Expression> predicate, + string partitionKey); + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs new file mode 100644 index 00000000..f9bd261f --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public interface IInvoicingRepository + { + Task> GetItemsAsync( + Expression> predicate, + string partitionKey); + + Task> GetAggreatedInvoincingDataAsync( + string ownerId, + int year, + int month); + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs new file mode 100644 index 00000000..27ac3e8c --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs @@ -0,0 +1,60 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class InvoicingRepository : IInvoicingRepository + { + private ICosmosRepository _repository; + + public InvoicingRepository( + ICosmosRepository repository) + => this._repository = repository; + + public async Task> GetItemsAsync( + Expression> predicate, + string partitionKey) + { + return await _repository + .GetItemsAsync( + predicate, + partitionKey); + } + + public async Task> GetAggreatedInvoincingDataAsync( + string ownerId, + int year, + int month) + { + var results = await this.GetItemsAsync( + d => d.Year == year && d.Month == month, + ownerId); + + var result = results + .Aggregate( + new + { + TraveledMiles = 0d, + AssignedHours = 0d + }, + (c, n) => new + { + TraveledMiles = c.TraveledMiles + n.TraveledMiles, + AssignedHours = c.AssignedHours + n.AssignedHours + }); + + return Tuple.Create( + result.TraveledMiles, + result.AssignedHours); + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs similarity index 89% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs index acea915b..acebffe2 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs @@ -13,8 +13,10 @@ using Swashbuckle.AspNetCore.Swagger; using Serilog; using Serilog.Formatting.Compact; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; -namespace MockDroneScheduler +namespace Fabrikam.DroneDelivery.DroneSchedulerService { public class Startup { @@ -55,6 +57,11 @@ public void ConfigureServices(IServiceCollection services) { c.SwaggerDoc("v1", new Info { Title = "Mock DroneScheduler API", Version = "v1" }); }); + + services.AddSingleton(); + services + .AddCosmosRepository< + InternalDroneUtilization>(Configuration); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/TracingExtensions.cs similarity index 93% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/TracingExtensions.cs index 48faa7e3..4ad59283 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/TracingExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/TracingExtensions.cs @@ -6,7 +6,7 @@ using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Extensions.DependencyInjection; -namespace MockDroneScheduler +namespace Fabrikam.DroneDelivery.DroneSchedulerService { public static class TracingExtensions { diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.Development.json similarity index 100% rename from src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneScheduler/appsettings.Development.json rename to src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.Development.json diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json new file mode 100644 index 00000000..8e39a665 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json @@ -0,0 +1,18 @@ +{ + "Logging": { + "IncludeScopes": false, + "ApplicationInsights": { + "LogLevel": { + "Default": "Error" + } + }, + "LogLevel": { + "Default": "Information" + } + }, + "Serilog": { + "MinimumLevel": "Verbose", + "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], + "WriteTo": [] + } +} diff --git a/src/shipping/dronescheduler/scripts/run.sh b/src/shipping/dronescheduler/scripts/run.sh index c692f57b..f2deec86 100644 --- a/src/shipping/dronescheduler/scripts/run.sh +++ b/src/shipping/dronescheduler/scripts/run.sh @@ -1,2 +1,2 @@ #!/bin/bash -dotnet Fabrikam.DroneDelivery.DroneScheduler.dll +dotnet Fabrikam.DroneDelivery.DroneSchedulerService.dll From 511f108767d595ad0f6c3242d6f081703a30049e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Jul 2019 17:18:35 +0000 Subject: [PATCH 179/246] Merged PR 818: Merged PR 808: Track metrics for RU consumption Track metrics for RU consumption Related work items: #9237 --- .../CosmosDBRespositoryTests.cs | 18 +++++--- ...neDeliveriesUtilizationIntegrationTests.cs | 2 + .../CosmosDBExtensions.cs | 3 +- .../Services/CosmosDBRepository.cs | 18 ++++++-- .../CosmosDBRepositoryMetricsTracker.cs | 46 +++++++++++++++++++ .../ICosmosDBRepositoryMetricsTracker.cs | 16 +++++++ 6 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs index e725e45b..e0937ddb 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -8,9 +8,9 @@ using System.Threading.Tasks; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; using Xunit; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -38,7 +38,7 @@ public CosmosDBRespositoryTests() ILogger< CosmosRepository< InternalDroneUtilization>>>(); - + _fakeResults = new List { new InternalDroneUtilization { Id = "d0001", @@ -93,7 +93,8 @@ public async Task WhenGetItemsAsync_ThenClientMakesAQuery() var repo = new CosmosRepository( _clientMockObject, _optionsMockObject, - _loggerDebug); + _loggerDebug, + Mock.Of>()); // Act var res = await repo.GetItemsAsync( @@ -103,8 +104,13 @@ public async Task WhenGetItemsAsync_ThenClientMakesAQuery() // Assert Assert.NotNull(res); Assert.Equal(_fakeResults.Count(), res.Count()); - Assert.True(res.All(r => r.PartitionKey == ownerId)); - Assert.True(res.All(r => r.DocumentType == typeof(InternalDroneUtilization).Name)); + Assert.All( + res, + r => + { + Assert.Equal(ownerId, r.PartitionKey); + Assert.Equal(typeof(InternalDroneUtilization).Name, r.DocumentType); + }); } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs index db9cb5f4..c6a12cc8 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs @@ -67,6 +67,7 @@ public DroneDeliveriesUtilizationIntegrationTests( DocumentType = typeof(InternalDroneUtilization).Name } }.AsQueryable())); + s.AddSingleton(Mock.Of>()); })) .CreateClient(); } @@ -118,6 +119,7 @@ public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusC .CreateDocumentClientMockObject( new List() .AsQueryable())); + s.AddSingleton(Mock.Of>()); })) .CreateClient(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs index ac023b67..5e76f8c5 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs @@ -4,12 +4,12 @@ // ------------------------------------------------------------ using System; +using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; -using Microsoft.Azure.Documents; namespace Fabrikam.DroneDelivery.DroneSchedulerService { @@ -27,6 +27,7 @@ public static IServiceCollection AddCosmosRepository( services.ConfigureOptions>(); services.AddSingleton, CosmosRepository>(); + services.AddSingleton, CosmosDBRepositoryMetricsTracker>(); return services; } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index 0e98c8e0..c86d123c 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.ApplicationInsights; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; @@ -17,21 +18,24 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { - public class CosmosRepository: ICosmosRepository + public class CosmosRepository : ICosmosRepository where T : BaseDocument { private readonly IDocumentClient _client; private readonly CosmosDBRepositoryOptions _options; private readonly ILogger> _logger; + private readonly ICosmosDBRepositoryMetricsTracker _metricsTracker; public CosmosRepository( IDocumentClient client, IOptions> options, - ILogger> logger) + ILogger> logger, + ICosmosDBRepositoryMetricsTracker metricsTracker) { this._client = client; this._options = options.Value; this._logger = logger; + this._metricsTracker = metricsTracker; } public async Task> GetItemsAsync( @@ -47,9 +51,11 @@ public async Task> GetItemsAsync( IDocumentQuery query = _client.CreateDocumentQuery( this._options.CollectionUri, - new FeedOptions { + new FeedOptions + { MaxItemCount = -1, - PartitionKey = new PartitionKey(partitionKey) }) + PartitionKey = new PartitionKey(partitionKey) + }) .Where(predicate) .Where(d => d.DocumentType == typeof(T).Name) .AsDocumentQuery(); @@ -59,7 +65,9 @@ public async Task> GetItemsAsync( _logger.LogInformation("Start: reading results from query"); while (query.HasMoreResults) { - results.AddRange(await query.ExecuteNextAsync()); + var feed = await query.ExecuteNextAsync(); + this._metricsTracker.TrackResponseMetrics(feed, this._options.CollectionUri.ToString(), partitionKey); + results.AddRange(feed); } _logger.LogInformation("End: reading results from query"); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs new file mode 100644 index 00000000..81725309 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System.Globalization; +using Microsoft.ApplicationInsights; +using Microsoft.Azure.Documents.Client; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class CosmosDBRepositoryMetricsTracker + : ICosmosDBRepositoryMetricsTracker + where T : BaseDocument + { + private const string RequestUnitsMetricId = "CosmosDb-RequestUnits"; + private const string CollectionDimensionName = "Collection"; + private const string DocumentDimensionName = "Document"; + private const string PartitionKeyDimensionName = "PartitionKey"; + private const string DocumentCountDimensionName = "DocumentCount"; + + private readonly Metric _metric; + + public CosmosDBRepositoryMetricsTracker(TelemetryClient telemetryClient) + { + this._metric = + telemetryClient.GetMetric( + RequestUnitsMetricId, + CollectionDimensionName, + DocumentDimensionName, + PartitionKeyDimensionName, + DocumentCountDimensionName); + } + + public void TrackResponseMetrics(FeedResponse response, string collection, string partitionKey) + { + this._metric.TrackValue( + response.RequestCharge, + collection, + typeof(T).Name, + partitionKey, + response.Count.ToString(CultureInfo.InvariantCulture)); + } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs new file mode 100644 index 00000000..5e0f5fb9 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.Azure.Documents.Client; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public interface ICosmosDBRepositoryMetricsTracker + where T : BaseDocument + { + void TrackResponseMetrics(FeedResponse response, string collection, string partitionKey); + } +} From 48aecabbe213926c3ff6f0b9db31d14878fe78c3 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 3 Jul 2019 17:48:09 +0000 Subject: [PATCH 180/246] Merged PR 819: Merged PR 809: drone utilization import for Cosmos DB drone utilization import for Cosmos DB drone utilization import for Cosmos DB a new console app that allows to bulk import data for DroneScheduler app. Specifically DroneUtilization documents with the following structure. It will be possible to pick out between deliveryid or ownerid as partitionkey value by switching true to false the --flattenPartitionKey arg - prescribe two collection creations with ~14 GB data each - first collection (utilization-cp) use a flatten partition key which happens to be 4X less efficient in terms of RU than the second one (utlization-sp). It is because cross partition vs single partition query performance. ```json { "id": "d0003054570", "partitionKey": "", "type": "InternalDroneUtilization", "ownerId": "o00042", "year": "1969", "month": "07", "travelledMiles": 466.977477407538, "assignedHours": 5.00024804519501 } ``` solved: #9228 Related work items: #9228 --- ...ry.DroneSchedulerService.BulkImport.csproj | 16 + .../ImportConfiguration.cs | 39 +++ .../Program.cs | 300 ++++++++++++++++++ .../Utils.cs | 59 ++++ 4 files changed, 414 insertions(+) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/ImportConfiguration.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Program.cs create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Utils.cs diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj new file mode 100644 index 00000000..013cdf20 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj @@ -0,0 +1,16 @@ + + + + Exe + netcoreapp2.2 + 7.1 + + + + + + + + + + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/ImportConfiguration.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/ImportConfiguration.cs new file mode 100644 index 00000000..5799d687 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/ImportConfiguration.cs @@ -0,0 +1,39 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport +{ + internal class ImportConfiguration + { + public string EndpointUrl { get; set; } + + public string AuthorizationKey { get; set; } + + public string DatabaseName { get; set; } + + public string CollectionName { get; set; } + + public string CollectionPartitionKey { get; set; } + + public int CollectionThroughput { get; set; } + + public int NumberOfBatches { get; set; } + + public long NumberOfDocuments { get; set; } + + public int NumberDocumentsPerPartitionExpFactor { get; set; } + + public string DocumentTypeName { get; set; } + + public bool FlattenPartitionKey { get; set; } + + public long NumberOfDocumentsPerBatch() + { + return (long)Math.Floor(((double)NumberOfDocuments) / NumberOfBatches); + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Program.cs new file mode 100644 index 00000000..56e0aa5c --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Program.cs @@ -0,0 +1,300 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.CosmosDB.BulkExecutor; +using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; +using Microsoft.Extensions.Configuration; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport +{ + class Program + { + private static readonly ConnectionPolicy ConnectionPolicy = + new ConnectionPolicy + { + ConnectionMode = ConnectionMode.Direct, + ConnectionProtocol = Protocol.Tcp + }; + + private static readonly Dictionary SwitchMappings = + new Dictionary + { + { "--auth-key", "importConfig:authorizationKey" }, + { "--endpoint-url", "importConfig:endpointUrl" }, + { "--database-name", "importConfig:databaseName" }, + { "--collection-name", "importConfig:collectionName" }, + { "--collection-partition-key", "importConfig:collectionPartitionKey" }, + { "--collection-throughput", "importConfig:collectionThroughput" }, + { "--document-type-name", "importConfig:documentTypeName" }, + { "--flatten-partition-key", "importConfig:flattenPartitionKey" }, + { "--number-of-batches", "importConfig:numberOfBatches" }, + { "--number-of-documents", "importConfig:numberOfDocuments" }, + { "--number-of-documents-exp-factor", "importConfig:numberOfDocumentsExpFactor" } + }; + + private readonly DocumentClient _client; + private readonly ImportConfiguration _importConfig; + + private readonly AsyncLazy _col; + private readonly AsyncLazy _bulkExecutor; + + private Program( + DocumentClient client, + ImportConfiguration importConfig) + { + this._client = client; + this._importConfig = importConfig; + + this._col = new AsyncLazy(async () + => await InitDbAndCollectionAsync().ConfigureAwait(false)); + + this._bulkExecutor = new AsyncLazy(async () => + { + // Set retry options high for initialization (default values). + _client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30; + _client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9; + + var bulkExecutor = new BulkExecutor(_client, await _col.Value); + await bulkExecutor.InitializeAsync(); + + // Set retries to 0 to pass control to bulk executor. + _client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0; + _client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0; + + return bulkExecutor; + }); + } + + private static async Task Main(string[] args) + { + try + { + IConfiguration config = new ConfigurationBuilder() + .AddCommandLine(args, SwitchMappings) + .Build(); + + var importCfg = config + .GetSection("importConfig") + .Get(); + + await Console.Out.WriteLineAsync("\n--------------------------------------------------------------------- "); + await Console.Out.WriteLineAsync($"Endpoint: {importCfg.EndpointUrl}"); + await Console.Out.WriteLineAsync($"Collections : {importCfg.DatabaseName}.{importCfg.CollectionName}"); + await Console.Out.WriteLineAsync("--------------------------------------------------------------------- \n"); + + using (var client = new DocumentClient( + new Uri(importCfg.EndpointUrl), + importCfg.AuthorizationKey, + ConnectionPolicy)) + { + var program = new Program(client, importCfg); + + await program.BulkImportAsync() + .ConfigureAwait(false); + } + } + catch (AggregateException e) + { + await Console.Out.WriteLineAsync($"Caught AggregateException in Main, Inner Exception:\n {e.Message}"); + } + catch (Exception e) + { + await Console.Out.WriteLineAsync($"Caught Exception in Main:\n {e.Message}"); + } + finally + { + await Console.Out.WriteLineAsync("\nPress any key to exit."); + Console.ReadKey(); + } + } + + private async Task BulkImportAsync() + { + var tokenSource = new CancellationTokenSource(); + var token = tokenSource.Token; + string partitionKeyProperty = (await _col.Value) + .PartitionKey + .Paths[0] + .Replace("/", ""); + + long totalNumberOfDocumentsInserted = 0; + double totalRequestUnitsConsumed = 0; + double totalTimeTakenSec = 0; + + for (int i = 0; i < _importConfig.NumberOfBatches; i++) + { + long numberDocsCurrentBatch = + _importConfig.NumberDocumentsPerPartitionExpFactor > 0 + ? ExpNumberOfDocumentsPerBatch(i) + : _importConfig.NumberOfDocumentsPerBatch(); + + var documentsToImportInBatch = new List(); + long seed = i * numberDocsCurrentBatch; + + await Console.Out.WriteLineAsync($"\nGenerating {numberDocsCurrentBatch} documents to import for batch {i}"); + for (int j = 0; j < numberDocsCurrentBatch; j++) + { + var id = (seed + j).ToString(); + string doc = Utils.GenerateSyntheticDoc( + id, + i, + j, + _importConfig.DocumentTypeName, + partitionKeyProperty, + _importConfig.FlattenPartitionKey); + + documentsToImportInBatch.Add(doc); + } + + var (batchDocsImported, batchReqUnitsConsumed, batchSeconds) = + await ImportBatch(i, documentsToImportInBatch, token); + + totalNumberOfDocumentsInserted += batchDocsImported; + totalRequestUnitsConsumed += batchReqUnitsConsumed; + totalTimeTakenSec += batchSeconds; + } + + await Console.Out.WriteLineAsync("\nOverall Summary:"); + await Console.Out.WriteLineAsync("--------------------------------------------------------------------- "); + await Console.Out.WriteLineAsync(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec", + totalNumberOfDocumentsInserted, + Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec), + Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec), + totalTimeTakenSec)); + await Console.Out.WriteLineAsync(String.Format("Average RU consumption per document: {0}", + (totalRequestUnitsConsumed / totalNumberOfDocumentsInserted))); + await Console.Out.WriteLineAsync("--------------------------------------------------------------------- "); + } + + private async Task> ImportBatch( + int batchNuber, + IEnumerable documentsToImportInBatch, + CancellationToken token) + { + await Console.Out.WriteLineAsync($"Executing bulk import for batch {batchNuber}"); + BulkImportResponse bulkImportResponse = null; + + do + { + try + { + bulkImportResponse = + await (await _bulkExecutor.Value).BulkImportAsync( + documents: documentsToImportInBatch, + enableUpsert: false, + disableAutomaticIdGeneration: true, + maxConcurrencyPerPartitionKeyRange: null, + maxInMemorySortingBatchSize: null, + cancellationToken: token); + } + catch (DocumentClientException de) + { + await Console.Out.WriteLineAsync($"Document client exception: {de.Message}"); + break; + } + catch (Exception e) + { + await Console.Out.WriteLineAsync($"Exception: {e.Message}"); + break; + } + } while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count()); + + await Console.Out.WriteLineAsync($"\nBatch Summary {batchNuber}:"); + await Console.Out.WriteLineAsync("--------------------------------------------------------------------- "); + await Console.Out.WriteLineAsync(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec", + bulkImportResponse.NumberOfDocumentsImported, + Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds), + Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds), + bulkImportResponse.TotalTimeTaken.TotalSeconds)); + await Console.Out.WriteLineAsync(String.Format("Average RU consumption per document: {0}", + (bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported))); + await Console.Out.WriteLineAsync("--------------------------------------------------------------------- "); + + return Tuple.Create( + bulkImportResponse.NumberOfDocumentsImported, + bulkImportResponse.TotalRequestUnitsConsumed, + bulkImportResponse.TotalTimeTaken.TotalSeconds); + } + + private long ExpNumberOfDocumentsPerBatch(double x) + => (long)(Math.Exp(x) * _importConfig.NumberDocumentsPerPartitionExpFactor); + + private async Task InitDbAndCollectionAsync() + { + try + { + Database db = _client + .CreateDatabaseQuery() + .Where(d => d.Id == _importConfig.DatabaseName) + .AsEnumerable() + .FirstOrDefault(); + + if (db != null) + { + await Console.Out.WriteLineAsync($"Deleting pre-existent database {_importConfig.DatabaseName}..."); + await _client.DeleteDatabaseAsync(db.SelfLink); + } + + await Console.Out.WriteLineAsync($"Creating database {_importConfig.DatabaseName}..."); + db = await _client.CreateDatabaseAsync( + new Database + { + Id = _importConfig.DatabaseName + }); + + var partitionKey = new PartitionKeyDefinition + { + Paths = new Collection + { + _importConfig.CollectionPartitionKey + } + }; + + var collection = new DocumentCollection + { + Id = _importConfig.CollectionName, + PartitionKey = partitionKey + }; + + await Console.Out.WriteLineAsync($"Creating collection {_importConfig.CollectionName} with {_importConfig.CollectionThroughput} RU/s..."); + collection = await _client.CreateDocumentCollectionAsync( + db.SelfLink, + collection, + new RequestOptions + { + OfferThroughput = _importConfig.CollectionThroughput + }); + + return collection; + } + catch (Exception e) + { + await Console.Out.WriteLineAsync($"Unable to initialize, exception message: {e.Message}"); + throw; + } + } + + private class AsyncLazy : Lazy> + { + public AsyncLazy(Func valueFactory) : + base(() => Task.Run(valueFactory)) + { + } + + public AsyncLazy(Func> taskFactory) : + base(() => Task.Run(taskFactory)) + { + } + } + } +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Utils.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Utils.cs new file mode 100644 index 00000000..6c09642a --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Utils.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Microsoft.Extensions.Configuration; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport +{ + internal static class Utils + { + internal static String GenerateSyntheticDoc( + string id, + int batchNumber, + int docNumberCurrentBatch, + string docTypeName, + string partitionKeyProperty, + bool flattenPartitionKey) + { + const string deliveryPrefix = "d000"; + const string ownerPrefix = "o000"; + const double MinMiles = 30d; + const double MaxMiles = 500d; + const double MinHours = 1d; + const double MaxHours = 30d; + + string deliveryId = string.Concat(deliveryPrefix, id); + string ownerId = string.Concat(ownerPrefix, docNumberCurrentBatch); + + int year = DateTime.UtcNow.Year + (batchNumber / 12); + int month = batchNumber % 12 + 1; + + var random = new Random(); + + double traveledMiles = MinMiles + + random.NextDouble() + * (MaxMiles - MinMiles); + double assignedHours = MinHours + + random.NextDouble() + * (MaxHours - MinHours); + + string partitonKeyValue = flattenPartitionKey + ? deliveryId + : ownerId; + + return "{\n" + + " \"id\": \"" + deliveryId + "\",\n" + + " \"" + partitionKeyProperty + "\": \"" + partitonKeyValue + "\",\n" + + " \"type\": \"" + docTypeName + "\",\n" + + " \"ownerId\": \"" + ownerId + "\",\n" + + " \"travelledMiles\": " + traveledMiles.ToString() + ",\n" + + " \"assignedHours\": " + assignedHours.ToString() + ",\n" + + " \"year\": " + year.ToString() + ",\n" + + " \"month\": " + month.ToString() + "\n" + + "}"; + } + } +} From 4332e204ba956a496a4ca9971ee2b244770108e5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 4 Jul 2019 16:25:46 +0000 Subject: [PATCH 181/246] Merged PR 821: Bug fixing: ingress, chart and instructions bug fixing: - fix package and dronescheduler deployment steps (ingress) - fix missing err messages (dronesaheduler chart) - fix add collection name to deployment steps related: #9240 --- .../dronescheduler/templates/dronescheduler-deploy.yaml | 6 +++--- deployment.md | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index 098e56fe..efe25813 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -50,9 +50,9 @@ spec: - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} - name: COSMOSDB_DATABASEID - value: {{ required .Values.cosmosdb.id }} - - name: COSMOSDB_COLLECTIONID - value: {{ required .Values.cosmosdb.collectionid }} + value: {{ required "Cosmos DB name is required" .Values.cosmosdb.id }} + - name: COSMOSDB_COLLECTIONID + value: {{ required "Cosmos DB container name is required" .Values.cosmosdb.collectionid }} - name: LOGGING__ApplicationInsights__LOGLEVEL__DEFAULT value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy diff --git a/deployment.md b/deployment.md index 38eb8a0a..562508a6 100644 --- a/deployment.md +++ b/deployment.md @@ -290,6 +290,9 @@ export COSMOSDB_COL_NAME=packages helm install $HELM_CHARTS/package/ \ --set image.tag=0.1.0 \ --set image.repository=package \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=package \ + --set ingress.hosts[0].tls=false \ --set secrets.appinsights.ikey=$AI_IKEY \ --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ @@ -423,7 +426,8 @@ export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP export DRONESCHEDULER_COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerCosmosDbName.value -o tsv) && \ export ENDPOINT_URL=$(az cosmosdb show -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query documentEndpoint -o tsv) && \ export AUTH_KEY=$(az cosmosdb list-keys -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query primaryMasterKey -o tsv) && \ -export DATABASE_NAME="invoicing" +export DATABASE_NAME="invoicing-sp" && \ +export COLLECTION_NAME="utilization-sp" ``` Build the dronescheduler services @@ -458,6 +462,9 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set image.tag=0.1.0 \ --set image.repository=dronescheduler \ --set dockerregistry=$ACR_SERVER \ + --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts[0].serviceName=dronescheduler \ + --set ingress.hosts[0].tls=false \ --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ From 898e1a76c686b0f4a67851fcbacdf4284e8bac3d Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Thu, 4 Jul 2019 16:49:49 +0000 Subject: [PATCH 182/246] Merged PR 822: Tweak the behavior of the cosmosdb connection using feature toggles Allows the connection mode and the use of partition keys in the query to be driven by feature toggles Related work items: #9242 --- azuredeploy.json | 26 +++++++++- .../CosmosDBRespositoryTests.cs | 47 ++++++++++++++++- .../InvoicingRepositoryTests.cs | 51 +++++++++++++++++-- .../appsettings.Test.json | 4 +- .../CosmosDBExtensions.cs | 18 +++++-- ...DroneDelivery.DroneSchedulerService.csproj | 1 + .../ConfigureCosmosDBRepositoryOptions.cs | 6 +++ .../Services/CosmosDBConnectionOptions.cs | 18 +++++++ .../Services/CosmosDBRepository.cs | 6 +-- .../CosmosDBRepositoryMetricsTracker.cs | 2 +- .../Services/CosmosDBRepositoryOptions.cs | 2 + .../Services/InvoicingRepository.cs | 20 +++++--- .../Startup.cs | 3 ++ .../appsettings.json | 5 +- 14 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs diff --git a/azuredeploy.json b/azuredeploy.json index 4e3a7904..1c6d4854 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -689,7 +689,7 @@ }, { "type": "secrets", - "name": "CosmosDB-Endpoint", + "name": "CosmosDBEndpoint", "apiVersion": "2015-06-01", "properties": { "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName)).documentEndpoint]" @@ -701,7 +701,7 @@ }, { "type": "secrets", - "name": "CosmosDB-Key", + "name": "CosmosDBKey", "apiVersion": "2015-06-01", "properties": { "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName), '2016-03-31').primaryMasterKey]" @@ -710,6 +710,28 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]", "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName)]" ] + }, + { + "type": "secrets", + "name": "CosmosDBConnectionMode", + "apiVersion": "2015-06-01", + "properties": { + "value": "Direct" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] + }, + { + "type": "secrets", + "name": "COSMOSDB_MAXPARALLELISM", + "apiVersion": "2015-06-01", + "properties": { + "value": "-1" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] } ] }, diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs index e0937ddb..e1d37bdb 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -85,7 +86,7 @@ public CosmosDBRespositoryTests() } [Fact] - public async Task WhenGetItemsAsync_ThenClientMakesAQuery() + public async Task WhenGetItemsAsyncWithPartitionId_ThenClientMakesAQueryWithPartitionId() { // Arrange string ownerId = "o00042"; @@ -111,6 +112,50 @@ public async Task WhenGetItemsAsync_ThenClientMakesAQuery() Assert.Equal(ownerId, r.PartitionKey); Assert.Equal(typeof(InternalDroneUtilization).Name, r.DocumentType); }); + Mock.Get(_clientMockObject) + .Verify(dc => + dc.CreateDocumentQuery( + It.IsAny(), + It.Is(fo => + fo.PartitionKey != null + && fo.PartitionKey.ToString().Contains(ownerId) + && fo.EnableCrossPartitionQuery == false))); + } + + [Fact] + public async Task WhenGetItemsAsyncWithoutPartitionId_ThenClientMakesAQueryWithoutPartitionIdAndEnablesCrossPartition() + { + // Arrange + string ownerId = "o00042"; + + var repo = new CosmosRepository( + _clientMockObject, + _optionsMockObject, + _loggerDebug, + Mock.Of>()); + + // Act + var res = await repo.GetItemsAsync( + p => true, + null); + + // Assert + Assert.NotNull(res); + Assert.Equal(_fakeResults.Count(), res.Count()); + Assert.All( + res, + r => + { + Assert.Equal(ownerId, r.PartitionKey); + Assert.Equal(typeof(InternalDroneUtilization).Name, r.DocumentType); + }); + Mock.Get(_clientMockObject) + .Verify(dc => + dc.CreateDocumentQuery( + It.IsAny(), + It.Is(fo => + fo.PartitionKey == null + && fo.EnableCrossPartitionQuery == true))); } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs index bb360a93..dc989876 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs @@ -8,10 +8,11 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; -using Moq; -using Xunit; +using Microsoft.FeatureManagement; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Moq; +using Xunit; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests { @@ -24,8 +25,13 @@ public async Task WhenGetAggregatedInvoincingData_ThenInvokesDroneUtilizationRep string ownerId = "o00042"; int year = 2019; int month = 6; + var cosmosDbMock = new Mock>(); - var repo = new InvoicingRepository(cosmosDbMock.Object); + + var featureToggleMock = new Mock(); + featureToggleMock.Setup(fm => fm.IsEnabled(It.IsAny())).Returns(true); + + var repo = new InvoicingRepository(cosmosDbMock.Object, featureToggleMock.Object); // Act var result = await repo.GetAggreatedInvoincingDataAsync(ownerId, year, month); @@ -42,6 +48,36 @@ public async Task WhenGetAggregatedInvoincingData_ThenInvokesDroneUtilizationRep Times.Once); } + [Fact] + public async Task WhenGetAggregatedInvoincingDataAndFeatureForPartitionKeyIsDisabled_ThenInvokesDroneUtilizationRepositoryWithoutPartitionKey() + { + // Arrange + string ownerId = "o00042"; + int year = 2019; + int month = 6; + + var cosmosDbMock = new Mock>(); + + var featureToggleMock = new Mock(); + featureToggleMock.Setup(fm => fm.IsEnabled(It.IsAny())).Returns(false); + + var repo = new InvoicingRepository(cosmosDbMock.Object, featureToggleMock.Object); + + // Act + var result = await repo.GetAggreatedInvoincingDataAsync(ownerId, year, month); + + // Assert + Assert.NotNull(result); + Assert.Equal(0d, result.Item1); + Assert.Equal(0d, result.Item2); + cosmosDbMock + .Verify(p => + p.GetItemsAsync( + It.IsAny>>(), + null), + Times.Once); + } + [Fact] public async Task WhenGetAggregatedInvoincingDataForAValidPeriod_ThenRepoReturnsData() { @@ -49,6 +85,7 @@ public async Task WhenGetAggregatedInvoincingDataForAValidPeriod_ThenRepoReturns string ownerId = "o00042"; int year = 2019; int month = 6; + var invoicingData = new List { new InternalDroneUtilization{ TraveledMiles=10d, @@ -65,10 +102,14 @@ public async Task WhenGetAggregatedInvoincingDataForAValidPeriod_ThenRepoReturns It.IsAny>>(), ownerId)) .ReturnsAsync(invoicingData.AsEnumerable()); - var repo = new InvoicingRepository(cosmosDbMock.Object); + + var featureToggleMock = new Mock(); + featureToggleMock.Setup(fm => fm.IsEnabled(It.IsAny())).Returns(true); + + var repo = new InvoicingRepository(cosmosDbMock.Object, featureToggleMock.Object); // Act - var (traveledMiles, assignedHours) = + var (traveledMiles, assignedHours) = await repo.GetAggreatedInvoincingDataAsync( ownerId, year, diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json index eb76cf6f..031a67b9 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/appsettings.Test.json @@ -15,6 +15,6 @@ "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], "WriteTo": [] }, - "CosmosDB-Endpoint": "https://fake-invoicing.documents.azure.com:443/", - "CosmosDB-Key": "ZmFrZXBhc3MK" + "CosmosDBEndpoint": "https://fake-invoicing.documents.azure.com:443/", + "CosmosDBKey": "ZmFrZXBhc3MK" } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs index 5e76f8c5..6bb990eb 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs @@ -4,10 +4,12 @@ // ------------------------------------------------------------ using System; +using System.Linq; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; @@ -20,12 +22,22 @@ public static IServiceCollection AddCosmosRepository( IConfiguration config) where T : BaseDocument { - var endpoint = config["CosmosDB-Endpoint"]; - var key = config["CosmosDB-Key"]; + services.AddSingleton(s => + { + var options = s.GetRequiredService>(); - services.AddSingleton(s => new DocumentClient(new Uri(endpoint), key)); + return new DocumentClient( + new Uri(options.Value.CosmosDBEndpoint), + options.Value.CosmosDBKey, + connectionPolicy: new ConnectionPolicy { ConnectionMode = options.Value.CosmosDBConnectionMode }); + }); services.ConfigureOptions>(); + if (services.Any(sr => sr.ServiceType == typeof(CosmosDBConnectionOptions)) == false) + { + services.Configure(config); + } + services.AddSingleton, CosmosRepository>(); services.AddSingleton, CosmosDBRepositoryMetricsTracker>(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj index b89e0d9f..6fb08cb6 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj @@ -19,6 +19,7 @@ + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs index fb64fed5..27b07b72 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------ using System; +using System.Globalization; using System.Linq; using Microsoft.Azure.Documents; using Microsoft.Extensions.Configuration; @@ -47,6 +48,11 @@ public void Configure(CosmosDBRepositoryOptions options) options.CollectionUri = uri; } } + + if (string.IsNullOrEmpty(_config["COSMOSDB_MAXPARALLELISM"]) == false) + { + options.MaxParallelism = int.Parse(_config["COSMOSDB_MAXPARALLELISM"], CultureInfo.InvariantCulture); + } } } } \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs new file mode 100644 index 00000000..becdf8e0 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using Microsoft.Azure.Documents.Client; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public class CosmosDBConnectionOptions + { + public string CosmosDBEndpoint { get; set; } + + public string CosmosDBKey { get; set; } + + public ConnectionMode CosmosDBConnectionMode { get; set; } + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index c86d123c..50a59f83 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; -using Microsoft.ApplicationInsights; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; @@ -53,8 +52,9 @@ public async Task> GetItemsAsync( this._options.CollectionUri, new FeedOptions { - MaxItemCount = -1, - PartitionKey = new PartitionKey(partitionKey) + MaxItemCount = this._options.MaxParallelism, + PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : null, + EnableCrossPartitionQuery = partitionKey == null }) .Where(predicate) .Where(d => d.DocumentType == typeof(T).Name) diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs index 81725309..55e6c03e 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs @@ -39,7 +39,7 @@ public void TrackResponseMetrics(FeedResponse response, string collection, st response.RequestCharge, collection, typeof(T).Name, - partitionKey, + partitionKey ?? "", response.Count.ToString(CultureInfo.InvariantCulture)); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs index 3b106927..66361bdc 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs @@ -12,5 +12,7 @@ public class CosmosDBRepositoryOptions where T : BaseDocument { public Uri CollectionUri { get; set; } + + public int MaxParallelism { get; set; } = -1; } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs index 27ac3e8c..f7d5c368 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs @@ -8,17 +8,24 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.FeatureManagement; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { public class InvoicingRepository : IInvoicingRepository { - private ICosmosRepository _repository; + private const string UsePartitionKeyFeatureName = "UsePartitionKey"; + private readonly ICosmosRepository _repository; + private readonly IFeatureManager _featureManager; public InvoicingRepository( - ICosmosRepository repository) - => this._repository = repository; + ICosmosRepository repository, + IFeatureManager featureManager) + { + this._repository = repository; + this._featureManager = featureManager; + } public async Task> GetItemsAsync( Expression> predicate, @@ -35,9 +42,10 @@ public async Task> GetAggreatedInvoincingDataAsync( int year, int month) { - var results = await this.GetItemsAsync( - d => d.Year == year && d.Month == month, - ownerId); + var results = await + (this._featureManager.IsEnabled(UsePartitionKeyFeatureName) + ? this.GetItemsAsync(d => d.Year == year && d.Month == month, ownerId) + : this.GetItemsAsync(d => d.Year == year && d.Month == month && d.OwnerId == ownerId, null)); var result = results .Aggregate( diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs index acebffe2..11e92754 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.FeatureManagement; using Swashbuckle.AspNetCore.Swagger; using Serilog; using Serilog.Formatting.Compact; @@ -45,6 +46,8 @@ public void ConfigureServices(IServiceCollection services) { services.AddSingleton(); + services.AddFeatureManagement(); + // Configure AppInsights services.AddApplicationInsightsKubernetesEnricher(); services.AddApplicationInsightsTelemetry(Configuration); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json index 8e39a665..66139898 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/appsettings.json @@ -14,5 +14,8 @@ "MinimumLevel": "Verbose", "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ], "WriteTo": [] + }, + "FeatureManagement": { + "UsePartitionKey": false } -} +} \ No newline at end of file From 915933a0ab6d6e721ac9400bc8e42958efd23966 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 8 Jul 2019 12:55:23 +0000 Subject: [PATCH 183/246] Merged PR 823: Tweak dimensions for the consumption metric --- .../Services/ConfigureCosmosDBRepositoryOptions.cs | 7 +++++-- .../Services/CosmosDBRepository.cs | 4 +++- .../Services/CosmosDBRepositoryOptions.cs | 4 ++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs index 27b07b72..d2344150 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -29,9 +29,12 @@ public ConfigureCosmosDBRepositoryOptions( public void Configure(CosmosDBRepositoryOptions options) { + options.DatabaseId = _config["COSMOSDB_DATABASEID"]; + options.CollectionId = _config["COSMOSDB_COLLECTIONID"]; + Database db = _client .CreateDatabaseQuery() - .Where(d => d.Id == _config["COSMOSDB_DATABASEID"]) + .Where(d => d.Id == options.DatabaseId) .AsEnumerable() .FirstOrDefault(); @@ -39,7 +42,7 @@ public void Configure(CosmosDBRepositoryOptions options) { DocumentCollection col = _client .CreateDocumentCollectionQuery(db.SelfLink) - .Where(d => d.Id == _config["COSMOSDB_COLLECTIONID"]) + .Where(d => d.Id == options.CollectionId) .AsEnumerable() .FirstOrDefault(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index 50a59f83..0175d50e 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -24,6 +24,7 @@ public class CosmosRepository : ICosmosRepository private readonly CosmosDBRepositoryOptions _options; private readonly ILogger> _logger; private readonly ICosmosDBRepositoryMetricsTracker _metricsTracker; + private readonly string _collectionIdentifier; public CosmosRepository( IDocumentClient client, @@ -35,6 +36,7 @@ public CosmosRepository( this._options = options.Value; this._logger = logger; this._metricsTracker = metricsTracker; + this._collectionIdentifier = $"{options.Value.DatabaseId}.{options.Value.CollectionId}"; } public async Task> GetItemsAsync( @@ -66,7 +68,7 @@ public async Task> GetItemsAsync( while (query.HasMoreResults) { var feed = await query.ExecuteNextAsync(); - this._metricsTracker.TrackResponseMetrics(feed, this._options.CollectionUri.ToString(), partitionKey); + this._metricsTracker.TrackResponseMetrics(feed, this._collectionIdentifier, partitionKey); results.AddRange(feed); } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs index 66361bdc..42ab1da0 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs @@ -11,6 +11,10 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services public class CosmosDBRepositoryOptions where T : BaseDocument { + public string DatabaseId { get; set; } + + public string CollectionId { get; set; } + public Uri CollectionUri { get; set; } public int MaxParallelism { get; set; } = -1; From dd531eb501138ae15a44964bc8fc8f0c24cc14ce Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 10 Jul 2019 13:10:10 +0000 Subject: [PATCH 184/246] Merged PR 829: change database and collection name change database and collection name from load test executions we find a single dataset and collection is enough to reprod fan out queries as long as owner id is not provided as partition key value related: #9241, #9243 Related work items: #9241, #9243 --- deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment.md b/deployment.md index 562508a6..81fd4b64 100644 --- a/deployment.md +++ b/deployment.md @@ -426,8 +426,8 @@ export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP export DRONESCHEDULER_COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerCosmosDbName.value -o tsv) && \ export ENDPOINT_URL=$(az cosmosdb show -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query documentEndpoint -o tsv) && \ export AUTH_KEY=$(az cosmosdb list-keys -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query primaryMasterKey -o tsv) && \ -export DATABASE_NAME="invoicing-sp" && \ -export COLLECTION_NAME="utilization-sp" +export DATABASE_NAME="invoicing" && \ +export COLLECTION_NAME="utilization" ``` Build the dronescheduler services From 7e66ab5300d4876617124b8bdea1ced10c04bb78 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Wed, 10 Jul 2019 20:32:16 +0000 Subject: [PATCH 185/246] Merged PR 825: Trace cosmosdb results, Update keyvault for drone scheduler Tracks a trace along with a metric Move config setup to program to avoid disconnected config. Renames max parallelism secret name to make it a valid key vault name. adds feature toggle to key vault --- azuredeploy.json | 17 +++- .../CosmosDBRespositoryTests.cs | 10 ++- ...neDeliveriesUtilizationIntegrationTests.cs | 10 ++- .../Program.cs | 10 +++ .../ConfigureCosmosDBRepositoryOptions.cs | 4 +- .../Services/CosmosDBRepository.cs | 43 +++++----- .../CosmosDBRepositoryMetricsTracker.cs | 78 +++++++++++++++++-- .../ICosmosDBRepositoryMetricsTracker.cs | 3 +- .../ICosmosDBRepositoryQueryMetricsTracker.cs | 17 ++++ .../Startup.cs | 25 ++---- 10 files changed, 163 insertions(+), 54 deletions(-) create mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs diff --git a/azuredeploy.json b/azuredeploy.json index 1c6d4854..283b162f 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -724,7 +724,7 @@ }, { "type": "secrets", - "name": "COSMOSDB_MAXPARALLELISM", + "name": "CosmosDBMaxParallelism", "apiVersion": "2015-06-01", "properties": { "value": "-1" @@ -732,6 +732,17 @@ "dependsOn": [ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" ] + }, + { + "type": "secrets", + "name": "FeatureManagement--UsePartitionKey", + "apiVersion": "2015-06-01", + "properties": { + "value": "false" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] } ] }, @@ -979,6 +990,10 @@ "value": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "type": "string" }, + "droneSchedulerKeyVaultName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName]", + "type": "string" + }, "droneSchedulerKeyVaultUri": { "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)).vaultUri]", "type": "string" diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs index e1d37bdb..0f6e3f2a 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -26,6 +26,7 @@ public class CosmosDBRespositoryTests private readonly IDocumentClient _clientMockObject; private readonly IOptions> _optionsMockObject; + private readonly ICosmosDBRepositoryMetricsTracker _metricsTrackerMockObject; private readonly IQueryable _fakeResults; @@ -83,6 +84,11 @@ public CosmosDBRespositoryTests() .Returns(fakeOptionsValue); _optionsMockObject = optionsMock.Object; + + _metricsTrackerMockObject = + Mock.Of>( + t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) + == Mock.Of>()); } [Fact] @@ -95,7 +101,7 @@ public async Task WhenGetItemsAsyncWithPartitionId_ThenClientMakesAQueryWithPart _clientMockObject, _optionsMockObject, _loggerDebug, - Mock.Of>()); + _metricsTrackerMockObject); // Act var res = await repo.GetItemsAsync( @@ -132,7 +138,7 @@ public async Task WhenGetItemsAsyncWithoutPartitionId_ThenClientMakesAQueryWitho _clientMockObject, _optionsMockObject, _loggerDebug, - Mock.Of>()); + _metricsTrackerMockObject); // Act var res = await repo.GetItemsAsync( diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs index c6a12cc8..720c0bc0 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs @@ -67,7 +67,10 @@ public DroneDeliveriesUtilizationIntegrationTests( DocumentType = typeof(InternalDroneUtilization).Name } }.AsQueryable())); - s.AddSingleton(Mock.Of>()); + s.AddSingleton( + Mock.Of>( + t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) + == Mock.Of>())); })) .CreateClient(); } @@ -119,7 +122,10 @@ public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusC .CreateDocumentClientMockObject( new List() .AsQueryable())); - s.AddSingleton(Mock.Of>()); + s.AddSingleton( + Mock.Of>( + t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) + == Mock.Of>())); })) .CreateClient(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs index f1d74e7f..2ed62f07 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs @@ -5,6 +5,7 @@ using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Serilog; @@ -25,6 +26,15 @@ public static void Main(string[] args) public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) .UseStartup() + .ConfigureAppConfiguration(configurationBuilder => + { + var buildConfig = configurationBuilder.Build(); + + if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) + { + configurationBuilder.AddAzureKeyVault(keyVaultUri); + } + }) .ConfigureLogging((hostingContext, loggingBuilder) => { loggingBuilder.AddApplicationInsights(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs index d2344150..41fb9784 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -52,9 +52,9 @@ public void Configure(CosmosDBRepositoryOptions options) } } - if (string.IsNullOrEmpty(_config["COSMOSDB_MAXPARALLELISM"]) == false) + if (string.IsNullOrEmpty(_config["CosmosDBMaxParallelism"]) == false) { - options.MaxParallelism = int.Parse(_config["COSMOSDB_MAXPARALLELISM"], CultureInfo.InvariantCulture); + options.MaxParallelism = int.Parse(_config["CosmosDBMaxParallelism"], CultureInfo.InvariantCulture); } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index 0175d50e..065d7492 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -43,33 +43,36 @@ public async Task> GetItemsAsync( Expression> predicate, string partitionKey) { + var results = new List(); + using (_logger.BeginScope(nameof(GetItemsAsync))) { _logger.LogInformation( "partitionKey: {PartitionKey}", partitionKey); - IDocumentQuery query = - _client.CreateDocumentQuery( - this._options.CollectionUri, - new FeedOptions - { - MaxItemCount = this._options.MaxParallelism, - PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : null, - EnableCrossPartitionQuery = partitionKey == null - }) - .Where(predicate) - .Where(d => d.DocumentType == typeof(T).Name) - .AsDocumentQuery(); - - var results = new List(); - - _logger.LogInformation("Start: reading results from query"); - while (query.HasMoreResults) + using (var queryMetricsTracker = this._metricsTracker.GetQueryMetricsTracker(this._collectionIdentifier, partitionKey)) { - var feed = await query.ExecuteNextAsync(); - this._metricsTracker.TrackResponseMetrics(feed, this._collectionIdentifier, partitionKey); - results.AddRange(feed); + IDocumentQuery query = + _client.CreateDocumentQuery( + this._options.CollectionUri, + new FeedOptions + { + MaxItemCount = this._options.MaxParallelism, + PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : null, + EnableCrossPartitionQuery = partitionKey == null + }) + .Where(predicate) + .Where(d => d.DocumentType == typeof(T).Name) + .AsDocumentQuery(); + + _logger.LogInformation("Start: reading results from query"); + while (query.HasMoreResults) + { + var feed = await query.ExecuteNextAsync(); + queryMetricsTracker.TrackResponseMetrics(feed); + results.AddRange(feed); + } } _logger.LogInformation("End: reading results from query"); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs index 55e6c03e..45e054b5 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs @@ -3,8 +3,10 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using System.Collections.Generic; using System.Globalization; using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; using Microsoft.Azure.Documents.Client; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -19,11 +21,14 @@ public class CosmosDBRepositoryMetricsTracker private const string DocumentDimensionName = "Document"; private const string PartitionKeyDimensionName = "PartitionKey"; private const string DocumentCountDimensionName = "DocumentCount"; + private const string RequestChargeKey = "RequestCharge"; + private readonly TelemetryClient _telemetryClient; private readonly Metric _metric; public CosmosDBRepositoryMetricsTracker(TelemetryClient telemetryClient) { + this._telemetryClient = telemetryClient; this._metric = telemetryClient.GetMetric( RequestUnitsMetricId, @@ -33,14 +38,75 @@ public CosmosDBRepositoryMetricsTracker(TelemetryClient telemetryClient) DocumentCountDimensionName); } + public ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker(string collection, string partitionKey) + { + return new QueryMetricsTracker(this._telemetryClient, this._metric, collection, partitionKey); + } + public void TrackResponseMetrics(FeedResponse response, string collection, string partitionKey) { - this._metric.TrackValue( - response.RequestCharge, - collection, - typeof(T).Name, - partitionKey ?? "", - response.Count.ToString(CultureInfo.InvariantCulture)); + } + + private sealed class QueryMetricsTracker : ICosmosDBRepositoryQueryMetricsTracker + { + private readonly TelemetryClient _telemetryClient; + private readonly Metric _metric; + private readonly string _collection; + private readonly string _partitionKey; + + private double _totalCharge; + private long _totalDocumentCount; + + + public QueryMetricsTracker(TelemetryClient telemetryClient, Metric metric, string collection, string partitionKey) + { + this._telemetryClient = telemetryClient; + this._metric = metric; + this._collection = collection; + this._partitionKey = partitionKey; + } + + public void Dispose() + { + this._telemetryClient.TrackTrace( + "Completed document query", + SeverityLevel.Information, + new Dictionary + { + [RequestChargeKey] = this._totalCharge.ToString(CultureInfo.InvariantCulture), + [CollectionDimensionName] = this._collection, + [DocumentDimensionName] = typeof(T).Name, + [PartitionKeyDimensionName] = this._partitionKey, + [DocumentCountDimensionName] = this._totalDocumentCount.ToString(CultureInfo.InvariantCulture) + }); + } + + public void TrackResponseMetrics(FeedResponse response) + { + var responseCount = response.Count.ToString(CultureInfo.InvariantCulture); + + this._metric.TrackValue( + response.RequestCharge, + this._collection, + typeof(T).Name, + this._partitionKey ?? "", + responseCount); + + this._telemetryClient.TrackTrace( + "Partial document query", + SeverityLevel.Information, + new Dictionary + { + [RequestChargeKey] = response.RequestCharge.ToString(CultureInfo.InvariantCulture), + [CollectionDimensionName] = this._collection, + [DocumentDimensionName] = typeof(T).Name, + [PartitionKeyDimensionName] = this._partitionKey, + [DocumentCountDimensionName] = responseCount + }); + + this._totalCharge += response.RequestCharge; + this._totalDocumentCount += response.Count; + } } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs index 5e0f5fb9..64f0db7d 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using Microsoft.Azure.Documents.Client; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services @@ -11,6 +10,6 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services public interface ICosmosDBRepositoryMetricsTracker where T : BaseDocument { - void TrackResponseMetrics(FeedResponse response, string collection, string partitionKey); + ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker(string collection, string partitionKey); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs new file mode 100644 index 00000000..d302bee9 --- /dev/null +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using Microsoft.Azure.Documents.Client; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; + +namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services +{ + public interface ICosmosDBRepositoryQueryMetricsTracker : IDisposable + where T : BaseDocument + { + void TrackResponseMetrics(FeedResponse response); + } +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs index 11e92754..8cfb014e 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs @@ -11,35 +11,22 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.FeatureManagement; -using Swashbuckle.AspNetCore.Swagger; -using Serilog; -using Serilog.Formatting.Compact; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; +using Serilog; +using Serilog.Formatting.Compact; +using Swashbuckle.AspNetCore.Swagger; namespace Fabrikam.DroneDelivery.DroneSchedulerService { public class Startup { - public Startup(IHostingEnvironment env) + public Startup(IConfiguration configuration) { - var builder = new ConfigurationBuilder() - .SetBasePath(env.ContentRootPath) - .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) - .AddEnvironmentVariables(); - - var buildConfig = builder.Build(); - - if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) - { - builder.AddAzureKeyVault(keyVaultUri); - } - - Configuration = builder.Build(); + Configuration = configuration; } - public IConfigurationRoot Configuration { get; } + public IConfiguration Configuration { get; } // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) From c6d5e6a2b01acc1680b7e57bfc4a554e195d95ea Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 15 Jul 2019 16:10:52 +0000 Subject: [PATCH 186/246] Merged PR 831: change default connection mode to gateway change default connection mode to gateway --- azuredeploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azuredeploy.json b/azuredeploy.json index 283b162f..9c7ac66a 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -716,7 +716,7 @@ "name": "CosmosDBConnectionMode", "apiVersion": "2015-06-01", "properties": { - "value": "Direct" + "value": "Gateway" }, "dependsOn": [ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" From e4689009d350c04db97796bb1d05a6e6abc8afc3 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 16 Jul 2019 14:17:44 +0000 Subject: [PATCH 187/246] Merged PR 832: Enhance cosmosdb configuration and telemetry Add config knobs for maxconnection, protocol and max buffered item count based on connection mode improve enrichment for telemetry fix configuration of max parallelism Related work items: #9242 --- azuredeploy.json | 33 ++++++ .../CosmosDBRespositoryTests.cs | 11 +- ...neDeliveriesUtilizationIntegrationTests.cs | 22 +++- .../Utils/DocumentClientMock.cs | 4 + .../CosmosDBExtensions.cs | 12 ++- .../ConfigureCosmosDBRepositoryOptions.cs | 5 + .../Services/CosmosDBConnectionOptions.cs | 4 + .../Services/CosmosDBRepository.cs | 15 ++- .../CosmosDBRepositoryMetricsTracker.cs | 102 +++++++++++++----- .../Services/CosmosDBRepositoryOptions.cs | 4 +- .../ICosmosDBRepositoryMetricsTracker.cs | 10 +- 11 files changed, 182 insertions(+), 40 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 9c7ac66a..90e86fcf 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -722,6 +722,28 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" ] }, + { + "type": "secrets", + "name": "CosmosDBConnectionProtocol", + "apiVersion": "2015-06-01", + "properties": { + "value": "Https" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] + }, + { + "type": "secrets", + "name": "CosmosDBMaxConnectionsLimit", + "apiVersion": "2015-06-01", + "properties": { + "value": "50" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] + }, { "type": "secrets", "name": "CosmosDBMaxParallelism", @@ -733,6 +755,17 @@ "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" ] }, + { + "type": "secrets", + "name": "CosmosDBMaxBufferedItemCount", + "apiVersion": "2015-06-01", + "properties": { + "value": "0" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + ] + }, { "type": "secrets", "name": "FeatureManagement--UsePartitionKey", diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs index 0f6e3f2a..24074856 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -87,8 +87,15 @@ public CosmosDBRespositoryTests() _metricsTrackerMockObject = Mock.Of>( - t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) - == Mock.Of>()); + t => t.GetQueryMetricsTracker( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()) + == Mock.Of>()); } [Fact] diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs index 720c0bc0..80928e49 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs @@ -69,8 +69,15 @@ public DroneDeliveriesUtilizationIntegrationTests( }.AsQueryable())); s.AddSingleton( Mock.Of>( - t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) - == Mock.Of>())); + t => t.GetQueryMetricsTracker( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()) + == Mock.Of>())); })) .CreateClient(); } @@ -124,8 +131,15 @@ public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusC .AsQueryable())); s.AddSingleton( Mock.Of>( - t => t.GetQueryMetricsTracker(It.IsAny(), It.IsAny()) - == Mock.Of>())); + t => t.GetQueryMetricsTracker( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()) + == Mock.Of>())); })) .CreateClient(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs index aaceae45..be17509a 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs @@ -123,6 +123,10 @@ public static IDocumentClient CreateDocumentClientMockObject( .Setup(q => q.CreateDocumentCollectionQuery(It.IsAny(), null)) .Returns(mockCollectionQuery.Object); + clientMock + .Setup(q => q.ConnectionPolicy) + .Returns(new ConnectionPolicy { ConnectionMode = ConnectionMode.Gateway }); + return clientMock.Object; } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs index 6bb990eb..c801db0c 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs @@ -26,10 +26,20 @@ public static IServiceCollection AddCosmosRepository( { var options = s.GetRequiredService>(); + var connectionPolicy = new ConnectionPolicy { ConnectionMode = options.Value.CosmosDBConnectionMode }; + if (connectionPolicy.ConnectionMode == ConnectionMode.Gateway) + { + connectionPolicy.MaxConnectionLimit = options.Value.CosmosDBMaxConnectionsLimit; + } + else + { + connectionPolicy.ConnectionProtocol = options.Value.CosmosDBConnectionProtocol; + } + return new DocumentClient( new Uri(options.Value.CosmosDBEndpoint), options.Value.CosmosDBKey, - connectionPolicy: new ConnectionPolicy { ConnectionMode = options.Value.CosmosDBConnectionMode }); + connectionPolicy: connectionPolicy); }); services.ConfigureOptions>(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs index 41fb9784..6ef7b4dc 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -56,6 +56,11 @@ public void Configure(CosmosDBRepositoryOptions options) { options.MaxParallelism = int.Parse(_config["CosmosDBMaxParallelism"], CultureInfo.InvariantCulture); } + + if (string.IsNullOrEmpty(_config["CosmosDBMaxBufferedItemCount"]) == false) + { + options.MaxBufferedItemCount = int.Parse(_config["CosmosDBMaxBufferedItemCount"], CultureInfo.InvariantCulture); + } } } } \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs index becdf8e0..a1ec0feb 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs @@ -14,5 +14,9 @@ public class CosmosDBConnectionOptions public string CosmosDBKey { get; set; } public ConnectionMode CosmosDBConnectionMode { get; set; } + + public Protocol CosmosDBConnectionProtocol { get; set; } + + public int CosmosDBMaxConnectionsLimit { get; set; } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index 065d7492..3e39e0be 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -51,16 +51,25 @@ public async Task> GetItemsAsync( "partitionKey: {PartitionKey}", partitionKey); - using (var queryMetricsTracker = this._metricsTracker.GetQueryMetricsTracker(this._collectionIdentifier, partitionKey)) + using (var queryMetricsTracker = + this._metricsTracker.GetQueryMetricsTracker( + this._collectionIdentifier, + partitionKey, + this._options.MaxParallelism, + this._client.ConnectionPolicy.MaxConnectionLimit, + this._client.ConnectionPolicy.ConnectionMode, + this._client.ConnectionPolicy.ConnectionProtocol, + this._options.MaxBufferedItemCount)) { IDocumentQuery query = _client.CreateDocumentQuery( this._options.CollectionUri, new FeedOptions { - MaxItemCount = this._options.MaxParallelism, + MaxDegreeOfParallelism = this._options.MaxParallelism, PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : null, - EnableCrossPartitionQuery = partitionKey == null + EnableCrossPartitionQuery = partitionKey == null, + MaxBufferedItemCount = _client.ConnectionPolicy.ConnectionMode == ConnectionMode.Direct ? this._options.MaxBufferedItemCount : 0 }) .Where(predicate) .Where(d => d.DocumentType == typeof(T).Name) diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs index 45e054b5..25cfba2c 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs @@ -21,7 +21,17 @@ public class CosmosDBRepositoryMetricsTracker private const string DocumentDimensionName = "Document"; private const string PartitionKeyDimensionName = "PartitionKey"; private const string DocumentCountDimensionName = "DocumentCount"; - private const string RequestChargeKey = "RequestCharge"; + private const string CosmosDbPropertyPrefix = "CosmosDb."; + private const string CollectionPropertyKey = CosmosDbPropertyPrefix + CollectionDimensionName; + private const string DocumentPropertyKey = CosmosDbPropertyPrefix + DocumentDimensionName; + private const string PartitionKeyPropertyKey = CosmosDbPropertyPrefix + PartitionKeyDimensionName; + private const string DocumentCountPropertyKey = CosmosDbPropertyPrefix + DocumentCountDimensionName; + private const string RequestChargePropertyKey = CosmosDbPropertyPrefix + "RequestCharge"; + private const string MaxParallelismPropertyKey = CosmosDbPropertyPrefix + "MaxParallelism"; + private const string MaxConnectionsPropertyKey = CosmosDbPropertyPrefix + "MaxConnections"; + private const string ConnectionModePropertyKey = CosmosDbPropertyPrefix + "ConnectionMode"; + private const string ConnectionProtocolPropertyKey = CosmosDbPropertyPrefix + "ConnectionProtocol"; + private const string MaxBufferedItemCountPropertyKey = CosmosDbPropertyPrefix + "MaxBufferedItemCount"; private readonly TelemetryClient _telemetryClient; private readonly Metric _metric; @@ -38,13 +48,25 @@ public CosmosDBRepositoryMetricsTracker(TelemetryClient telemetryClient) DocumentCountDimensionName); } - public ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker(string collection, string partitionKey) - { - return new QueryMetricsTracker(this._telemetryClient, this._metric, collection, partitionKey); - } - - public void TrackResponseMetrics(FeedResponse response, string collection, string partitionKey) + public ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker( + string collection, + string partitionKey, + int maxParallelism, + int maxConnections, + ConnectionMode connectionMode, + Protocol connectionProtocol, + int maxBufferedItemCount) { + return new QueryMetricsTracker( + this._telemetryClient, + this._metric, + collection, + partitionKey, + maxParallelism, + maxConnections, + connectionMode, + connectionProtocol, + maxBufferedItemCount); } private sealed class QueryMetricsTracker : ICosmosDBRepositoryQueryMetricsTracker @@ -53,32 +75,43 @@ private sealed class QueryMetricsTracker : ICosmosDBRepositoryQueryMetricsTracke private readonly Metric _metric; private readonly string _collection; private readonly string _partitionKey; - + private readonly string _maxParallelism; + private readonly string _maxConnections; + private readonly string _connectionMode; + private readonly string _connectionProtocol; + private readonly string _maxBufferedItemCount; private double _totalCharge; private long _totalDocumentCount; - public QueryMetricsTracker(TelemetryClient telemetryClient, Metric metric, string collection, string partitionKey) + public QueryMetricsTracker( + TelemetryClient telemetryClient, + Metric metric, + string collection, + string partitionKey, + int maxParallelism, + int maxConnections, + ConnectionMode connectionMode, + Protocol connectionProtocol, + int maxBufferedItemCount) { this._telemetryClient = telemetryClient; this._metric = metric; this._collection = collection; this._partitionKey = partitionKey; + this._maxParallelism = maxParallelism.ToString(CultureInfo.InvariantCulture); + this._maxConnections = maxConnections.ToString(CultureInfo.InvariantCulture); + this._connectionMode = connectionMode.ToString(); + this._connectionProtocol = connectionProtocol.ToString(); + this._maxBufferedItemCount = maxBufferedItemCount.ToString(CultureInfo.InvariantCulture); } public void Dispose() { - this._telemetryClient.TrackTrace( + TraceMetrics( "Completed document query", - SeverityLevel.Information, - new Dictionary - { - [RequestChargeKey] = this._totalCharge.ToString(CultureInfo.InvariantCulture), - [CollectionDimensionName] = this._collection, - [DocumentDimensionName] = typeof(T).Name, - [PartitionKeyDimensionName] = this._partitionKey, - [DocumentCountDimensionName] = this._totalDocumentCount.ToString(CultureInfo.InvariantCulture) - }); + this._totalCharge.ToString(CultureInfo.InvariantCulture), + this._totalDocumentCount.ToString(CultureInfo.InvariantCulture)); } public void TrackResponseMetrics(FeedResponse response) @@ -92,20 +125,33 @@ public void TrackResponseMetrics(FeedResponse response) this._partitionKey ?? "", responseCount); - this._telemetryClient.TrackTrace( + TraceMetrics( "Partial document query", + response.RequestCharge.ToString(CultureInfo.InvariantCulture), + responseCount); + + this._totalCharge += response.RequestCharge; + this._totalDocumentCount += response.Count; + } + + private void TraceMetrics(string message, string requestCharge, string documentCount) + { + this._telemetryClient.TrackTrace( + message, SeverityLevel.Information, new Dictionary { - [RequestChargeKey] = response.RequestCharge.ToString(CultureInfo.InvariantCulture), - [CollectionDimensionName] = this._collection, - [DocumentDimensionName] = typeof(T).Name, - [PartitionKeyDimensionName] = this._partitionKey, - [DocumentCountDimensionName] = responseCount + [RequestChargePropertyKey] = requestCharge, + [CollectionPropertyKey] = this._collection, + [DocumentPropertyKey] = typeof(T).Name, + [PartitionKeyPropertyKey] = this._partitionKey, + [DocumentCountPropertyKey] = documentCount, + [MaxParallelismPropertyKey] = this._maxParallelism, + [MaxConnectionsPropertyKey] = this._maxConnections, + [ConnectionModePropertyKey] = this._connectionMode, + [ConnectionProtocolPropertyKey] = this._connectionProtocol, + [MaxBufferedItemCountPropertyKey] = this._maxBufferedItemCount, }); - - this._totalCharge += response.RequestCharge; - this._totalDocumentCount += response.Count; } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs index 42ab1da0..42880cee 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs @@ -8,7 +8,7 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { - public class CosmosDBRepositoryOptions + public class CosmosDBRepositoryOptions where T : BaseDocument { public string DatabaseId { get; set; } @@ -18,5 +18,7 @@ public class CosmosDBRepositoryOptions public Uri CollectionUri { get; set; } public int MaxParallelism { get; set; } = -1; + + public int MaxBufferedItemCount { get; set; } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs index 64f0db7d..8324b7cd 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using Microsoft.Azure.Documents.Client; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services @@ -10,6 +11,13 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services public interface ICosmosDBRepositoryMetricsTracker where T : BaseDocument { - ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker(string collection, string partitionKey); + ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker( + string collection, + string partitionKey, + int maxParallelism, + int maxConnections, + ConnectionMode connectionMode, + Protocol connectionProtocol, + int maxBufferedItemCount); } } From 91f2a0219e1ed5d8d80a119d17e7e5b9f795d133 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 18 Jul 2019 16:01:48 +0000 Subject: [PATCH 188/246] Merged PR 837: upgrade Cosmos SDK v3 upgrade Cosmos SDK v3 solved: #9267 Related work items: #9267 --- .../CosmosDBRespositoryTests.cs | 68 +++++---- ...neDeliveriesUtilizationIntegrationTests.cs | 86 +++++------ .../InvoicingRepositoryTests.cs | 14 +- .../Utils/DocumentClientMock.cs | 133 ------------------ .../Utils/IFakeDocumentQuery.cs | 16 --- .../CosmosDBExtensions.cs | 21 ++- ...DroneDelivery.DroneSchedulerService.csproj | 2 +- .../ConfigureCosmosDBRepositoryOptions.cs | 31 +--- .../Services/CosmosDBConnectionOptions.cs | 4 +- .../Services/CosmosDBRepository.cs | 42 +++--- .../CosmosDBRepositoryMetricsTracker.cs | 11 +- .../Services/CosmosDBRepositoryOptions.cs | 4 +- .../Services/ICosmosDBRepository.cs | 6 +- .../ICosmosDBRepositoryMetricsTracker.cs | 3 +- .../ICosmosDBRepositoryQueryMetricsTracker.cs | 2 +- .../Services/IInvoicingRepository.cs | 9 +- .../Services/InvoicingRepository.cs | 29 ++-- 17 files changed, 138 insertions(+), 343 deletions(-) delete mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs delete mode 100644 src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs index 24074856..2ad0ed20 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CosmosDBRespositoryTests.cs @@ -3,12 +3,11 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -16,7 +15,6 @@ using Xunit; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; -using Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests { @@ -24,11 +22,12 @@ public class CosmosDBRespositoryTests { private readonly ILogger> _loggerDebug; - private readonly IDocumentClient _clientMockObject; private readonly IOptions> _optionsMockObject; + private readonly Container _containerMockObject; + private readonly CosmosClient _clientMockObject; private readonly ICosmosDBRepositoryMetricsTracker _metricsTrackerMockObject; - private readonly IQueryable _fakeResults; + private readonly List _fakeResults; public CosmosDBRespositoryTests() { @@ -62,17 +61,27 @@ public CosmosDBRespositoryTests() AssignedHours=2d, DocumentType = typeof(InternalDroneUtilization).Name } - }.AsQueryable(); + }; - _clientMockObject = DocumentClientMock. - CreateDocumentClientMockObject(_fakeResults); + _clientMockObject = Mock.Of(c => c.ClientOptions == new CosmosClientOptions()); + + var responseMock = new Mock>(); + responseMock.Setup(r => r.Count).Returns(() => _fakeResults.Count); + responseMock.Setup(r => r.GetEnumerator()).Returns(() => _fakeResults.GetEnumerator()); + + var mockFeedIterator = new Mock>(); + mockFeedIterator.Setup(i => i.HasMoreResults).Returns(new Queue(new[] { true, false }).Dequeue); + mockFeedIterator.Setup(i => i.ReadNextAsync(It.IsAny())).ReturnsAsync(responseMock.Object); + + _containerMockObject = + Mock.Of(c => + c.GetItemQueryIterator(It.IsAny(), It.IsAny(), It.IsAny()) + == mockFeedIterator.Object); var fakeOptionsValue = new CosmosDBRepositoryOptions { - CollectionUri = UriFactory.CreateDocumentCollectionUri( - "fakeDb", - "fakeCol") + Container = _containerMockObject }; var optionsMock = new Mock< @@ -93,7 +102,6 @@ public CosmosDBRespositoryTests() It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny()) == Mock.Of>()); } @@ -112,7 +120,7 @@ public async Task WhenGetItemsAsyncWithPartitionId_ThenClientMakesAQueryWithPart // Act var res = await repo.GetItemsAsync( - p => true, + new QueryDefinition("SELECT *"), ownerId); // Assert @@ -125,14 +133,15 @@ public async Task WhenGetItemsAsyncWithPartitionId_ThenClientMakesAQueryWithPart Assert.Equal(ownerId, r.PartitionKey); Assert.Equal(typeof(InternalDroneUtilization).Name, r.DocumentType); }); - Mock.Get(_clientMockObject) - .Verify(dc => - dc.CreateDocumentQuery( - It.IsAny(), - It.Is(fo => - fo.PartitionKey != null - && fo.PartitionKey.ToString().Contains(ownerId) - && fo.EnableCrossPartitionQuery == false))); + + Mock.Get(_containerMockObject) + .Verify(c => + c.GetItemQueryIterator( + It.IsAny(), + null, + It.Is(ro => + ro.PartitionKey != null + && ro.PartitionKey.ToString().Contains(ownerId)))); } [Fact] @@ -149,7 +158,7 @@ public async Task WhenGetItemsAsyncWithoutPartitionId_ThenClientMakesAQueryWitho // Act var res = await repo.GetItemsAsync( - p => true, + new QueryDefinition("SELECT *"), null); // Assert @@ -162,13 +171,12 @@ public async Task WhenGetItemsAsyncWithoutPartitionId_ThenClientMakesAQueryWitho Assert.Equal(ownerId, r.PartitionKey); Assert.Equal(typeof(InternalDroneUtilization).Name, r.DocumentType); }); - Mock.Get(_clientMockObject) - .Verify(dc => - dc.CreateDocumentQuery( - It.IsAny(), - It.Is(fo => - fo.PartitionKey == null - && fo.EnableCrossPartitionQuery == true))); + Mock.Get(_containerMockObject) + .Verify(c => + c.GetItemQueryIterator( + It.IsAny(), + null, + It.Is(ro => ro.PartitionKey == null))); } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs index 80928e49..9a8c5647 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/DroneDeliveriesUtilizationIntegrationTests.cs @@ -5,18 +5,17 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.TestHost; -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using Moq; using Xunit; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; -using Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests { @@ -30,43 +29,44 @@ public class DroneDeliveriesUtilizationIntegrationTests : private readonly HttpClient _client; private readonly CustomWebApplicationFactory _factory; - private readonly IConfigureOptions> _configOptMockObject; + private readonly List _fakeResults; public DroneDeliveriesUtilizationIntegrationTests( CustomWebApplicationFactory factory) { - Uri fakeCollectionUri = UriFactory.CreateDocumentCollectionUri( - "fakeDb", - "fakeCol"); + _fakeResults = new List(); + + var responseMock = new Mock>(); + responseMock.Setup(r => r.Count).Returns(() => _fakeResults.Count); + responseMock.Setup(r => r.GetEnumerator()).Returns(() => _fakeResults.GetEnumerator()); + + var mockFeedIterator = new Mock>(); + mockFeedIterator.Setup(i => i.HasMoreResults).Returns(new Queue(new[] { true, false }).Dequeue); + mockFeedIterator.Setup(i => i.ReadNextAsync(It.IsAny())).ReturnsAsync(responseMock.Object); + + var _containerMockObject = Mock.Of(); + _containerMockObject = + Mock.Of(c => + c.GetItemQueryIterator(It.IsAny(), It.IsAny(), It.IsAny()) + == mockFeedIterator.Object); var configOptMock = new Mock>>(); configOptMock .Setup(c => c.Configure( It.IsAny>())) .Callback>( - o => o.CollectionUri = fakeCollectionUri); + o => o.Container = _containerMockObject); - _configOptMockObject = configOptMock.Object; + var configOptMockObject = configOptMock.Object; + + var clientMockObject = Mock.Of(c => c.ClientOptions == new CosmosClientOptions()); _factory = factory; _client = factory.WithWebHostBuilder(b => b.ConfigureTestServices(s => { - s.ConfigureOptions(_configOptMockObject); - s.AddSingleton(DocumentClientMock - .CreateDocumentClientMockObject( - new List { - new InternalDroneUtilization { - Id = "d0001", - PartitionKey = "o00042", - OwnerId = "o00042", - Month = 6, - Year = 2019, - TraveledMiles =10d, - AssignedHours=1d, - DocumentType = typeof(InternalDroneUtilization).Name - } - }.AsQueryable())); + s.ConfigureOptions(configOptMockObject); + s.AddSingleton(clientMockObject); s.AddSingleton( Mock.Of>( t => t.GetQueryMetricsTracker( @@ -75,7 +75,6 @@ public DroneDeliveriesUtilizationIntegrationTests( It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny()) == Mock.Of>())); })) @@ -103,6 +102,19 @@ public async Task GetInvoicingMonthlyBasisWithValidCriteria_ThenResponseWithSucc string ownerId = "o00042"; int year = 2019, month = 6; + _fakeResults.Add( + new InternalDroneUtilization + { + Id = "d0001", + PartitionKey = "o00042", + OwnerId = "o00042", + Month = 6, + Year = 2019, + TraveledMiles = 10d, + AssignedHours = 1d, + DocumentType = typeof(InternalDroneUtilization).Name + }); + var droneUtilizationUriWithParams = new UriBuilder(RequestUri); droneUtilizationUriWithParams.Query = $"ownerId={ownerId}&year={year}&month={month}"; @@ -121,28 +133,6 @@ public async Task GetInvoicingMonthlyBasisWithValidCriteria_ThenResponseWithSucc public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusCode() { // Arrange - HttpClient client = _factory.WithWebHostBuilder(b => - b.ConfigureTestServices(s => - { - s.ConfigureOptions(_configOptMockObject); - s.AddSingleton(DocumentClientMock - .CreateDocumentClientMockObject( - new List() - .AsQueryable())); - s.AddSingleton( - Mock.Of>( - t => t.GetQueryMetricsTracker( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny()) - == Mock.Of>())); - })) - .CreateClient(); - string ownerId = "o00042"; var minValidDateTime = DateTime.MinValue; int year = minValidDateTime.Year, month = minValidDateTime.Month; @@ -152,7 +142,7 @@ public async Task GetInvoicingForNotExistentData_ThenResponseWithNotFoundStatusC $"ownerId={ownerId}&year={year}&month={month}"; // Act - var response = await client.GetAsync(droneUtilizationUriWithParams.ToString()); + var response = await _client.GetAsync(droneUtilizationUriWithParams.ToString()); // Assert Assert.NotNull(response); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs index dc989876..bba68339 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/InvoicingRepositoryTests.cs @@ -6,13 +6,13 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; using Microsoft.FeatureManagement; -using Fabrikam.DroneDelivery.DroneSchedulerService.Models; -using Fabrikam.DroneDelivery.DroneSchedulerService.Services; using Moq; using Xunit; +using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Fabrikam.DroneDelivery.DroneSchedulerService.Services; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests { @@ -43,7 +43,7 @@ public async Task WhenGetAggregatedInvoincingData_ThenInvokesDroneUtilizationRep cosmosDbMock .Verify(p => p.GetItemsAsync( - It.IsAny>>(), + It.IsAny(), ownerId), Times.Once); } @@ -73,7 +73,7 @@ public async Task WhenGetAggregatedInvoincingDataAndFeatureForPartitionKeyIsDisa cosmosDbMock .Verify(p => p.GetItemsAsync( - It.IsAny>>(), + It.IsAny(), null), Times.Once); } @@ -99,7 +99,7 @@ public async Task WhenGetAggregatedInvoincingDataForAValidPeriod_ThenRepoReturns var cosmosDbMock = new Mock>(); cosmosDbMock.Setup(r => r.GetItemsAsync( - It.IsAny>>(), + It.IsAny(), ownerId)) .ReturnsAsync(invoicingData.AsEnumerable()); @@ -125,7 +125,7 @@ await repo.GetAggreatedInvoincingDataAsync( cosmosDbMock .Verify(p => p.GetItemsAsync( - It.IsAny>>(), + It.IsAny(), ownerId), Times.Once); } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs deleted file mode 100644 index be17509a..00000000 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/DocumentClientMock.cs +++ /dev/null @@ -1,133 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Threading; -using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Client; -using Moq; - -namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils -{ - public static class DocumentClientMock - { - public static IDocumentClient CreateDocumentClientMockObject( - IQueryable fakeResults) - { - // doc query - var fakeResponse = new FeedResponse(fakeResults); - - var mockDocumentQuery = new Mock>(); - - var docProvider = new Mock(); - docProvider - .Setup(p => p.CreateQuery( - It.IsAny())) - .Returns(mockDocumentQuery.Object); - - mockDocumentQuery - .SetupSequence(q => q.HasMoreResults) - .Returns(true) - .Returns(false); - - mockDocumentQuery - .Setup(q => - q.ExecuteNextAsync( - It.IsAny())) - .ReturnsAsync(fakeResponse); - - mockDocumentQuery - .Setup(q => q.Provider) - .Returns(docProvider.Object); - mockDocumentQuery - .Setup(q => q.ElementType) - .Returns(fakeResults.ElementType); - mockDocumentQuery - .Setup(q => q.Expression) - .Returns(fakeResults.Expression); - mockDocumentQuery - .Setup(q => q.GetEnumerator()) - .Returns(fakeResults.GetEnumerator()); - - // db query - var mockDatabaseQuery = new Mock>(); - var dbProvider = new Mock(); - dbProvider - .Setup(p => p.CreateQuery( - It.IsAny())) - .Returns(mockDatabaseQuery.Object); - - var fakeDbResutls = new List - { - Mock.Of() - }.AsQueryable(); - - mockDatabaseQuery - .Setup(q => q.Provider) - .Returns(dbProvider.Object); - mockDatabaseQuery - .Setup(q => q.ElementType) - .Returns(fakeDbResutls.ElementType); - mockDatabaseQuery - .Setup(q => q.Expression) - .Returns(fakeDbResutls.Expression); - mockDatabaseQuery - .Setup(q => q.GetEnumerator()) - .Returns(fakeDbResutls.GetEnumerator()); - - // collection query - var mockCollectionQuery = new Mock>(); - var colProvider = new Mock(); - colProvider - .Setup(p => p.CreateQuery( - It.IsAny())) - .Returns(mockCollectionQuery.Object); - - var fakeColResutls = new List - { - Mock.Of() - }.AsQueryable(); - - mockCollectionQuery - .Setup(q => q.Provider) - .Returns(colProvider.Object); - mockCollectionQuery - .Setup(q => q.ElementType) - .Returns(fakeColResutls.ElementType); - mockCollectionQuery - .Setup(q => q.Expression) - .Returns(fakeColResutls.Expression); - mockCollectionQuery - .Setup(q => q.GetEnumerator()) - .Returns(fakeColResutls.GetEnumerator()); - - // DocumentClient mock - var clientMock = new Mock(); - - clientMock - .Setup(q => q.CreateDocumentQuery( - It.IsAny(), - It.IsAny())) - .Returns(mockDocumentQuery.Object); - - clientMock - .Setup(q => q.CreateDatabaseQuery(null)) - .Returns(mockDatabaseQuery.Object); - - clientMock - .Setup(q => q.CreateDocumentCollectionQuery(It.IsAny(), null)) - .Returns(mockCollectionQuery.Object); - - clientMock - .Setup(q => q.ConnectionPolicy) - .Returns(new ConnectionPolicy { ConnectionMode = ConnectionMode.Gateway }); - - return clientMock.Object; - } - } -} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs deleted file mode 100644 index f0962eca..00000000 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Utils/IFakeDocumentQuery.cs +++ /dev/null @@ -1,16 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -using System.Linq; -using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Linq; - -namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests.Utils -{ - public interface IFakeDocumentQuery : - IDocumentQuery, - IOrderedQueryable - { } -} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs index c801db0c..d470eecc 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/CosmosDBExtensions.cs @@ -5,8 +5,7 @@ using System; using System.Linq; -using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -22,24 +21,20 @@ public static IServiceCollection AddCosmosRepository( IConfiguration config) where T : BaseDocument { - services.AddSingleton(s => + services.AddSingleton(s => { var options = s.GetRequiredService>(); - var connectionPolicy = new ConnectionPolicy { ConnectionMode = options.Value.CosmosDBConnectionMode }; - if (connectionPolicy.ConnectionMode == ConnectionMode.Gateway) + var clientOptions = new CosmosClientOptions { ConnectionMode = options.Value.CosmosDBConnectionMode }; + if (clientOptions.ConnectionMode == ConnectionMode.Gateway) { - connectionPolicy.MaxConnectionLimit = options.Value.CosmosDBMaxConnectionsLimit; - } - else - { - connectionPolicy.ConnectionProtocol = options.Value.CosmosDBConnectionProtocol; + clientOptions.GatewayModeMaxConnectionLimit = options.Value.CosmosDBMaxConnectionsLimit; } - return new DocumentClient( - new Uri(options.Value.CosmosDBEndpoint), + return new CosmosClient( + options.Value.CosmosDBEndpoint, options.Value.CosmosDBKey, - connectionPolicy: connectionPolicy); + clientOptions: clientOptions); }); services.ConfigureOptions>(); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj index 6fb08cb6..03919a13 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs index 6ef7b4dc..c8f3eed9 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ConfigureCosmosDBRepositoryOptions.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; using System.Globalization; -using System.Linq; -using Microsoft.Azure.Documents; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -17,11 +15,11 @@ public class ConfigureCosmosDBRepositoryOptions : IConfigureOptions options) options.DatabaseId = _config["COSMOSDB_DATABASEID"]; options.CollectionId = _config["COSMOSDB_COLLECTIONID"]; - Database db = _client - .CreateDatabaseQuery() - .Where(d => d.Id == options.DatabaseId) - .AsEnumerable() - .FirstOrDefault(); - - if (db != null) - { - DocumentCollection col = _client - .CreateDocumentCollectionQuery(db.SelfLink) - .Where(d => d.Id == options.CollectionId) - .AsEnumerable() - .FirstOrDefault(); - - if (Uri.TryCreate(col?.SelfLink, UriKind.Relative, out Uri uri)) - { - options.CollectionUri = uri; - } - } - + options.Container = _client + .GetContainer(options.DatabaseId, options.CollectionId); + if (string.IsNullOrEmpty(_config["CosmosDBMaxParallelism"]) == false) { options.MaxParallelism = int.Parse(_config["CosmosDBMaxParallelism"], CultureInfo.InvariantCulture); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs index a1ec0feb..9c93af56 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBConnectionOptions.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { @@ -15,8 +15,6 @@ public class CosmosDBConnectionOptions public ConnectionMode CosmosDBConnectionMode { get; set; } - public Protocol CosmosDBConnectionProtocol { get; set; } - public int CosmosDBMaxConnectionsLimit { get; set; } } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs index 3e39e0be..3458b2c0 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepository.cs @@ -3,14 +3,9 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; using System.Threading.Tasks; -using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Client; -using Microsoft.Azure.Documents.Linq; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -20,14 +15,14 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services public class CosmosRepository : ICosmosRepository where T : BaseDocument { - private readonly IDocumentClient _client; + private readonly CosmosClient _client; private readonly CosmosDBRepositoryOptions _options; private readonly ILogger> _logger; private readonly ICosmosDBRepositoryMetricsTracker _metricsTracker; private readonly string _collectionIdentifier; public CosmosRepository( - IDocumentClient client, + CosmosClient client, IOptions> options, ILogger> logger, ICosmosDBRepositoryMetricsTracker metricsTracker) @@ -40,7 +35,7 @@ public CosmosRepository( } public async Task> GetItemsAsync( - Expression> predicate, + QueryDefinition query, string partitionKey) { var results = new List(); @@ -56,29 +51,24 @@ public async Task> GetItemsAsync( this._collectionIdentifier, partitionKey, this._options.MaxParallelism, - this._client.ConnectionPolicy.MaxConnectionLimit, - this._client.ConnectionPolicy.ConnectionMode, - this._client.ConnectionPolicy.ConnectionProtocol, + this._client.ClientOptions.GatewayModeMaxConnectionLimit, + this._client.ClientOptions.ConnectionMode, this._options.MaxBufferedItemCount)) { - IDocumentQuery query = - _client.CreateDocumentQuery( - this._options.CollectionUri, - new FeedOptions + FeedIterator iterator = + this._options.Container.GetItemQueryIterator( + query, + requestOptions: new QueryRequestOptions { - MaxDegreeOfParallelism = this._options.MaxParallelism, - PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : null, - EnableCrossPartitionQuery = partitionKey == null, - MaxBufferedItemCount = _client.ConnectionPolicy.ConnectionMode == ConnectionMode.Direct ? this._options.MaxBufferedItemCount : 0 - }) - .Where(predicate) - .Where(d => d.DocumentType == typeof(T).Name) - .AsDocumentQuery(); + MaxConcurrency = this._options.MaxParallelism, + PartitionKey = partitionKey != null ? new PartitionKey(partitionKey) : new PartitionKey?(), + MaxBufferedItemCount = this._options.MaxBufferedItemCount + }); _logger.LogInformation("Start: reading results from query"); - while (query.HasMoreResults) + while (iterator.HasMoreResults) { - var feed = await query.ExecuteNextAsync(); + var feed = await iterator.ReadNextAsync(); queryMetricsTracker.TrackResponseMetrics(feed); results.AddRange(feed); } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs index 25cfba2c..3c6cace9 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryMetricsTracker.cs @@ -7,7 +7,7 @@ using System.Globalization; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services @@ -30,7 +30,6 @@ public class CosmosDBRepositoryMetricsTracker private const string MaxParallelismPropertyKey = CosmosDbPropertyPrefix + "MaxParallelism"; private const string MaxConnectionsPropertyKey = CosmosDbPropertyPrefix + "MaxConnections"; private const string ConnectionModePropertyKey = CosmosDbPropertyPrefix + "ConnectionMode"; - private const string ConnectionProtocolPropertyKey = CosmosDbPropertyPrefix + "ConnectionProtocol"; private const string MaxBufferedItemCountPropertyKey = CosmosDbPropertyPrefix + "MaxBufferedItemCount"; private readonly TelemetryClient _telemetryClient; @@ -54,7 +53,6 @@ public ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker( int maxParallelism, int maxConnections, ConnectionMode connectionMode, - Protocol connectionProtocol, int maxBufferedItemCount) { return new QueryMetricsTracker( @@ -65,7 +63,6 @@ public ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker( maxParallelism, maxConnections, connectionMode, - connectionProtocol, maxBufferedItemCount); } @@ -78,7 +75,6 @@ private sealed class QueryMetricsTracker : ICosmosDBRepositoryQueryMetricsTracke private readonly string _maxParallelism; private readonly string _maxConnections; private readonly string _connectionMode; - private readonly string _connectionProtocol; private readonly string _maxBufferedItemCount; private double _totalCharge; private long _totalDocumentCount; @@ -92,7 +88,6 @@ public QueryMetricsTracker( int maxParallelism, int maxConnections, ConnectionMode connectionMode, - Protocol connectionProtocol, int maxBufferedItemCount) { this._telemetryClient = telemetryClient; @@ -102,7 +97,6 @@ public QueryMetricsTracker( this._maxParallelism = maxParallelism.ToString(CultureInfo.InvariantCulture); this._maxConnections = maxConnections.ToString(CultureInfo.InvariantCulture); this._connectionMode = connectionMode.ToString(); - this._connectionProtocol = connectionProtocol.ToString(); this._maxBufferedItemCount = maxBufferedItemCount.ToString(CultureInfo.InvariantCulture); } @@ -149,10 +143,9 @@ private void TraceMetrics(string message, string requestCharge, string documentC [MaxParallelismPropertyKey] = this._maxParallelism, [MaxConnectionsPropertyKey] = this._maxConnections, [ConnectionModePropertyKey] = this._connectionMode, - [ConnectionProtocolPropertyKey] = this._connectionProtocol, [MaxBufferedItemCountPropertyKey] = this._maxBufferedItemCount, }); } } } -} +} \ No newline at end of file diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs index 42880cee..f2579403 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/CosmosDBRepositoryOptions.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; +using Microsoft.Azure.Cosmos; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services @@ -15,7 +15,7 @@ public class CosmosDBRepositoryOptions public string CollectionId { get; set; } - public Uri CollectionUri { get; set; } + public Container Container { get; set; } public int MaxParallelism { get; set; } = -1; diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs index 5b93de51..52543924 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepository.cs @@ -3,18 +3,16 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System; using System.Collections.Generic; -using System.Linq.Expressions; using System.Threading.Tasks; -using Fabrikam.DroneDelivery.DroneSchedulerService.Models; +using Microsoft.Azure.Cosmos; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { public interface ICosmosRepository { Task> GetItemsAsync( - Expression> predicate, + QueryDefinition query, string partitionKey); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs index 8324b7cd..5e3aa617 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryMetricsTracker.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services @@ -17,7 +17,6 @@ ICosmosDBRepositoryQueryMetricsTracker GetQueryMetricsTracker( int maxParallelism, int maxConnections, ConnectionMode connectionMode, - Protocol connectionProtocol, int maxBufferedItemCount); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs index d302bee9..82a9ffc4 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/ICosmosDBRepositoryQueryMetricsTracker.cs @@ -4,7 +4,7 @@ // ------------------------------------------------------------ using System; -using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Cosmos; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs index f9bd261f..03180a25 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/IInvoicingRepository.cs @@ -4,20 +4,13 @@ // ------------------------------------------------------------ using System; -using System.Collections.Generic; -using System.Linq.Expressions; using System.Threading.Tasks; -using Fabrikam.DroneDelivery.DroneSchedulerService.Models; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Services { public interface IInvoicingRepository { - Task> GetItemsAsync( - Expression> predicate, - string partitionKey); - - Task> GetAggreatedInvoincingDataAsync( + Task> GetAggreatedInvoincingDataAsync( string ownerId, int year, int month); diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs index f7d5c368..f1c2efec 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Services/InvoicingRepository.cs @@ -6,8 +6,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; using Microsoft.FeatureManagement; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -27,25 +27,24 @@ public InvoicingRepository( this._featureManager = featureManager; } - public async Task> GetItemsAsync( - Expression> predicate, - string partitionKey) - { - return await _repository - .GetItemsAsync( - predicate, - partitionKey); - } - public async Task> GetAggreatedInvoincingDataAsync( string ownerId, int year, int month) { - var results = await - (this._featureManager.IsEnabled(UsePartitionKeyFeatureName) - ? this.GetItemsAsync(d => d.Year == year && d.Month == month, ownerId) - : this.GetItemsAsync(d => d.Year == year && d.Month == month && d.OwnerId == ownerId, null)); + (QueryDefinition query, string partitionKey) = + this._featureManager.IsEnabled(UsePartitionKeyFeatureName) + ? (new QueryDefinition("SELECT VALUE root FROM root WHERE root.year = @year AND root.month = @month") + .WithParameter("@year", year) + .WithParameter("@month", month), + ownerId) + : (new QueryDefinition("SELECT VALUE root FROM root WHERE root.year = @year AND root.month = @month AND root.ownerId = @ownerId") + .WithParameter("@year", year) + .WithParameter("@month", month) + .WithParameter("@ownerId", ownerId), + default(string)); + + var results = await _repository.GetItemsAsync(query, partitionKey); var result = results .Aggregate( From 1e9fe3f460f3ad7a89821b1665b09160c73bd797 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 2 Oct 2019 07:03:01 -0500 Subject: [PATCH 189/246] Update deployment.md (#137) --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 913fb0c1..d3656c3a 100644 --- a/deployment.md +++ b/deployment.md @@ -52,7 +52,7 @@ Infrastructure Prerequisites az login # Create service principal for AKS -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor") && \ +export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor" -o json) && \ export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \ export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId) From f19a9c31375e022540aa1f1ae6b3945e17ecc5ab Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 28 Oct 2019 16:30:49 +0000 Subject: [PATCH 190/246] Merged PR 866: enforce resource quota and resource requests/limits 1. bug fixes resource requests/limits yaml identation 2. enforce resource quotas per environment now the logical cluster isolation reserves resources per environment while at the same rejects deployments if pods do not define resource requests and limits Cluster Resource Quota sharing resource CPU requests/limits based on a cluster with 3 x Standard D2 v2 (2 vcpus, 7 GiB memory) | | Dev (1 core reserved capacity w/ up to 2 shared w/ QA) | QA (1 core reserved capacity w/ up to 2 shared w/ Dev) | Staging (2 cores reserved capacity) | Prod (2 cores reserved capacity) | |-----------|--------------------------------------------------------|--------------------------------------------------------|-------------------------------------|----------------------------------| | | Request/Limit | Request/Limit | Request/Limit | Request/Limit | | delivery | 91/130 | 106/159 | 106/212 | 106/212 | | drone | 115/175 | 136/204 | 136/272 | 136/272 | | ingestion | 129/194 | 151/226.5 | 151/302 | 151/302 | | package | 78/117 | 91/136.5 | 91/182 | 91/182 | | workflow | 442/664 | 515/772.5 | 515/1000 | 515/1000 | solved: #132996 3. increase retry attempts and delay for ingestion liveness and readiness probes related: #133002 4. bug fix default number of replicas for dronescheduler --- azuredeploy.json | 2 +- charts/delivery/envs/delivery-dev/values.yaml | 7 ++++ .../delivery/envs/delivery-prod/values.yaml | 9 ++++- charts/delivery/envs/delivery-qa/values.yaml | 7 ++++ .../envs/delivery-staging/values.yaml | 9 ++++- .../delivery/templates/delivery-deploy.yaml | 16 ++++---- charts/delivery/values.yaml | 9 +---- .../envs/dronescheduler-dev/values.yaml | 7 ++++ .../envs/dronescheduler-prod/values.yaml | 7 ++++ .../envs/dronescheduler-qa/values.yaml | 7 ++++ .../envs/dronescheduler-staging/values.yaml | 7 ++++ .../templates/dronescheduler-deploy.yaml | 16 ++++---- charts/dronescheduler/values.yaml | 11 +----- .../ingestion/envs/ingestion-dev/values.yaml | 7 ++++ .../ingestion/envs/ingestion-prod/values.yaml | 7 ++++ .../ingestion/envs/ingestion-qa/values.yaml | 7 ++++ .../envs/ingestion-staging/values.yaml | 7 ++++ .../ingestion/templates/ingestion-deploy.yaml | 26 +++++++------ charts/ingestion/values.yaml | 9 +---- charts/package/envs/package-dev/values.yaml | 7 ++++ charts/package/envs/package-prod/values.yaml | 7 ++++ charts/package/envs/package-qa/values.yaml | 7 ++++ .../package/envs/package-staging/values.yaml | 7 ++++ charts/package/templates/package-deploy.yaml | 14 +++---- charts/package/values.yaml | 9 +---- charts/workflow/envs/workflow-dev/values.yaml | 7 ++++ .../workflow/envs/workflow-prod/values.yaml | 7 ++++ charts/workflow/envs/workflow-qa/values.yaml | 7 ++++ .../envs/workflow-staging/values.yaml | 7 ++++ .../workflow/templates/workflow-deploy.yaml | 14 +++---- charts/workflow/values.yaml | 9 +---- deployment.md | 8 +++- deploymentCICD.md | 9 ++++- k8s/k8s-resource-quotas-dev.yaml | 17 ++++++++ k8s/k8s-resource-quotas-qa-stg-prod.yaml | 39 +++++++++++++++++++ 35 files changed, 265 insertions(+), 87 deletions(-) create mode 100644 k8s/k8s-resource-quotas-dev.yaml create mode 100644 k8s/k8s-resource-quotas-qa-stg-prod.yaml diff --git a/azuredeploy.json b/azuredeploy.json index 90e86fcf..9447c6e1 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -80,7 +80,7 @@ }, "agentCount": { "type": "int", - "defaultValue": 3, + "defaultValue": 1, "metadata": { "description": "The number of agents for the cluster. This value can be from 1 to 100 (note, for Kubernetes clusters you will also get 1 or 2 public agents in addition to these seleted masters)" }, diff --git a/charts/delivery/envs/delivery-dev/values.yaml b/charts/delivery/envs/delivery-dev/values.yaml index 7935b27b..2d62ea5d 100644 --- a/charts/delivery/envs/delivery-dev/values.yaml +++ b/charts/delivery/envs/delivery-dev/values.yaml @@ -9,3 +9,10 @@ exports: level: "Information" reason: "new dev deploy" current: true + resources: + requests: + cpu: 91m + memory: 350Mi + limits: + cpu: 130m + memory: 500Mi diff --git a/charts/delivery/envs/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml index 3a1ba13f..f2577b01 100644 --- a/charts/delivery/envs/delivery-prod/values.yaml +++ b/charts/delivery/envs/delivery-prod/values.yaml @@ -2,9 +2,16 @@ nameOverride: delivery exports: data: - replicaCount: 3 + replicaCount: 2 image: pullPolicy: IfNotPresent telemetry: level: "Error" reason: "new prod deploy" + resources: + requests: + cpu: 100m + memory: 350Mi + limits: + cpu: 200m + memory: 500Mi \ No newline at end of file diff --git a/charts/delivery/envs/delivery-qa/values.yaml b/charts/delivery/envs/delivery-qa/values.yaml index 6cb3d57b..66ce0c20 100644 --- a/charts/delivery/envs/delivery-qa/values.yaml +++ b/charts/delivery/envs/delivery-qa/values.yaml @@ -9,3 +9,10 @@ exports: level: "Information" reason: "new qa deploy" current: true + resources: + requests: + cpu: 100m + memory: 350Mi + limits: + cpu: 150m + memory: 500Mi diff --git a/charts/delivery/envs/delivery-staging/values.yaml b/charts/delivery/envs/delivery-staging/values.yaml index 74532ab6..5c69c857 100644 --- a/charts/delivery/envs/delivery-staging/values.yaml +++ b/charts/delivery/envs/delivery-staging/values.yaml @@ -2,10 +2,17 @@ nameOverride: delivery exports: data: - replicaCount: 3 + replicaCount: 2 image: pullPolicy: Always telemetry: level: "Information" reason: "new staging deploy" current: true + resources: + requests: + cpu: 100m + memory: 350Mi + limits: + cpu: 200m + memory: 500Mi \ No newline at end of file diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index deea1345..2aebb96c 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -46,6 +46,13 @@ spec: - name: fabrikam-delivery image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} env: - name: DOCDB_DATABASEID value: {{ .Values.cosmosdb.id }} @@ -59,11 +66,4 @@ spec: value: 169.254.169.254 ports: - name: service - containerPort: 8080 - resources: - requests: - cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} - memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} - limits: - cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} - memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} + containerPort: 8080 \ No newline at end of file diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index 2da6c55b..fa70cfda 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -1,4 +1,4 @@ -# Default values for dronedelivery. +# Default values for delivery. nameOverride: delivery replicaCount: 1 identity: @@ -18,13 +18,6 @@ keyvault: telemetry: level: "Error" reason: unknown -resources: - requests: - cpu: 350m - memory: 350Mi - limits: - cpu: 500m - memory: 500Mi tags: dev: false prod: false diff --git a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml index 2f8bff92..67e16d11 100644 --- a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml @@ -9,3 +9,10 @@ exports: level: "Information" reason: "new dev deploy" current: true + resources: + requests: + cpu: 115m + memory: 350Mi + limits: + cpu: 175m + memory: 500Mi \ No newline at end of file diff --git a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml index 81788552..09c64fb3 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml @@ -8,3 +8,10 @@ exports: telemetry: level: "Error" reason: "new prod deploy" + resources: + requests: + cpu: 130m + memory: 350Mi + limits: + cpu: 270m + memory: 500Mi \ No newline at end of file diff --git a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml index 80584f16..26785142 100644 --- a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml @@ -9,3 +9,10 @@ exports: level: "Information" reason: "new qa deploy" current: true + resources: + requests: + cpu: 130m + memory: 350Mi + limits: + cpu: 200m + memory: 500Mi \ No newline at end of file diff --git a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml index aa43bb50..db5fea9f 100644 --- a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml @@ -9,3 +9,10 @@ exports: level: "Information" reason: "new staging deploy" current: true + resources: + requests: + cpu: 130m + memory: 350Mi + limits: + cpu: 270m + memory: 500Mi \ No newline at end of file diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index efe25813..e691a614 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -46,6 +46,13 @@ spec: - name: fabrikam-dronescheduler image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} env: - name: KEY_VAULT_URI value: {{ .Values.keyvault.uri }} @@ -59,11 +66,4 @@ spec: value: 169.254.169.254 ports: - name: service - containerPort: 8080 - resources: - requests: - cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} - memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} - limits: - cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} - memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} + containerPort: 8080 \ No newline at end of file diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index d014d0c3..fd7ff2da 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -1,5 +1,5 @@ # Default values for dronescheduler. -replicaCount: 3 +replicaCount: 1 identity: clientid: resourceid: @@ -17,16 +17,9 @@ cosmosdb: reason: unknown telemetry: level: "Error" -resources: - requests: - cpu: 450m - memory: 350Mi - limits: - cpu: 600m - memory: 500Mi tags: dev: false prod: false qa: false staging: false -current: false +current: false \ No newline at end of file diff --git a/charts/ingestion/envs/ingestion-dev/values.yaml b/charts/ingestion/envs/ingestion-dev/values.yaml index 53ea4f1d..bf1eb186 100644 --- a/charts/ingestion/envs/ingestion-dev/values.yaml +++ b/charts/ingestion/envs/ingestion-dev/values.yaml @@ -9,3 +9,10 @@ exports: level: "info" reason: "new dev deploy" current: true + resources: + requests: + cpu: 129m + memory: 600Mi + limits: + cpu: 194m + memory: 800Mi \ No newline at end of file diff --git a/charts/ingestion/envs/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml index 51f25b01..2d4f77fc 100644 --- a/charts/ingestion/envs/ingestion-prod/values.yaml +++ b/charts/ingestion/envs/ingestion-prod/values.yaml @@ -11,3 +11,10 @@ exports: nginx-ingress: controller: replicaCount: 2 + resources: + requests: + cpu: 150m + memory: 600Mi + limits: + cpu: 300m + memory: 800Mi \ No newline at end of file diff --git a/charts/ingestion/envs/ingestion-qa/values.yaml b/charts/ingestion/envs/ingestion-qa/values.yaml index 105b36e4..0c5a0513 100644 --- a/charts/ingestion/envs/ingestion-qa/values.yaml +++ b/charts/ingestion/envs/ingestion-qa/values.yaml @@ -9,3 +9,10 @@ exports: level: "info" reason: "new qa deploy" current: true + resources: + requests: + cpu: 150m + memory: 600Mi + limits: + cpu: 220m + memory: 800Mi \ No newline at end of file diff --git a/charts/ingestion/envs/ingestion-staging/values.yaml b/charts/ingestion/envs/ingestion-staging/values.yaml index e43ac83d..e94d1cde 100644 --- a/charts/ingestion/envs/ingestion-staging/values.yaml +++ b/charts/ingestion/envs/ingestion-staging/values.yaml @@ -12,3 +12,10 @@ exports: controller: replicaCount: 2 current: true + resources: + requests: + cpu: 150m + memory: 600Mi + limits: + cpu: 300m + memory: 800Mi \ No newline at end of file diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index 8f85dc29..c211a466 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -45,14 +45,25 @@ spec: httpGet: path: /actuator/health port: 80 - initialDelaySeconds: 30 - periodSeconds: 20 + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 5 readinessProbe: httpGet: path: /api/probe port: 80 - initialDelaySeconds: 30 - periodSeconds: 20 + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 5 + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} env: - name: QUEUE_NAMESPACE valueFrom: @@ -83,10 +94,3 @@ spec: value: {{ default "error" .Values.telemetry.level | quote }} - name: CONTAINER_NAME value: *ingestion-container_name - resources: - requests: - cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} - memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} - limits: - cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} - memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index fb4468d4..eb47cfa1 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -9,13 +9,6 @@ image: reason: unknown telemetry: level: "error" -resources: - requests: - cpu: 500m - memory: 600Mi - limits: - cpu: 650m - memory: 800Mi nginx-ingress: fullnameOverride: nginx-ingress rbac: @@ -29,4 +22,4 @@ tags: prod: false qa: false staging: false -current: false +current: false \ No newline at end of file diff --git a/charts/package/envs/package-dev/values.yaml b/charts/package/envs/package-dev/values.yaml index 9cf0ef2c..d1d4fb1d 100644 --- a/charts/package/envs/package-dev/values.yaml +++ b/charts/package/envs/package-dev/values.yaml @@ -9,3 +9,10 @@ exports: level: "info" reason: "new dev deploy" current: true + resources: + requests: + cpu: 78m + memory: 100Mi + limits: + cpu: 117m + memory: 140Mi \ No newline at end of file diff --git a/charts/package/envs/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml index aecca82e..3b280e36 100644 --- a/charts/package/envs/package-prod/values.yaml +++ b/charts/package/envs/package-prod/values.yaml @@ -8,3 +8,10 @@ exports: log: level: "error" reason: "new prod deploy" + resources: + requests: + cpu: 90m + memory: 100Mi + limits: + cpu: 180m + memory: 140Mi \ No newline at end of file diff --git a/charts/package/envs/package-qa/values.yaml b/charts/package/envs/package-qa/values.yaml index 04ec88c0..49141ff4 100644 --- a/charts/package/envs/package-qa/values.yaml +++ b/charts/package/envs/package-qa/values.yaml @@ -9,3 +9,10 @@ exports: level: "info" reason: "new qa deploy" current: true + resources: + requests: + cpu: 90m + memory: 100Mi + limits: + cpu: 130m + memory: 140Mi \ No newline at end of file diff --git a/charts/package/envs/package-staging/values.yaml b/charts/package/envs/package-staging/values.yaml index 1cb8b548..115d2c78 100644 --- a/charts/package/envs/package-staging/values.yaml +++ b/charts/package/envs/package-staging/values.yaml @@ -9,3 +9,10 @@ exports: level: "info" reason: "new staging deploy" current: true + resources: + requests: + cpu: 90m + memory: 100Mi + limits: + cpu: 180m + memory: 140Mi \ No newline at end of file diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index dc162481..d45af23c 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -41,6 +41,13 @@ spec: - name: &package-container_name fabrikam-package image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} env: - name: CONNECTION_STRING valueFrom: @@ -61,10 +68,3 @@ spec: ports: - name: service containerPort: 80 - resources: - requests: - cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} - memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} - limits: - cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} - memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 239908f0..3d6d4518 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -12,16 +12,9 @@ log: level: error cosmosDb: collectionName: -resources: - requests: - cpu: 300m - memory: 100Mi - limits: - cpu: 410m - memory: 140Mi tags: dev: false prod: false qa: false staging: false -current: false +current: false \ No newline at end of file diff --git a/charts/workflow/envs/workflow-dev/values.yaml b/charts/workflow/envs/workflow-dev/values.yaml index 6f4a6a53..b18c5462 100644 --- a/charts/workflow/envs/workflow-dev/values.yaml +++ b/charts/workflow/envs/workflow-dev/values.yaml @@ -8,3 +8,10 @@ exports: telemetry: level: "Information" reason: "new dev deploy" + resources: + requests: + cpu: 442m + memory: 100Mi + limits: + cpu: 664m + memory: 140Mi \ No newline at end of file diff --git a/charts/workflow/envs/workflow-prod/values.yaml b/charts/workflow/envs/workflow-prod/values.yaml index 22edfc42..b0754a46 100644 --- a/charts/workflow/envs/workflow-prod/values.yaml +++ b/charts/workflow/envs/workflow-prod/values.yaml @@ -8,3 +8,10 @@ exports: telemetry: level: "Error" reason: "new prod deploy" + resources: + requests: + cpu: 515m + memory: 100Mi + limits: + cpu: 1000m + memory: 140Mi \ No newline at end of file diff --git a/charts/workflow/envs/workflow-qa/values.yaml b/charts/workflow/envs/workflow-qa/values.yaml index 78f674e6..db35b4a2 100644 --- a/charts/workflow/envs/workflow-qa/values.yaml +++ b/charts/workflow/envs/workflow-qa/values.yaml @@ -8,3 +8,10 @@ exports: telemetry: level: "Information" reason: "new qa deploy" + resources: + requests: + cpu: 515m + memory: 100Mi + limits: + cpu: 770m + memory: 140Mi \ No newline at end of file diff --git a/charts/workflow/envs/workflow-staging/values.yaml b/charts/workflow/envs/workflow-staging/values.yaml index 92a59fca..3c2f2561 100644 --- a/charts/workflow/envs/workflow-staging/values.yaml +++ b/charts/workflow/envs/workflow-staging/values.yaml @@ -8,3 +8,10 @@ exports: telemetry: level: "Information" reason: "new staging deploy" + resources: + requests: + cpu: 515m + memory: 100Mi + limits: + cpu: 1000m + memory: 140Mi \ No newline at end of file diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 2a342aaa..97d6fa3c 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -46,6 +46,13 @@ spec: - name: fabrikam-workflow image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} + memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} + limits: + cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} + memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} volumeMounts: - name: workflow mountPath: /kvmnt @@ -90,10 +97,3 @@ spec: resourcegroup: {{ .Values.keyvault.resourcegroup }} subscriptionid: {{ .Values.keyvault.subscriptionid }} tenantid: {{ .Values.keyvault.tenantid }} - resources: - requests: - cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} - memory: {{ required "A valid .Values.resources.requests.memory entry required!" .Values.resources.requests.memory }} - limits: - cpu: {{ required "A valid .Values.resources.limits.cpu entry required!" .Values.resources.limits.cpu }} - memory: {{ required "A valid .Values.resources.limits.memory entry required!" .Values.resources.limits.memory }} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index f21754a5..8e04f00c 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -30,15 +30,8 @@ keyvault: tenantid: telemetry: level: "Error" -resources: - requests: - cpu: 1700m - memory: 100Mi - limits: - cpu: 2300m - memory: 140Mi tags: dev: false prod: false qa: false - staging: false + staging: false \ No newline at end of file diff --git a/deployment.md b/deployment.md index 4f2b02bc..d64ac575 100644 --- a/deployment.md +++ b/deployment.md @@ -136,7 +136,7 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces -kubectl create namespace backend +kubectl create namespace backend-dev ``` Setup Helm in the container @@ -194,6 +194,12 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` +## Setup cluster resource quota + +```bash +kubectl apply -f $K8S/k8s-resource-quotas-dev.yaml +``` + ## Deploy the Delivery service Extract resource details from deployment diff --git a/deploymentCICD.md b/deploymentCICD.md index 9d0570ec..1a644650 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -51,7 +51,8 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa workflowPrincipalId=$WORKFLOW_ID_PRINCIPAL_ID \ acrResourceGroupName=${RESOURCE_GROUP_ACR} \ acrResourceGroupLocation=$LOCATION \ - environmentName=${env} + environmentName=${env} \ + agentCount=3 export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) @@ -111,6 +112,12 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` +## Setup cluster resource quota + +```bash +kubectl apply -f $K8S/k8s-resource-quotas-dev.yaml -f $K8S/k8s-resource-quotas-qa-stg-prod.yaml +``` + ## Setup Azure DevOps ``` diff --git a/k8s/k8s-resource-quotas-dev.yaml b/k8s/k8s-resource-quotas-dev.yaml new file mode 100644 index 00000000..b919c811 --- /dev/null +++ b/k8s/k8s-resource-quotas-dev.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +apiVersion: v1 +kind: ResourceQuota +metadata: + name: dev + namespace: backend-dev +spec: + hard: + requests.cpu: "1" + requests.memory: 2Gi + limits.cpu: "2" + limits.memory: 5Gi + pods: "5" diff --git a/k8s/k8s-resource-quotas-qa-stg-prod.yaml b/k8s/k8s-resource-quotas-qa-stg-prod.yaml new file mode 100644 index 00000000..797ae7d5 --- /dev/null +++ b/k8s/k8s-resource-quotas-qa-stg-prod.yaml @@ -0,0 +1,39 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + + apiVersion: v1 + kind: ResourceQuota + metadata: + name: qa + namespace: backend-qa + spec: + hard: + requests.cpu: "1" + requests.memory: 2Gi + limits.cpu: "2" + limits.memory: 5Gi + pods: "5" + --- + apiVersion: v1 + kind: ResourceQuota + metadata: + name: staging + namespace: backend-staging + spec: + hard: + cpu: "2" + memory: 8Gi + pods: "10" + --- + apiVersion: v1 + kind: ResourceQuota + metadata: + name: prod + namespace: backend + spec: + hard: + cpu: "2" + memory: 8Gi + pods: "10" From 33c24312a0f3cc9d40c461a0d7c11ff392f7c537 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 4 Nov 2019 13:38:46 +0000 Subject: [PATCH 191/246] Merged PR 872: add delivery readiness/liveness probe - add /healthz endpoint for readiness and liveness - add readiness probe cgf - add liveness probe cfg solved: #133004 --- .../delivery/templates/delivery-deploy.yaml | 34 ++++++++++++++++++- charts/delivery/values.yaml | 14 ++++++++ .../Startup.cs | 12 +++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 2aebb96c..0e04a84f 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -46,6 +46,38 @@ spec: - name: fabrikam-delivery image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + readinessProbe: + httpGet: + path: {{ required "readinessProbe.httpGet.path is required" .Values.readinessProbe.httpGet.path }} + port: {{ required "readinessProbe.httpGet.port is required" .Values.readinessProbe.httpGet.port }} +{{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.readinessProbe.periodSeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} +{{- end }} +{{- if .Values.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.readinessProbe.failureThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} +{{- end }} + livenessProbe: + httpGet: + path: {{ required "livenessProbe.httpGet.path is required" .Values.livenessProbe.httpGet.path }} + port: {{ required "livenessProbe.httpGet.port is required" .Values.livenessProbe.httpGet.port }} +{{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.livenessProbe.periodSeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} +{{- end }} +{{- if .Values.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.livenessProbe.failureThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} +{{- end }} resources: requests: cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} @@ -66,4 +98,4 @@ spec: value: 169.254.169.254 ports: - name: service - containerPort: 8080 \ No newline at end of file + containerPort: 8080 diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index fa70cfda..0f3cf918 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -15,6 +15,20 @@ cosmosdb: collectionid: keyvault: uri: +readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 40 + periodSeconds: 15 + timeoutSeconds: 2 + failureThreshold: 5 +livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 50 + periodSeconds: 15 telemetry: level: "Error" reason: unknown diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 66ef9f84..57d4e325 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -4,11 +4,13 @@ // ------------------------------------------------------------ using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; using Swashbuckle.AspNetCore.Swagger; using Fabrikam.DroneDelivery.DeliveryService.Models; @@ -21,6 +23,8 @@ namespace Fabrikam.DroneDelivery.DeliveryService { public class Startup { + private const string HealCheckName = "ReadinessLiveness"; + public Startup(IHostingEnvironment env) { var builder = new ConfigurationBuilder() @@ -53,6 +57,11 @@ public void ConfigureServices(IServiceCollection services) // Add framework services. services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); + // Add health check + services.AddHealthChecks().AddCheck( + HealCheckName, + () => HealthCheckResult.Healthy("OK")); + // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => { @@ -79,6 +88,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF // Important: it has to be second: Enable global exception, error handling app.UseGlobalExceptionHandler(); + // Map health checks + app.UseHealthChecks("/healthz"); + // TODO: Add middleware AuthZ here app.UseMvc(); From 37be908e08d37e5b82a3de7733d81445148d3d7a Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 4 Nov 2019 13:53:54 +0000 Subject: [PATCH 192/246] Merged PR 873: add dronescheduler readiness/liveness probe - add /health endpoint for readiness and liveness - add liveness/readiness probe cfg solve: #133008 --- .../templates/dronescheduler-deploy.yaml | 34 ++++++++++++++++++- charts/dronescheduler/values.yaml | 16 ++++++++- .../Startup.cs | 12 +++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index e691a614..c607520a 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -46,6 +46,38 @@ spec: - name: fabrikam-dronescheduler image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + readinessProbe: + httpGet: + path: {{ required "readinessProbe.httpGet.path is required" .Values.readinessProbe.httpGet.path }} + port: {{ required "readinessProbe.httpGet.port is required" .Values.readinessProbe.httpGet.port }} +{{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.readinessProbe.periodSeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} +{{- end }} +{{- if .Values.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.readinessProbe.failureThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} +{{- end }} + livenessProbe: + httpGet: + path: {{ required "livenessProbe.httpGet.path is required" .Values.livenessProbe.httpGet.path }} + port: {{ required "livenessProbe.httpGet.port is required" .Values.livenessProbe.httpGet.port }} +{{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.livenessProbe.periodSeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} +{{- end }} +{{- if .Values.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.livenessProbe.failureThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} +{{- end }} resources: requests: cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} @@ -66,4 +98,4 @@ spec: value: 169.254.169.254 ports: - name: service - containerPort: 8080 \ No newline at end of file + containerPort: 8080 diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index fd7ff2da..34af4816 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -14,6 +14,20 @@ keyvault: cosmosdb: id: collectionid: +readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 40 + periodSeconds: 15 + timeoutSeconds: 2 + failureThreshold: 5 +livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 50 + periodSeconds: 15 reason: unknown telemetry: level: "Error" @@ -22,4 +36,4 @@ tags: prod: false qa: false staging: false -current: false \ No newline at end of file +current: false diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs index 8cfb014e..a127d59b 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs @@ -4,11 +4,13 @@ // ------------------------------------------------------------ using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; using Microsoft.FeatureManagement; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; @@ -21,6 +23,8 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService { public class Startup { + private const string HealCheckName = "ReadinessLiveness"; + public Startup(IConfiguration configuration) { Configuration = configuration; @@ -42,6 +46,11 @@ public void ConfigureServices(IServiceCollection services) // Add framework services. services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); + // Add health check + services.AddHealthChecks().AddCheck( + HealCheckName, + () => HealthCheckResult.Healthy("OK")); + // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => { @@ -62,6 +71,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF .ReadFrom.Configuration(Configuration) .CreateLogger(); + // Map health checks + app.UseHealthChecks("/healthz"); + app.UseMvc(); // Enable middleware to serve generated Swagger as a JSON endpoint. From cbaa51391bd1580079f5cb855dcec98a6313269f Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 4 Nov 2019 14:04:52 +0000 Subject: [PATCH 193/246] Merged PR 870: resource quota multi-env support add support for multi-environemnt - add agent count and size per environment - multiple nginx controllers using classes instead of scopes since resources are not requested/limited in nginx, so it can not be deployed in the same namespace any more due to resource quotas. - align deployment steps and charts - remove nginx controller as dependency | NAME | REVISION | UPDATED | STATUS | CHART | APP VERSION | NAMESPACE | |-------------------------------|----------|--------------------------|----------|-----------------------|-------------|---------------------| | delivery-v0.1.0-dev | 1 | Tue Oct 22 10:44:46 2019 | DEPLOYED | delivery-v0.1.0 | v0.1.0 | backend-dev | | delivery-v0.1.0-prod | 1 | Mon Oct 28 09:05:02 2019 | DEPLOYED | delivery-v0.1.0 | v0.1.0 | backend-prod | | delivery-v0.1.0-qa | 1 | Fri Oct 25 14:30:00 2019 | DEPLOYED | delivery-v0.1.0 | v0.1.0 | backend-qa | | delivery-v0.1.0-staging | 1 | Mon Oct 28 07:59:17 2019 | DEPLOYED | delivery-v0.1.0 | v0.1.0 | backend-staging | | dronescheduler-v0.1.0-dev | 1 | Tue Oct 22 10:46:26 2019 | DEPLOYED | dronescheduler-v0.1.0 | v0.1.0 | backend-dev | | dronescheduler-v0.1.0-prod | 1 | Mon Oct 28 10:08:19 2019 | DEPLOYED | dronescheduler-v0.1.0 | v0.1.0 | backend-prod | | dronescheduler-v0.1.0-qa | 1 | Fri Oct 25 14:06:30 2019 | DEPLOYED | dronescheduler-v0.1.0 | v0.1.0 | backend-qa | | dronescheduler-v0.1.0-staging | 1 | Mon Oct 28 08:31:40 2019 | DEPLOYED | dronescheduler-v0.1.0 | v0.1.0 | backend-staging | | ingestion-v0.1.0-dev | 1 | Tue Oct 22 12:03:20 2019 | DEPLOYED | ingestion-v0.1.0 | v0.1.0 | backend-dev | | ingestion-v0.1.0-prod | 1 | Mon Oct 28 09:47:02 2019 | DEPLOYED | ingestion-v0.1.0 | v0.1.0 | backend-prod | | ingestion-v0.1.0-qa | 1 | Fri Oct 25 14:26:39 2019 | DEPLOYED | ingestion-v0.1.0 | v0.1.0 | backend-qa | | ingestion-v0.1.0-staging | 1 | Mon Oct 28 08:19:48 2019 | DEPLOYED | ingestion-v0.1.0 | v0.1.0 | backend-staging | | nginx-ingress-dev | 1 | Fri Oct 25 14:18:15 2019 | DEPLOYED | nginx-ingress-1.24.4 | 0.26.1 | ingress-controllers | | nginx-ingress-prod | 1 | Mon Oct 28 08:53:16 2019 | DEPLOYED | nginx-ingress-1.24.4 | 0.26.1 | ingress-controllers | | nginx-ingress-qa | 1 | Fri Oct 25 14:08:12 2019 | DEPLOYED | nginx-ingress-1.24.4 | 0.26.1 | ingress-controllers | | nginx-ingress-staging | 1 | Mon Oct 28 07:22:39 2019 | DEPLOYED | nginx-ingress-1.24.4 | 0.26.1 | ingress-controllers | | package-v0.1.0-dev | 1 | Tue Oct 22 10:45:43 2019 | DEPLOYED | package-v0.1.0 | v0.1.0 | backend-dev | | package-v0.1.0-prod | 1 | Mon Oct 28 09:04:37 2019 | DEPLOYED | package-v0.1.0 | v0.1.0 | backend-prod | | package-v0.1.0-qa | 1 | Fri Oct 25 14:27:25 2019 | DEPLOYED | package-v0.1.0 | v0.1.0 | backend-qa | | package-v0.1.0-staging | 1 | Mon Oct 28 08:03:02 2019 | DEPLOYED | package-v0.1.0 | v0.1.0 | backend-staging | | workflow-v0.1.0-dev | 1 | Tue Oct 22 10:45:53 2019 | DEPLOYED | workflow-v0.1.0 | v0.1.0 | backend-dev | | workflow-v0.1.0-prod | 1 | Mon Oct 28 09:4 ... --- azuredeploy.json | 40 ++++------- charts/delivery/envs/delivery-dev/values.yaml | 2 + .../delivery/envs/delivery-prod/values.yaml | 6 +- charts/delivery/envs/delivery-qa/values.yaml | 2 + .../envs/delivery-staging/values.yaml | 6 +- charts/delivery/requirements.lock | 15 ++++ .../delivery/templates/delivery-ingress.yaml | 1 + .../templates/delivery-internal-ingress.yaml | 1 + .../envs/dronescheduler-dev/values.yaml | 4 +- .../envs/dronescheduler-prod/values.yaml | 6 +- .../envs/dronescheduler-qa/values.yaml | 4 +- .../envs/dronescheduler-staging/values.yaml | 6 +- charts/dronescheduler/requirements.lock | 15 ++++ .../dronescheduler-internal-ingress.yaml | 1 + .../ingestion/envs/ingestion-dev/values.yaml | 4 +- .../ingestion/envs/ingestion-prod/values.yaml | 7 +- .../ingestion/envs/ingestion-qa/values.yaml | 4 +- .../envs/ingestion-staging/values.yaml | 7 +- charts/ingestion/requirements.lock | 7 +- charts/ingestion/requirements.yaml | 5 -- .../templates/ingestion-ingress.yaml | 1 + charts/ingestion/values.yaml | 10 +-- charts/package/envs/package-dev/values.yaml | 4 +- charts/package/envs/package-prod/values.yaml | 6 +- charts/package/envs/package-qa/values.yaml | 4 +- .../package/envs/package-staging/values.yaml | 6 +- charts/package/requirements.lock | 15 ++++ .../templates/package-internal-ingress.yaml | 1 + .../workflow/envs/workflow-prod/values.yaml | 4 +- .../envs/workflow-staging/values.yaml | 4 +- charts/workflow/requirements.lock | 15 ++++ deployment.md | 6 +- deploymentCICD.md | 16 +++-- k8s/k8s-resource-quotas-qa-stg-prod.yaml | 68 +++++++++---------- .../ingestion/azure-pipelines-cd.json | 22 ++---- 35 files changed, 189 insertions(+), 136 deletions(-) create mode 100644 charts/delivery/requirements.lock create mode 100644 charts/dronescheduler/requirements.lock create mode 100644 charts/package/requirements.lock create mode 100644 charts/workflow/requirements.lock diff --git a/azuredeploy.json b/azuredeploy.json index 9447c6e1..00b7acb0 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -78,26 +78,6 @@ }, "type": "string" }, - "agentCount": { - "type": "int", - "defaultValue": 1, - "metadata": { - "description": "The number of agents for the cluster. This value can be from 1 to 100 (note, for Kubernetes clusters you will also get 1 or 2 public agents in addition to these seleted masters)" - }, - "minValue": 1, - "maxValue": 100 - }, - "agentVMSize": { - "type": "string", - "defaultValue": "Standard_D2_v2", - "allowedValues": [ - "Standard_D2_v2", - "Standard_F8s_v2" - ], - "metadata": { - "description": "The size of the Virtual Machine." - } - }, "osType": { "type": "string", "defaultValue": "Linux", @@ -185,7 +165,9 @@ "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", - "workflowServiceAccessKey": "WorkflowServiceAccessKey" + "workflowServiceAccessKey": "WorkflowServiceAccessKey", + "agentCount": 1, + "agentVMSize": "Standard_D2_v2" }, "qa": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -206,7 +188,9 @@ "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", - "workflowServiceAccessKey": "WorkflowServiceAccessKey" + "workflowServiceAccessKey": "WorkflowServiceAccessKey", + "agentCount": 3, + "agentVMSize": "Standard_D2_v2" }, "staging": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -227,7 +211,9 @@ "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", - "workflowServiceAccessKey": "WorkflowServiceAccessKey" + "workflowServiceAccessKey": "WorkflowServiceAccessKey", + "agentCount": 3, + "agentVMSize": "Standard_D2_v2" }, "prod": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -248,7 +234,9 @@ "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "droneSchedulerCosmosDbName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", - "workflowServiceAccessKey": "WorkflowServiceAccessKey" + "workflowServiceAccessKey": "WorkflowServiceAccessKey", + "agentCount": 3, + "agentVMSize": "Standard_D2_v2" } } }, @@ -329,8 +317,8 @@ { "name": "agentpool", "osDiskSizeGB": "[parameters('osDiskSizeGB')]", - "count": "[parameters('agentCount')]", - "vmSize": "[parameters('agentVMSize')]", + "count": "[variables('environmentSettings')[parameters('environmentName')].agentCount]", + "vmSize": "[variables('environmentSettings')[parameters('environmentName')].agentVMSize]", "osType": "[parameters('osType')]", "storageProfile": "ManagedDisks" } diff --git a/charts/delivery/envs/delivery-dev/values.yaml b/charts/delivery/envs/delivery-dev/values.yaml index 2d62ea5d..6f56e4b7 100644 --- a/charts/delivery/envs/delivery-dev/values.yaml +++ b/charts/delivery/envs/delivery-dev/values.yaml @@ -9,6 +9,8 @@ exports: level: "Information" reason: "new dev deploy" current: true + ingress: + class: "nginx-dev" resources: requests: cpu: 91m diff --git a/charts/delivery/envs/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml index f2577b01..0b75ce2f 100644 --- a/charts/delivery/envs/delivery-prod/values.yaml +++ b/charts/delivery/envs/delivery-prod/values.yaml @@ -2,16 +2,18 @@ nameOverride: delivery exports: data: - replicaCount: 2 + replicaCount: 1 image: pullPolicy: IfNotPresent telemetry: level: "Error" reason: "new prod deploy" + ingress: + class: "nginx-prod" resources: requests: cpu: 100m memory: 350Mi limits: cpu: 200m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/delivery/envs/delivery-qa/values.yaml b/charts/delivery/envs/delivery-qa/values.yaml index 66ce0c20..668e722f 100644 --- a/charts/delivery/envs/delivery-qa/values.yaml +++ b/charts/delivery/envs/delivery-qa/values.yaml @@ -9,6 +9,8 @@ exports: level: "Information" reason: "new qa deploy" current: true + ingress: + class: "nginx-qa" resources: requests: cpu: 100m diff --git a/charts/delivery/envs/delivery-staging/values.yaml b/charts/delivery/envs/delivery-staging/values.yaml index 5c69c857..40ccdc91 100644 --- a/charts/delivery/envs/delivery-staging/values.yaml +++ b/charts/delivery/envs/delivery-staging/values.yaml @@ -2,17 +2,19 @@ nameOverride: delivery exports: data: - replicaCount: 2 + replicaCount: 1 image: pullPolicy: Always telemetry: level: "Information" reason: "new staging deploy" current: true + ingress: + class: "nginx-staging" resources: requests: cpu: 100m memory: 350Mi limits: cpu: 200m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/delivery/requirements.lock b/charts/delivery/requirements.lock new file mode 100644 index 00000000..2b8be7d5 --- /dev/null +++ b/charts/delivery/requirements.lock @@ -0,0 +1,15 @@ +dependencies: +- name: delivery-dev + repository: file://envs/delivery-dev + version: v0.1.0 +- name: delivery-prod + repository: file://envs/delivery-prod + version: v0.1.0 +- name: delivery-qa + repository: file://envs/delivery-qa + version: v0.1.0 +- name: delivery-staging + repository: file://envs/delivery-staging + version: v0.1.0 +digest: sha256:8f25310facbaa5f8791424964158ceddd0f5da6994d316422eab26dd02e1e3ec +generated: "2019-10-22T07:45:01.368035081-06:00" diff --git a/charts/delivery/templates/delivery-ingress.yaml b/charts/delivery/templates/delivery-ingress.yaml index d90c0b85..f90a95e9 100644 --- a/charts/delivery/templates/delivery-ingress.yaml +++ b/charts/delivery/templates/delivery-ingress.yaml @@ -15,6 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-ingress annotations: + kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries/public$1 spec: {{- if .Values.ingress.tls }} diff --git a/charts/delivery/templates/delivery-internal-ingress.yaml b/charts/delivery/templates/delivery-internal-ingress.yaml index 867221b6..417b440b 100644 --- a/charts/delivery/templates/delivery-internal-ingress.yaml +++ b/charts/delivery/templates/delivery-internal-ingress.yaml @@ -15,6 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-internal-ingress annotations: + kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries$1 nginx.ingress.kubernetes.io/configuration-snippet: | internal; diff --git a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml index 67e16d11..20906d58 100644 --- a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml @@ -9,10 +9,12 @@ exports: level: "Information" reason: "new dev deploy" current: true + ingress: + class: "nginx-dev" resources: requests: cpu: 115m memory: 350Mi limits: cpu: 175m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml index 09c64fb3..d5f29bcb 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml @@ -2,16 +2,18 @@ nameOverride: dronescheduler exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: IfNotPresent telemetry: level: "Error" reason: "new prod deploy" + ingress: + class: "nginx-prod" resources: requests: cpu: 130m memory: 350Mi limits: cpu: 270m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml index 26785142..5f0b9786 100644 --- a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml @@ -9,10 +9,12 @@ exports: level: "Information" reason: "new qa deploy" current: true + ingress: + class: "nginx-qa" resources: requests: cpu: 130m memory: 350Mi limits: cpu: 200m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml index db5fea9f..353a744c 100644 --- a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml @@ -2,17 +2,19 @@ nameOverride: dronescheduler exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: Always telemetry: level: "Information" reason: "new staging deploy" current: true + ingress: + class: "nginx-staging" resources: requests: cpu: 130m memory: 350Mi limits: cpu: 270m - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/charts/dronescheduler/requirements.lock b/charts/dronescheduler/requirements.lock new file mode 100644 index 00000000..a08805eb --- /dev/null +++ b/charts/dronescheduler/requirements.lock @@ -0,0 +1,15 @@ +dependencies: +- name: dronescheduler-dev + repository: file://envs/dronescheduler-dev + version: v0.1.0 +- name: dronescheduler-prod + repository: file://envs/dronescheduler-prod + version: v0.1.0 +- name: dronescheduler-qa + repository: file://envs/dronescheduler-qa + version: v0.1.0 +- name: dronescheduler-staging + repository: file://envs/dronescheduler-staging + version: v0.1.0 +digest: sha256:37064a63155187c00aa56a0083ee9a241f69d0c0cdbdce487b3bcbb40c81b179 +generated: "2019-10-16T15:49:58.129570048-06:00" diff --git a/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml b/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml index 375e2899..99b15734 100644 --- a/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml +++ b/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml @@ -15,6 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-internal-ingress annotations: + kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} nginx.ingress.kubernetes.io/rewrite-target: /api/dronedeliveries$1 nginx.ingress.kubernetes.io/configuration-snippet: | internal; diff --git a/charts/ingestion/envs/ingestion-dev/values.yaml b/charts/ingestion/envs/ingestion-dev/values.yaml index bf1eb186..c66eb389 100644 --- a/charts/ingestion/envs/ingestion-dev/values.yaml +++ b/charts/ingestion/envs/ingestion-dev/values.yaml @@ -9,10 +9,12 @@ exports: level: "info" reason: "new dev deploy" current: true + ingress: + class: "nginx-dev" resources: requests: cpu: 129m memory: 600Mi limits: cpu: 194m - memory: 800Mi \ No newline at end of file + memory: 800Mi diff --git a/charts/ingestion/envs/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml index 2d4f77fc..06575b29 100644 --- a/charts/ingestion/envs/ingestion-prod/values.yaml +++ b/charts/ingestion/envs/ingestion-prod/values.yaml @@ -8,13 +8,12 @@ exports: telemetry: level: "error" reason: "new prod deploy" - nginx-ingress: - controller: - replicaCount: 2 + ingress: + class: "nginx-prod" resources: requests: cpu: 150m memory: 600Mi limits: cpu: 300m - memory: 800Mi \ No newline at end of file + memory: 800Mi diff --git a/charts/ingestion/envs/ingestion-qa/values.yaml b/charts/ingestion/envs/ingestion-qa/values.yaml index 0c5a0513..40473576 100644 --- a/charts/ingestion/envs/ingestion-qa/values.yaml +++ b/charts/ingestion/envs/ingestion-qa/values.yaml @@ -9,10 +9,12 @@ exports: level: "info" reason: "new qa deploy" current: true + ingress: + class: "nginx-qa" resources: requests: cpu: 150m memory: 600Mi limits: cpu: 220m - memory: 800Mi \ No newline at end of file + memory: 800Mi diff --git a/charts/ingestion/envs/ingestion-staging/values.yaml b/charts/ingestion/envs/ingestion-staging/values.yaml index e94d1cde..3b3cb756 100644 --- a/charts/ingestion/envs/ingestion-staging/values.yaml +++ b/charts/ingestion/envs/ingestion-staging/values.yaml @@ -8,9 +8,8 @@ exports: telemetry: level: "info" reason: "new staging deploy" - nginx-ingress: - controller: - replicaCount: 2 + ingress: + class: "nginx-staging" current: true resources: requests: @@ -18,4 +17,4 @@ exports: memory: 600Mi limits: cpu: 300m - memory: 800Mi \ No newline at end of file + memory: 800Mi diff --git a/charts/ingestion/requirements.lock b/charts/ingestion/requirements.lock index c10856a9..7e7ca650 100644 --- a/charts/ingestion/requirements.lock +++ b/charts/ingestion/requirements.lock @@ -1,7 +1,4 @@ dependencies: -- name: nginx-ingress - repository: https://kubernetes-charts.storage.googleapis.com - version: 1.6.15 - name: ingestion-dev repository: file://envs/ingestion-dev version: v0.1.0 @@ -14,5 +11,5 @@ dependencies: - name: ingestion-staging repository: file://envs/ingestion-staging version: v0.1.0 -digest: sha256:c2cdc67015f90e1ff33c49603a6ece3f2a99dd8e28e88e8756ef99041820b47a -generated: 2019-05-29T10:41:19.509573491-03:00 +digest: sha256:feee2c2842b5de8b8e7fda4a388b62ff5e85438633a480b40518c1c3371adcc5 +generated: "2019-10-25T14:37:00.76647856-06:00" diff --git a/charts/ingestion/requirements.yaml b/charts/ingestion/requirements.yaml index ad03c9b6..b09f4aff 100644 --- a/charts/ingestion/requirements.yaml +++ b/charts/ingestion/requirements.yaml @@ -1,9 +1,4 @@ dependencies: -- name: nginx-ingress - repository: https://kubernetes-charts.storage.googleapis.com - version: ">=1.6.9" - condition: nginx-ingress.enabled - - name: ingestion-dev repository: "file://envs/ingestion-dev" version: ">= 0.0.1" diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index 4ad01fc1..8581606b 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -15,6 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-ingress annotations: + kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} nginx.ingress.kubernetes.io/rewrite-target: /api/deliveryrequests$1 spec: {{- if .Values.ingress.tls }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index eb47cfa1..c887002d 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -9,17 +9,9 @@ image: reason: unknown telemetry: level: "error" -nginx-ingress: - fullnameOverride: nginx-ingress - rbac: - create: true - controller: - scope: - enabled: true - enabled: false tags: dev: false prod: false qa: false staging: false -current: false \ No newline at end of file +current: false diff --git a/charts/package/envs/package-dev/values.yaml b/charts/package/envs/package-dev/values.yaml index d1d4fb1d..dade913a 100644 --- a/charts/package/envs/package-dev/values.yaml +++ b/charts/package/envs/package-dev/values.yaml @@ -9,10 +9,12 @@ exports: level: "info" reason: "new dev deploy" current: true + ingress: + class: "nginx-dev" resources: requests: cpu: 78m memory: 100Mi limits: cpu: 117m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/package/envs/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml index 3b280e36..b29a0cb7 100644 --- a/charts/package/envs/package-prod/values.yaml +++ b/charts/package/envs/package-prod/values.yaml @@ -2,16 +2,18 @@ nameOverride: package exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: IfNotPresent log: level: "error" reason: "new prod deploy" + ingress: + class: "nginx-prod" resources: requests: cpu: 90m memory: 100Mi limits: cpu: 180m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/package/envs/package-qa/values.yaml b/charts/package/envs/package-qa/values.yaml index 49141ff4..439ccf81 100644 --- a/charts/package/envs/package-qa/values.yaml +++ b/charts/package/envs/package-qa/values.yaml @@ -9,10 +9,12 @@ exports: level: "info" reason: "new qa deploy" current: true + ingress: + class: "nginx-qa" resources: requests: cpu: 90m memory: 100Mi limits: cpu: 130m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/package/envs/package-staging/values.yaml b/charts/package/envs/package-staging/values.yaml index 115d2c78..ee3acef2 100644 --- a/charts/package/envs/package-staging/values.yaml +++ b/charts/package/envs/package-staging/values.yaml @@ -2,17 +2,19 @@ nameOverride: package exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: Always log: level: "info" reason: "new staging deploy" current: true + ingress: + class: "nginx-staging" resources: requests: cpu: 90m memory: 100Mi limits: cpu: 180m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/package/requirements.lock b/charts/package/requirements.lock new file mode 100644 index 00000000..74e81899 --- /dev/null +++ b/charts/package/requirements.lock @@ -0,0 +1,15 @@ +dependencies: +- name: package-dev + repository: file://envs/package-dev + version: v0.1.0 +- name: package-prod + repository: file://envs/package-prod + version: v0.1.0 +- name: package-qa + repository: file://envs/package-qa + version: v0.1.0 +- name: package-staging + repository: file://envs/package-staging + version: v0.1.0 +digest: sha256:058409196b33c962ddb1c1a554013e7840c7ccfc1ddf3a98e12559aeeef05b91 +generated: "2019-10-22T08:05:25.923366271-06:00" diff --git a/charts/package/templates/package-internal-ingress.yaml b/charts/package/templates/package-internal-ingress.yaml index dbad0b09..969b7136 100644 --- a/charts/package/templates/package-internal-ingress.yaml +++ b/charts/package/templates/package-internal-ingress.yaml @@ -15,6 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-internal-ingress annotations: + kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} nginx.ingress.kubernetes.io/rewrite-target: /api/package$1 nginx.ingress.kubernetes.io/configuration-snippet: | internal; diff --git a/charts/workflow/envs/workflow-prod/values.yaml b/charts/workflow/envs/workflow-prod/values.yaml index b0754a46..d8f0f287 100644 --- a/charts/workflow/envs/workflow-prod/values.yaml +++ b/charts/workflow/envs/workflow-prod/values.yaml @@ -2,7 +2,7 @@ nameOverride: workflow exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: IfNotPresent telemetry: @@ -14,4 +14,4 @@ exports: memory: 100Mi limits: cpu: 1000m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/workflow/envs/workflow-staging/values.yaml b/charts/workflow/envs/workflow-staging/values.yaml index 3c2f2561..14d57289 100644 --- a/charts/workflow/envs/workflow-staging/values.yaml +++ b/charts/workflow/envs/workflow-staging/values.yaml @@ -2,7 +2,7 @@ nameOverride: workflow exports: data: - replicaCount: 3 + replicaCount: 1 image: pullPolicy: Always telemetry: @@ -14,4 +14,4 @@ exports: memory: 100Mi limits: cpu: 1000m - memory: 140Mi \ No newline at end of file + memory: 140Mi diff --git a/charts/workflow/requirements.lock b/charts/workflow/requirements.lock new file mode 100644 index 00000000..a7674a08 --- /dev/null +++ b/charts/workflow/requirements.lock @@ -0,0 +1,15 @@ +dependencies: +- name: workflow-dev + repository: file://envs/workflow-dev + version: v0.1.0 +- name: workflow-prod + repository: file://envs/workflow-prod + version: v0.1.0 +- name: workflow-qa + repository: file://envs/workflow-qa + version: v0.1.0 +- name: workflow-staging + repository: file://envs/workflow-staging + version: v0.1.0 +digest: sha256:55b157e79d9bdb5a1a768545096e255d5b3fec4cc5a47a622983ebfa62b04b39 +generated: "2019-10-22T08:12:09.57949386-06:00" diff --git a/deployment.md b/deployment.md index d64ac575..d0d16ec8 100644 --- a/deployment.md +++ b/deployment.md @@ -179,12 +179,12 @@ kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-fl ```bash # Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress --namespace ingress-controllers --set rbac.create=true +helm install stable/nginx-ingress --name nginx-ingress-dev --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-dev # Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done +until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-dev-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" +export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest-dev" export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) # Create a self-signed certificate for TLS diff --git a/deploymentCICD.md b/deploymentCICD.md index 1a644650..af64df69 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -23,7 +23,7 @@ az deployment create \ resourceGroupLocation=$LOCATION \ environmentName=${env} -export {${ENV}_IDENTITIES_DEPLOYMENT_NAME,IDENTITIES_DEPLOYMENT_NAME}=$(az deployment show -n azuredeploy-prereqs-dev --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ +export {${ENV}_IDENTITIES_DEPLOYMENT_NAME,IDENTITIES_DEPLOYMENT_NAME}=$(az deployment show -n azuredeploy-adv-prereqs-adv-${env} --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ export {${ENV}_DELIVERY_ID_NAME,DELIVERY_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryIdName.value -o tsv) export DELIVERY_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query principalId -o tsv) export {${ENV}_DRONESCHEDULER_ID_NAME,DRONESCHEDULER_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerIdName.value -o tsv) @@ -51,8 +51,7 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa workflowPrincipalId=$WORKFLOW_ID_PRINCIPAL_ID \ acrResourceGroupName=${RESOURCE_GROUP_ACR} \ acrResourceGroupLocation=$LOCATION \ - environmentName=${env} \ - agentCount=3 + environmentName=${env} export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) @@ -82,6 +81,8 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces +kubectl create namespace backend-qa && \ +kubectl create namespace backend-staging && \ kubectl create namespace backend ``` @@ -205,10 +206,15 @@ export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" for env in dev qa staging prod;do ENV=${env^^} + az network public-ip create --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --dns-name ${EXTERNAL_INGEST_DNS_NAME}-${env} --allocation-method static -g $RESOURCE_GROUP_NODE EXTERNAL_INGEST_FQDN=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "dnsSettings.fqdn" -g $RESOURCE_GROUP_NODE --output tsv) export ${ENV}_EXTERNAL_INGEST_FQDN=${EXTERNAL_INGEST_FQDN} export ${ENV}_INGRESS_LOAD_BALANCER_IP=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "ipAddress" -g $RESOURCE_GROUP_NODE --output tsv) + +# Deploy the ngnix ingress controller +helm install stable/nginx-ingress --name nginx-ingress-${env} --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-${env} --set controller.service.loadBalancerIP=${ENV}_INGRESS_LOAD_BALANCER_IP + openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -out ingestion-ingress-tls-${env}.crt \ -keyout ingestion-ingress-tls-${env}.key \ @@ -574,7 +580,6 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#DEV_AI_IKEY_VAR_VAL#$DEV_AI_IKEY#g" | \ sed "s#DEV_ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" | \ sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ - sed "s#DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$DEV_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#DEV_EXTERNAL_INGEST_FQDN_VAR_VAL#$DEV_EXTERNAL_INGEST_FQDN#g" | \ sed "s#DEV_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$DEV_INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#DEV_INGESTION_QUEUE_NAME_VAR_VAL#$DEV_INGESTION_QUEUE_NAME#g" | \ @@ -586,7 +591,6 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ - sed "s#QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$QA_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#QA_EXTERNAL_INGEST_FQDN_VAR_VAL#$QA_EXTERNAL_INGEST_FQDN#g" | \ sed "s#QA_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$QA_INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#QA_INGESTION_QUEUE_NAME_VAR_VAL#$QA_INGESTION_QUEUE_NAME#g" | \ @@ -598,7 +602,6 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ - sed "s#STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$STAGING_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL#$STAGING_EXTERNAL_INGEST_FQDN#g" | \ sed "s#STAGING_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$STAGING_INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#STAGING_INGESTION_QUEUE_NAME_VAR_VAL#$STAGING_INGESTION_QUEUE_NAME#g" | \ @@ -612,7 +615,6 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ - sed "s#PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL#$PROD_INGRESS_LOAD_BALANCER_IP#g" | \ sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ sed "s#PROD_INGESTION_QUEUE_NAMESPACE_VAR_VAL#$PROD_INGESTION_QUEUE_NAMESPACE#g" | \ sed "s#PROD_INGESTION_QUEUE_NAME_VAR_VAL#$PROD_INGESTION_QUEUE_NAME#g" | \ diff --git a/k8s/k8s-resource-quotas-qa-stg-prod.yaml b/k8s/k8s-resource-quotas-qa-stg-prod.yaml index 797ae7d5..c0e65c95 100644 --- a/k8s/k8s-resource-quotas-qa-stg-prod.yaml +++ b/k8s/k8s-resource-quotas-qa-stg-prod.yaml @@ -3,37 +3,37 @@ # Licensed under the MIT License (MIT). See License.txt in the repo root for license information. # ------------------------------------------------------------ - apiVersion: v1 - kind: ResourceQuota - metadata: - name: qa - namespace: backend-qa - spec: - hard: - requests.cpu: "1" - requests.memory: 2Gi - limits.cpu: "2" - limits.memory: 5Gi - pods: "5" - --- - apiVersion: v1 - kind: ResourceQuota - metadata: - name: staging - namespace: backend-staging - spec: - hard: - cpu: "2" - memory: 8Gi - pods: "10" - --- - apiVersion: v1 - kind: ResourceQuota - metadata: - name: prod - namespace: backend - spec: - hard: - cpu: "2" - memory: 8Gi - pods: "10" +apiVersion: v1 +kind: ResourceQuota +metadata: + name: qa + namespace: backend-qa +spec: + hard: + requests.cpu: "1" + requests.memory: 2Gi + limits.cpu: "2" + limits.memory: 5Gi + pods: "5" +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: staging + namespace: backend-staging +spec: + hard: + cpu: "2" + memory: 8Gi + pods: "10" +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: prod + namespace: backend +spec: + hard: + cpu: "2" + memory: 8Gi + pods: "10" diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 8b222409..26506053 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -29,9 +29,6 @@ "value": "DEV_AI_IKEY_VAR_VAL", "isSecret": true }, - "INGRESS_LOAD_BALANCER_IP": { - "value": "DEV_INGRESS_LOAD_BALANCER_IP_VAR_VAL" - }, "EXTERNAL_INGEST_FQDN": { "value": "DEV_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -243,7 +240,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -415,9 +412,6 @@ "value": "QA_AI_IKEY_VAR_VAL", "isSecret": true }, - "INGRESS_LOAD_BALANCER_IP": { - "value": "QA_INGRESS_LOAD_BALANCER_IP_VAR_VAL" - }, "EXTERNAL_INGEST_FQDN": { "value": "QA_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -629,7 +623,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -801,9 +795,6 @@ "value": "STAGING_AI_IKEY_VAR_VAL", "isSecret": true }, - "INGRESS_LOAD_BALANCER_IP": { - "value": "STAGING_INGRESS_LOAD_BALANCER_IP_VAR_VAL" - }, "EXTERNAL_INGEST_FQDN": { "value": "STAGING_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -1015,7 +1006,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1187,9 +1178,6 @@ "value": "PROD_AI_IKEY_VAR_VAL", "isSecret": true }, - "INGRESS_LOAD_BALANCER_IP": { - "value": "PROD_INGRESS_LOAD_BALANCER_IP_VAR_VAL" - }, "EXTERNAL_INGEST_FQDN": { "value": "PROD_EXTERNAL_INGEST_FQDN_VAR_VAL" }, @@ -1435,7 +1423,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1729,7 +1717,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),nginx-ingress.enabled=true,nginx-ingress.controller.service.loadBalancerIP=$(INGRESS_LOAD_BALANCER_IP),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", From 2ce470fd1321d7e9d3d30ff71dec714b10d492d0 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 4 Nov 2019 14:26:12 +0000 Subject: [PATCH 194/246] Merged PR 867: implement advanced networking implement advanced networking There is no need to explicitly grant the permission documented in https://docs.microsoft.com/en-us/azure/aks/configure-azure-cni#prerequisites to the cluster's service principal as it is created as a contributor for the subscription. --- azuredeploy.json | 55 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 9447c6e1..d2016dc6 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -159,16 +159,24 @@ "acrNamePrefix": "acr", "aiNamePrefix": "ai", "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "aksVnetAddressPrefix": "10.10.0.0/16", + "aksClusterSubnetPrefix": "10.10.0.0/21", + "aksVnetNamePrefix": "vnet", + "aksClusterSubnetNamePrefix": "subnet", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", + "networkContributorRoleObjectId": "4d97b98b-1d4f-4787-a291-c67834d212e7", "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]", + "networkContributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('networkContributorRoleObjectId'))]", "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", "nestedACRDeploymentName": "[concat('azuredeploy-acr-',parameters('acrResourceGroupName'),parameters('environmentName'))]", "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", "acrName": "[variables('acrName')]", + "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", + "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -190,6 +198,8 @@ "qa": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", "acrName": "[variables('acrName')]", + "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", + "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -211,6 +221,8 @@ "staging": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", "acrName": "[variables('acrName')]", + "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", + "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -232,6 +244,8 @@ "prod": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", "acrName": "[concat(parameters('environmentName'),variables('acrName'))]", + "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", + "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -314,6 +328,33 @@ } } }, + { + "name": "[variables('environmentSettings')[parameters('environmentName')].aksVnetName]", + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2018-06-01", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('aksVnetAddressPrefix')]" + ] + } + }, + "resources": [ + { + "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", + "type": "subnets", + "apiVersion": "2018-06-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('environmentSettings')[parameters('environmentName')].aksVnetName]" + ], + "properties": { + "addressPrefix": "[variables('aksClusterSubnetPrefix')]" + } + } + ] + }, { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", @@ -322,6 +363,9 @@ "tags": { "environment": "shared cluster" }, + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "properties": { "kubernetesVersion": "[parameters('kubernetesVersion')]", "dnsPrefix": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", @@ -332,7 +376,8 @@ "count": "[parameters('agentCount')]", "vmSize": "[parameters('agentVMSize')]", "osType": "[parameters('osType')]", - "storageProfile": "ManagedDisks" + "storageProfile": "ManagedDisks", + "vnetSubnetID": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" } ], "linuxProfile": { @@ -349,7 +394,13 @@ "clientId": "[parameters('servicePrincipalClientId')]", "secret": "[parameters('servicePrincipalClientSecret')]" }, - "enableRBAC": true + "enableRBAC": true, + "networkProfile": { + "networkPlugin": "azure", + "serviceCidr": "10.2.0.0/24", + "dnsServiceIP": "10.2.0.10", + "dockerBridgeCidr": "172.17.0.1/16" + } } }, { From c7881c44f2a3e1538cf0825e641cb7e049ed1c7f Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Wed, 6 Nov 2019 16:26:59 +0000 Subject: [PATCH 195/246] Merged PR 876: make readiness/liveness configurable with helm values make readiness/liveness configurable with helm values solved: #133002 --- .../ingestion/templates/ingestion-deploy.yaml | 44 +++++++++++++------ charts/ingestion/values.yaml | 16 +++++++ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index c211a466..f4bedcce 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -41,22 +41,38 @@ spec: - name: &ingestion-container_name fabrikam-ingestion image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - livenessProbe: - httpGet: - path: /actuator/health - port: 80 - initialDelaySeconds: 120 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 5 readinessProbe: httpGet: - path: /api/probe - port: 80 - initialDelaySeconds: 120 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 5 + path: {{ required "readinessProbe.httpGet.path is required" .Values.readinessProbe.httpGet.path }} + port: {{ required "readinessProbe.httpGet.port is required" .Values.readinessProbe.httpGet.port }} +{{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.readinessProbe.periodSeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} +{{- end }} +{{- if .Values.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.readinessProbe.failureThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} +{{- end }} + livenessProbe: + httpGet: + path: {{ required "livenessProbe.httpGet.path is required" .Values.livenessProbe.httpGet.path }} + port: {{ required "livenessProbe.httpGet.port is required" .Values.livenessProbe.httpGet.port }} +{{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.livenessProbe.periodSeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} +{{- end }} +{{- if .Values.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.livenessProbe.failureThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} +{{- end }} resources: requests: cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index c887002d..cc6b14ee 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -7,6 +7,22 @@ image: tag: pullPolicy: IfNotPresent reason: unknown +livenessProbe: + httpGet: + path: /actuator/health + port: 80 + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 5 +readinessProbe: + httpGet: + path: /api/probe + port: 80 + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 5 telemetry: level: "error" tags: From 78cd88b84f850cb83abfe566a32387b5f3a2cc17 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 13:51:20 +0000 Subject: [PATCH 196/246] Merged PR 878: add package readiness/liveness probe - add /healthz endpoint for readiness and liveness - add liveness/readiness probe cfg solved: #133005 --- charts/package/templates/package-deploy.yaml | 32 ++++++++++++++++++++ charts/package/values.yaml | 16 +++++++++- src/shipping/package/app/server.ts | 10 ++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index d45af23c..5c6db0f5 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -41,6 +41,38 @@ spec: - name: &package-container_name fabrikam-package image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + readinessProbe: + httpGet: + path: {{ required "readinessProbe.httpGet.path is required" .Values.readinessProbe.httpGet.path }} + port: {{ required "readinessProbe.httpGet.port is required" .Values.readinessProbe.httpGet.port }} +{{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.readinessProbe.periodSeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} +{{- end }} +{{- if .Values.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.readinessProbe.failureThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} +{{- end }} + livenessProbe: + httpGet: + path: {{ required "livenessProbe.httpGet.path is required" .Values.livenessProbe.httpGet.path }} + port: {{ required "livenessProbe.httpGet.port is required" .Values.livenessProbe.httpGet.port }} +{{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.livenessProbe.periodSeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} +{{- end }} +{{- if .Values.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.livenessProbe.failureThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} +{{- end }} resources: requests: cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 3d6d4518..75c6bf44 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -8,6 +8,20 @@ image: tag: pullPolicy: IfNotPresent reason: unknown +readinessProbe: + httpGet: + path: /healthz + port: 80 + initialDelaySeconds: 40 + periodSeconds: 15 + timeoutSeconds: 2 + failureThreshold: 5 +livenessProbe: + httpGet: + path: /healthz + port: 80 + initialDelaySeconds: 50 + periodSeconds: 15 log: level: error cosmosDb: @@ -17,4 +31,4 @@ tags: prod: false qa: false staging: false -current: false \ No newline at end of file +current: false diff --git a/src/shipping/package/app/server.ts b/src/shipping/package/app/server.ts index f5a8ad7d..b25065f2 100644 --- a/src/shipping/package/app/server.ts +++ b/src/shipping/package/app/server.ts @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +const _ = require('koa-route'); import * as Koa from 'koa'; import * as bodyParser from "koa-bodyparser"; @@ -40,6 +41,15 @@ export class PackageService { // Configure logging app.use(logger(Settings.logLevel())); + // Add simple health check endpoint + app.use(_.get('/healthz', (ctx) => { + var logger : ILogger = ctx.state.logger; + logger.info('Readiness/Liveness Probe Status: %s', "OK"); + + ctx.status = 200; + ctx.body = {status: 'OK'}; + })); + // Configure global exception handling // Use: ctx.throw('Error Message', 500); // in the controller methods to set the status code and exception message From 9c5dbcaffe60ce8205578ef064d916bdf710f826 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 13:53:30 +0000 Subject: [PATCH 197/246] Merged PR 875: add readiness/liveness probes to workflow add healthz file creation based on health check status now workflow adds a readiness/liveness health check, and also a health check publisher. This way, the health check system periodically executes this check. - create file when reporting healthy - delete file when reporting unhealthy - add health check - add health check publisher to o the service container, - add publisher unit tests - add readiness/liveness probes cfg solved: #133010 --- .../workflow/templates/workflow-deploy.yaml | 38 ++++++ charts/workflow/values.yaml | 22 +++- .../ReadinessLivenessPublisherTests.cs | 120 ++++++++++++++++++ .../Fabrikam.Workflow.Service.csproj | 1 + .../ServiceStartup.cs | 27 ++++ .../Services/ReadinessLivenessPublisher.cs | 81 ++++++++++++ 6 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs create mode 100644 src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 97d6fa3c..8e1768aa 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -46,6 +46,42 @@ spec: - name: fabrikam-workflow image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + readinessProbe: + exec: + command: +{{- range .Values.readinessProbe.exec.command }} + - {{ . | quote }} +{{- end }} +{{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.readinessProbe.periodSeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} +{{- end }} +{{- if .Values.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.readinessProbe.failureThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} +{{- end }} + livenessProbe: + exec: + command: +{{- range .Values.livenessProbe.exec.command }} + - {{ . | quote }} +{{- end }} +{{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- if .Values.livenessProbe.periodSeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} +{{- end }} +{{- if .Values.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} +{{- end }} +{{- if .Values.livenessProbe.failureThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} +{{- end }} resources: requests: cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }} @@ -60,6 +96,8 @@ spec: env: - name: CONFIGURATION_FOLDER value: /kvmnt + - name: HEALTHCHECK_INITIAL_DELAY + value: {{ default "30000" .Values.healthcheck.delay | quote }} - name: SERVICE_URI_DELIVERY value: {{ .Values.serviceuri.delivery }} - name: SERVICE_URI_DRONE diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 8e04f00c..ef27c6bf 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -23,6 +23,26 @@ servicerequest: circuitbreakerbreakduration: 30 maxbulkheadsize: 100 maxbulkheadqueuesize: 25 +healthcheck: + delay: +readinessProbe: + exec: + command: + - cat + - /app/healthz + initialDelaySeconds: 40 + periodSeconds: 15 + timeoutSeconds: 2 + failureThreshold: 5 +livenessProbe: + exec: + command: + - find + - /app/healthz + - -mmin + - -1 + initialDelaySeconds: 50 + periodSeconds: 30 keyvault: name: resourcegroup: @@ -34,4 +54,4 @@ tags: dev: false prod: false qa: false - staging: false \ No newline at end of file + staging: false diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs new file mode 100644 index 00000000..37aa7a76 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs @@ -0,0 +1,120 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Fabrikam.Workflow.Service.Services; +using Moq; +using Xunit; + +namespace Fabrikam.Workflow.Service.Tests +{ + public class ReadinessLivenessPublisherTests + { + private const int DelayCompletionMs = 1000; + + private readonly ReadinessLivenessPublisher _publisher; + + public ReadinessLivenessPublisherTests() + { + var servicesBuilder = new ServiceCollection(); + servicesBuilder.AddLogging(logging => logging.AddDebug()); + var services = servicesBuilder.BuildServiceProvider(); + + _publisher = + new ReadinessLivenessPublisher( + services.GetService>()); + } + + [Fact] + public async Task WhenPublishingAndReportIsHealthy_FileExists() + { + // Arrange + var healthReportEntries = new Dictionary() + { + {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } + }; + + // Act + await _publisher.PublishAsync( + new HealthReport(healthReportEntries, TimeSpan.MinValue), + new CancellationTokenSource().Token); + + // Arrange + Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath)); + } + + [Fact] + public async Task WhenPublishingAndReportIsUnhealthy_FileDateTimeIsNotModified() + { + // Arrange + var healthReportEntries = new Dictionary() + { + {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } + }; + + await _publisher.PublishAsync( + new HealthReport( + healthReportEntries, + TimeSpan.MinValue), + new CancellationTokenSource().Token); + + healthReportEntries.Add( + "unhealthy", + new HealthReportEntry( + HealthStatus.Unhealthy, + null,TimeSpan.MinValue, null, null)); + + // Act + DateTime healthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); + await _publisher.PublishAsync( + new HealthReport(healthReportEntries, TimeSpan.MinValue), + new CancellationTokenSource().Token); + + // Arrange + Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath)); + Assert.Equal(healthyWriteTime, File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath)); + } + + [Fact(Timeout = DelayCompletionMs * 3)] + public async Task WhenPublishingAndReportIsHealthyTwice_FileDateTimeIsModified() + { + // Arrange + Func emulatePeriodicHealthCheckAsync = + () => Task.Delay(DelayCompletionMs); + + var healthReportEntries = new Dictionary() + { + {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } + }; + + // Act + await _publisher.PublishAsync( + new HealthReport( + healthReportEntries, + TimeSpan.MinValue), + new CancellationTokenSource().Token); + + DateTime firstTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); + + await emulatePeriodicHealthCheckAsync(); + + await _publisher.PublishAsync( + new HealthReport(healthReportEntries, TimeSpan.MinValue), + new CancellationTokenSource().Token); + + DateTime sencondTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); + + // Arrange + Assert.True(firstTimehealthyWriteTime < sencondTimehealthyWriteTime); + } + } +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index 08e604dc..ce5a6eae 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -8,6 +8,7 @@ + diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs index a98e48aa..0c63fd57 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs @@ -5,6 +5,8 @@ using System; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; using Fabrikam.Workflow.Service.RequestProcessing; using Fabrikam.Workflow.Service.Services; @@ -13,6 +15,9 @@ namespace Fabrikam.Workflow.Service { public static class ServiceStartup { + private const string HealthCheckName = "ReadinessLiveness"; + private const string HealthCheckServiceAssembly = "Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckPublisherHostedService"; + public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { services.AddOptions(); @@ -26,6 +31,20 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect services.AddTransient(); + // Add health check │ + services.AddHealthChecks().AddCheck( + HealthCheckName, + () => HealthCheckResult.Healthy("OK")); + + if (context.Configuration["HEALTHCHECK_INITIAL_DELAY"] is var configuredDelay && + double.TryParse(configuredDelay, out double delay)) + { + services.Configure(options => + { + options.Delay = TimeSpan.FromMilliseconds(delay); + }); + } + services .AddHttpClient(c => { @@ -46,6 +65,14 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DELIVERY"]); }) .AddResiliencyPolicies(context.Configuration); + + // workaround .NET Core 2.2: for more info https://github.com/aspnet/AspNetCore.Docs/blob/master/aspnetcore/host-and-deploy/health-checks/samples/2.x/HealthChecksSample/LivenessProbeStartup.cs#L51 + services.TryAddEnumerable( + ServiceDescriptor.Singleton(typeof(IHostedService), + typeof(HealthCheckPublisherOptions).Assembly + .GetType(HealthCheckServiceAssembly))); + + services.AddSingleton(); } } } diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs new file mode 100644 index 00000000..f36dc887 --- /dev/null +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace Fabrikam.Workflow.Service.Services +{ + public class ReadinessLivenessPublisher : IHealthCheckPublisher + { + public const string FilePath = "healthz"; + + private readonly ILogger _logger; + + public ReadinessLivenessPublisher(ILogger logger) + { + this._logger = logger; + } + + public Task PublishAsync(HealthReport report, + CancellationToken cancellationToken) + { + switch (report.Status) + { + case HealthStatus.Healthy: + { + this._logger.LogInformation( + "{Timestamp} Readiness/Liveness Probe Status: {Result}", + DateTime.UtcNow, + report.Status); + + CreateOrUpdateHealthz(); + + break; + } + + case HealthStatus.Degraded: + { + this._logger.LogWarning( + "{Timestamp} Readiness/Liveness Probe Status: {Result}", + DateTime.UtcNow, + report.Status); + + break; + } + + case HealthStatus.Unhealthy: + { + this._logger.LogError( + "{Timestamp} Readiness Probe/Liveness Status: {Result}", + DateTime.UtcNow, + report.Status); + + break; + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + return Task.CompletedTask; + } + + private static void CreateOrUpdateHealthz() + { + if (File.Exists(FilePath)) + { + File.SetLastWriteTimeUtc(FilePath, DateTime.UtcNow); + } + else + { + File.AppendText(FilePath).Close(); + } + } + } +} From 1ae161294515044d1f9593de60370db27bb3069c Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 14:10:49 +0000 Subject: [PATCH 198/246] Merged PR 879: add hpa for delivery add hpa for delivery now delivery autoscales using HPA in staging and prod when it passes 50% of CPU utilization - it will scale up to 50 replicas - increment resource quota - modify helm chart ==> v2beta1/HorizontalPodAutoscaler | NAME | REFERENCE | TARGETS | MINPODS | MAXPODS | REPLICAS | AGE | |------------------------|-------------------------------|---------|---------|---------|----------|------| | delivery-v010-prod-hpa | Deployment/delivery-v010-prod | 1%/50% | 1 | 50 | 1 | 4m2s | solved: #133012 --- .../delivery/envs/delivery-prod/values.yaml | 5 +++ .../envs/delivery-staging/values.yaml | 5 +++ charts/delivery/templates/delivery-hpa.yaml | 35 +++++++++++++++++++ charts/delivery/values.yaml | 5 +++ 4 files changed, 50 insertions(+) create mode 100644 charts/delivery/templates/delivery-hpa.yaml diff --git a/charts/delivery/envs/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml index 0b75ce2f..0c9dd8b9 100644 --- a/charts/delivery/envs/delivery-prod/values.yaml +++ b/charts/delivery/envs/delivery-prod/values.yaml @@ -17,3 +17,8 @@ exports: limits: cpu: 200m memory: 500Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/delivery/envs/delivery-staging/values.yaml b/charts/delivery/envs/delivery-staging/values.yaml index 40ccdc91..091823ba 100644 --- a/charts/delivery/envs/delivery-staging/values.yaml +++ b/charts/delivery/envs/delivery-staging/values.yaml @@ -18,3 +18,8 @@ exports: limits: cpu: 200m memory: 500Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/delivery/templates/delivery-hpa.yaml b/charts/delivery/templates/delivery-hpa.yaml new file mode 100644 index 00000000..4c8b22bc --- /dev/null +++ b/charts/delivery/templates/delivery-hpa.yaml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery HPA Resrouce Metrics (CPU utilization threshold) +################################################################################################### +{{- if .Values.autoscaling.enabled }} +{{- $fullname := include "delivery.fullname" . | replace "." "" }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullname }}-hpa + labels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "delivery.chart" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index 0f3cf918..b4801cca 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -38,3 +38,8 @@ tags: qa: false staging: false current: false +autoscaling: + enabled: false + maxReplicas: + minReplicas: + targetCPUUtilizationPercentage: From f05306ac1d085f71a9002472cc8ce22a6cde562f Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 14:13:48 +0000 Subject: [PATCH 199/246] Merged PR 880: add hpa for dronescheduler add hpa for dronescheduler now dronescheduler autoscales using HPA in staging and prod when it passes 50% of CPU utilization - it will scale up to 50 replicas - modify helm chart ==> v2beta1/HorizontalPodAutoscaler | NAME | REFERENCE | TARGETS | MINPODS | MAXPODS | REPLICAS | AGE | |------------------------------|-------------------------------------|---------|---------|---------|----------|------| | dronescheduler-v010-prod-hpa | Deployment/dronescheduler-v010-prod | 0%/50% | 1 | 50 | 1 | 4m3s | solved: #133019 --- .../envs/dronescheduler-prod/values.yaml | 5 +++ .../envs/dronescheduler-staging/values.yaml | 5 +++ .../templates/dronescheduler-hpa.yaml | 34 +++++++++++++++++++ charts/dronescheduler/values.yaml | 5 +++ 4 files changed, 49 insertions(+) create mode 100644 charts/dronescheduler/templates/dronescheduler-hpa.yaml diff --git a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml index d5f29bcb..621194b5 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml @@ -17,3 +17,8 @@ exports: limits: cpu: 270m memory: 500Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml index 353a744c..8a252fe0 100644 --- a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml @@ -18,3 +18,8 @@ exports: limits: cpu: 270m memory: 500Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/dronescheduler/templates/dronescheduler-hpa.yaml b/charts/dronescheduler/templates/dronescheduler-hpa.yaml new file mode 100644 index 00000000..fdec84ba --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-hpa.yaml @@ -0,0 +1,34 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ +################################################################################################### +# Dronescheduler HPA Resrouce Metrics (CPU utilization threshold) +################################################################################################### +{{- if .Values.autoscaling.enabled }} +{{- $fullname := include "dronescheduler.fullname" . | replace "." "" }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullname }}-hpa + labels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "dronescheduler.chart" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 34af4816..6e6f2aad 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -37,3 +37,8 @@ tags: qa: false staging: false current: false +autoscaling: + enabled: false + maxReplicas: + minReplicas: + targetCPUUtilizationPercentage: From 6f91235ce014e7c6c5ca33cc9704d7e75b75b19e Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 14:18:02 +0000 Subject: [PATCH 200/246] Merged PR 881: add hpa for workflow add hpa for workflow now workflow autoscales using HPA in staging and prod when it passes 50% of CPU utilization - it will scale up to 50 replicas - modify helm chart ==> v2beta1/HorizontalPodAutoscaler | NAME | REFERENCE | TARGETS | MINPODS | MAXPODS | REPLICAS | AGE | |------------------------------|-------------------------------------|---------|---------|---------|----------|------| | workflow-v010-prod-hpa | Deployment/workflow-v010-prod | 0%/50% | 1 | 50 | 1 | 18m | solved: #133014 --- .../workflow/envs/workflow-prod/values.yaml | 5 +++ .../envs/workflow-staging/values.yaml | 5 +++ charts/workflow/templates/workflow-hpa.yaml | 34 +++++++++++++++++++ charts/workflow/values.yaml | 5 +++ 4 files changed, 49 insertions(+) create mode 100644 charts/workflow/templates/workflow-hpa.yaml diff --git a/charts/workflow/envs/workflow-prod/values.yaml b/charts/workflow/envs/workflow-prod/values.yaml index d8f0f287..79e382a1 100644 --- a/charts/workflow/envs/workflow-prod/values.yaml +++ b/charts/workflow/envs/workflow-prod/values.yaml @@ -15,3 +15,8 @@ exports: limits: cpu: 1000m memory: 140Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/workflow/envs/workflow-staging/values.yaml b/charts/workflow/envs/workflow-staging/values.yaml index 14d57289..9bc915ae 100644 --- a/charts/workflow/envs/workflow-staging/values.yaml +++ b/charts/workflow/envs/workflow-staging/values.yaml @@ -15,3 +15,8 @@ exports: limits: cpu: 1000m memory: 140Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/workflow/templates/workflow-hpa.yaml b/charts/workflow/templates/workflow-hpa.yaml new file mode 100644 index 00000000..02ec80a1 --- /dev/null +++ b/charts/workflow/templates/workflow-hpa.yaml @@ -0,0 +1,34 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ +################################################################################################### +# Workflow HPA Resrouce Metrics (CPU utilization threshold) +################################################################################################### +{{- if .Values.autoscaling.enabled }} +{{- $fullname := include "workflow.fullname" . | replace "." "" }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullname }}-hpa + labels: + app.kubernetes.io/name: {{ include "workflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "workflow.chart" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index ef27c6bf..157179d4 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -55,3 +55,8 @@ tags: prod: false qa: false staging: false +autoscaling: + enabled: false + maxReplicas: + minReplicas: + targetCPUUtilizationPercentage: From 76d70ded787946f041b91dd1b26dbdd8623dad8d Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 14:19:01 +0000 Subject: [PATCH 201/246] Merged PR 882: add hpa for ingestion add hpa for ingestion now ingestion autoscales using HPA in staging and prod when it passes 50% of CPU utilization - it will scale up to 10 replicas - modify helm chart ==> v2beta1/HorizontalPodAutoscaler | NAME | REFERENCE | TARGETS | MINPODS | MAXPODS | REPLICAS | AGE | |------------------------------|-------------------------------------|---------|---------|---------|----------|------| | ingestion-v010-prod-hpa | Deployment/ingestion-v010-prod | 2%/50% | 1 | 10 | 1 | 14m | solved: #133016 --- .../ingestion/envs/ingestion-prod/values.yaml | 5 +++ .../envs/ingestion-staging/values.yaml | 5 +++ charts/ingestion/templates/ingestion-hpa.yaml | 34 +++++++++++++++++++ charts/ingestion/values.yaml | 5 +++ 4 files changed, 49 insertions(+) create mode 100644 charts/ingestion/templates/ingestion-hpa.yaml diff --git a/charts/ingestion/envs/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml index 06575b29..eb7c3936 100644 --- a/charts/ingestion/envs/ingestion-prod/values.yaml +++ b/charts/ingestion/envs/ingestion-prod/values.yaml @@ -17,3 +17,8 @@ exports: limits: cpu: 300m memory: 800Mi + autoscaling: + enabled: true + maxReplicas: 10 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/ingestion/envs/ingestion-staging/values.yaml b/charts/ingestion/envs/ingestion-staging/values.yaml index 3b3cb756..731b4765 100644 --- a/charts/ingestion/envs/ingestion-staging/values.yaml +++ b/charts/ingestion/envs/ingestion-staging/values.yaml @@ -18,3 +18,8 @@ exports: limits: cpu: 300m memory: 800Mi + autoscaling: + enabled: true + maxReplicas: 10 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/ingestion/templates/ingestion-hpa.yaml b/charts/ingestion/templates/ingestion-hpa.yaml new file mode 100644 index 00000000..2a21c8d7 --- /dev/null +++ b/charts/ingestion/templates/ingestion-hpa.yaml @@ -0,0 +1,34 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ +################################################################################################### +# Ingestion HPA Resrouce Metrics (CPU utilization threshold) +################################################################################################### +{{- if .Values.autoscaling.enabled }} +{{- $fullname := include "ingestion.fullname" . | replace "." "" }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullname }}-hpa + labels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "ingestion.chart" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index cc6b14ee..992ad10f 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -31,3 +31,8 @@ tags: qa: false staging: false current: false +autoscaling: + enabled: false + maxReplicas: + minReplicas: + targetCPUUtilizationPercentage: From 73ca62654e997e6f4b284966e023552ce6fedc5e Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 14:21:52 +0000 Subject: [PATCH 202/246] Merged PR 883: add hpa for package add hpa for package now package autoscales using HPA in staging and prod when it passes 50% of CPU utilization - it will scale up to 50 replicas - modify helm chart ==> v2beta1/HorizontalPodAutoscaler NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE package-v010-prod-hpa Deployment/package-v010-prod 1%/50% 1 50 1 2m7s solved: #133020 --- charts/package/envs/package-prod/values.yaml | 5 +++ .../package/envs/package-staging/values.yaml | 5 +++ charts/package/templates/package-hpa.yaml | 35 +++++++++++++++++++ charts/package/values.yaml | 5 +++ 4 files changed, 50 insertions(+) create mode 100644 charts/package/templates/package-hpa.yaml diff --git a/charts/package/envs/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml index b29a0cb7..99cbba6e 100644 --- a/charts/package/envs/package-prod/values.yaml +++ b/charts/package/envs/package-prod/values.yaml @@ -17,3 +17,8 @@ exports: limits: cpu: 180m memory: 140Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/package/envs/package-staging/values.yaml b/charts/package/envs/package-staging/values.yaml index ee3acef2..20520a6e 100644 --- a/charts/package/envs/package-staging/values.yaml +++ b/charts/package/envs/package-staging/values.yaml @@ -18,3 +18,8 @@ exports: limits: cpu: 180m memory: 140Mi + autoscaling: + enabled: true + maxReplicas: 50 + minReplicas: 1 + targetCPUUtilizationPercentage: 50 diff --git a/charts/package/templates/package-hpa.yaml b/charts/package/templates/package-hpa.yaml new file mode 100644 index 00000000..de87e323 --- /dev/null +++ b/charts/package/templates/package-hpa.yaml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package HPA Resrouce Metrics (CPU utilization threshold) +################################################################################################### +{{- if .Values.autoscaling.enabled }} +{{- $fullname := include "package.fullname" . | replace "." "" }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullname }}-hpa + labels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + helm.sh/chart: {{ include "package.chart" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 75c6bf44..9dbff564 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -32,3 +32,8 @@ tags: qa: false staging: false current: false +autoscaling: + enabled: false + maxReplicas: + minReplicas: + targetCPUUtilizationPercentage: From 8bdb22366a2cd2a1d34a35a7a0c6cb682cc7ea65 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 11 Nov 2019 15:05:52 +0000 Subject: [PATCH 203/246] Merged PR 871: add service endpoints add service endpoints for key vault and cosmosdb. Other services (redis cache and service bus) only support Service Endpoints in the premium tier. tweak resource limits to fit in single node cluster --- azuredeploy.json | 91 ++++++++++++++++++- charts/delivery/envs/delivery-dev/values.yaml | 2 +- .../envs/dronescheduler-dev/values.yaml | 2 +- .../ingestion/envs/ingestion-dev/values.yaml | 2 +- charts/package/envs/package-dev/values.yaml | 2 +- charts/workflow/envs/workflow-dev/values.yaml | 2 +- 6 files changed, 92 insertions(+), 9 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index a6006c16..65d161e1 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -338,8 +338,30 @@ "[variables('environmentSettings')[parameters('environmentName')].aksVnetName]" ], "properties": { - "addressPrefix": "[variables('aksClusterSubnetPrefix')]" - } + "addressPrefix": "[variables('aksClusterSubnetPrefix')]", + "serviceEndpoints": [ + { + "service": "Microsoft.KeyVault", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.AzureCosmosDB", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.Storage", + "locations": [ + "[resourceGroup().location]" + ] + } + ], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } } ] }, @@ -473,6 +495,9 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "apiVersion": "2016-03-31", "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "tags": { "displayName": "Delivery Cosmos Db", "app": "fabrikam-delivery", @@ -481,6 +506,12 @@ "properties": { "name": "[variables('environmentSettings')[parameters('environmentName')].deliveryCosmosDbName]", "databaseAccountOfferType": "Standard", + "isVirtualNetworkFilterEnabled": true, + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ], "locations": [ { "locationName": "[resourceGroup().location]", @@ -495,6 +526,9 @@ "kind": "MongoDB", "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "tags": { "displayName": "Package Cosmos Db", "app": "fabrikam-package", @@ -502,7 +536,13 @@ }, "properties": { "databaseAccountOfferType": "Standard", - "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]" + "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", + "isVirtualNetworkFilterEnabled": true, + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ] } }, { @@ -510,9 +550,18 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", "apiVersion": "2015-04-08", "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "properties": { "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", "databaseAccountOfferType": "Standard", + "isVirtualNetworkFilterEnabled": true, + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ], "locations": [ { "locationName": "[resourceGroup().location]", @@ -585,7 +634,8 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]" + "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]", + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" ], "tags": { "displayName": "Delivery Key Vault", @@ -598,6 +648,15 @@ "name": "standard" }, "tenantId": "[subscription().tenantId]", + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Deny", + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ] + }, "accessPolicies": [ { "tenantId": "[subscription().tenantId]", @@ -689,6 +748,9 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "tags": { "displayName": "DroneScheduler Key Vault", "app": "fabrikam-dronescheduler", @@ -700,6 +762,15 @@ "name": "standard" }, "tenantId": "[subscription().tenantId]", + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Deny", + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ] + }, "accessPolicies": [ { "tenantId": "[subscription().tenantId]", @@ -823,6 +894,9 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].workflowKeyVaultName]", "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + ], "tags": { "displayName": "Workflow Key Vault", "app": "fabrikam-workflow", @@ -834,6 +908,15 @@ "name": "standard" }, "tenantId": "[subscription().tenantId]", + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Deny", + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + } + ] + }, "accessPolicies": [ { "tenantId": "[subscription().tenantId]", diff --git a/charts/delivery/envs/delivery-dev/values.yaml b/charts/delivery/envs/delivery-dev/values.yaml index 6f56e4b7..78d9f7f9 100644 --- a/charts/delivery/envs/delivery-dev/values.yaml +++ b/charts/delivery/envs/delivery-dev/values.yaml @@ -13,7 +13,7 @@ exports: class: "nginx-dev" resources: requests: - cpu: 91m + cpu: 80m memory: 350Mi limits: cpu: 130m diff --git a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml index 20906d58..87346616 100644 --- a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml @@ -13,7 +13,7 @@ exports: class: "nginx-dev" resources: requests: - cpu: 115m + cpu: 105m memory: 350Mi limits: cpu: 175m diff --git a/charts/ingestion/envs/ingestion-dev/values.yaml b/charts/ingestion/envs/ingestion-dev/values.yaml index c66eb389..dfaa1ba2 100644 --- a/charts/ingestion/envs/ingestion-dev/values.yaml +++ b/charts/ingestion/envs/ingestion-dev/values.yaml @@ -13,7 +13,7 @@ exports: class: "nginx-dev" resources: requests: - cpu: 129m + cpu: 110m memory: 600Mi limits: cpu: 194m diff --git a/charts/package/envs/package-dev/values.yaml b/charts/package/envs/package-dev/values.yaml index dade913a..dddcf9c1 100644 --- a/charts/package/envs/package-dev/values.yaml +++ b/charts/package/envs/package-dev/values.yaml @@ -13,7 +13,7 @@ exports: class: "nginx-dev" resources: requests: - cpu: 78m + cpu: 70m memory: 100Mi limits: cpu: 117m diff --git a/charts/workflow/envs/workflow-dev/values.yaml b/charts/workflow/envs/workflow-dev/values.yaml index b18c5462..4a5c9e1c 100644 --- a/charts/workflow/envs/workflow-dev/values.yaml +++ b/charts/workflow/envs/workflow-dev/values.yaml @@ -10,7 +10,7 @@ exports: reason: "new dev deploy" resources: requests: - cpu: 442m + cpu: 400m memory: 100Mi limits: cpu: 664m From 76a9b3f5bfffe3d32a11e5f43deb80d227e18d2e Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 11 Nov 2019 17:21:10 +0000 Subject: [PATCH 204/246] Merged PR 884: CA ### prep quotas in staging and prod for CA now resource quotas will allow greater CPU/mem utilization and allow more pods to be created taking into account CA max nodes is 50 Standard_D2_v2 Name: staging Resource Used Hard -------- ---- ---- cpu 985m 50 memory 1500Mi 200Gi pods 5 250 Name: staging Resource Used Hard -------- ---- ---- cpu 985m 50 memory 1500Mi 200Gi pods 5 250 prerequisite: #139661 ### enable Cluster Autoscaling now the drone delivery AKS cluster will start auto scaling when running out of capacity. It is only when choosing the multi-env path - min nodes 3 | max nodes 50 - enable vmss - upgrade api solved: #139661 --- azuredeploy.json | 18 +++++++++++++++++- k8s/k8s-resource-quotas-qa-stg-prod.yaml | 12 ++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 65d161e1..1fa50698 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -154,6 +154,9 @@ "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "aksMinCount": null, + "aksMaxCount": null, + "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", @@ -179,6 +182,9 @@ }, "qa": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "aksMinCount": null, + "aksMaxCount": null, + "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", @@ -204,6 +210,9 @@ }, "staging": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "aksMinCount": 3, + "aksMaxCount": 50, + "aksEnableAutoScaling": true, "acrName": "[variables('acrName')]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", @@ -229,6 +238,9 @@ }, "prod": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", + "aksMinCount": 3, + "aksMaxCount": 50, + "aksEnableAutoScaling": true, "acrName": "[concat(parameters('environmentName'),variables('acrName'))]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", @@ -368,7 +380,7 @@ { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", - "apiVersion": "2018-03-31", + "apiVersion": "2019-02-01", "location": "[resourceGroup().location]", "tags": { "environment": "shared cluster" @@ -383,6 +395,10 @@ { "name": "agentpool", "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "minCount": "[variables('environmentSettings')[parameters('environmentName')].aksMinCount]", + "maxCount": "[variables('environmentSettings')[parameters('environmentName')].aksMaxCount]", + "enableAutoScaling": "[variables('environmentSettings')[parameters('environmentName')].aksEnableAutoScaling]", + "type": "VirtualMachineScaleSets", "count": "[variables('environmentSettings')[parameters('environmentName')].agentCount]", "vmSize": "[variables('environmentSettings')[parameters('environmentName')].agentVMSize]", "osType": "[parameters('osType')]", diff --git a/k8s/k8s-resource-quotas-qa-stg-prod.yaml b/k8s/k8s-resource-quotas-qa-stg-prod.yaml index c0e65c95..3e90cbd2 100644 --- a/k8s/k8s-resource-quotas-qa-stg-prod.yaml +++ b/k8s/k8s-resource-quotas-qa-stg-prod.yaml @@ -23,9 +23,9 @@ metadata: namespace: backend-staging spec: hard: - cpu: "2" - memory: 8Gi - pods: "10" + cpu: "50" + memory: 200Gi + pods: "250" --- apiVersion: v1 kind: ResourceQuota @@ -34,6 +34,6 @@ metadata: namespace: backend spec: hard: - cpu: "2" - memory: 8Gi - pods: "10" + cpu: "50" + memory: 200Gi + pods: "250" From f478e542307bd7b74d4d9c2e2b4a8118bf33fc35 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Mon, 25 Nov 2019 13:03:59 +0000 Subject: [PATCH 205/246] Merged PR 886: Use appgateway ingress controller for accessing services * Adds deployment for app gateway, with an additional public ip address and subnet. * Install ingress controller * update templates to use the new ingress controller * update deployment instructions as needed. Pending: * Configure app gateway parameters * Ensure deployment works with CI --- azuredeploy-prereqs.json | 24 ++ azuredeploy.json | 292 +++++++++++++++--- .../delivery/templates/delivery-deploy.yaml | 6 +- .../delivery/templates/delivery-ingress.yaml | 9 +- .../ingestion/templates/ingestion-deploy.yaml | 3 + .../templates/ingestion-ingress.yaml | 9 +- deployment.md | 43 ++- deploymentCICD.md | 53 +++- 8 files changed, 363 insertions(+), 76 deletions(-) diff --git a/azuredeploy-prereqs.json b/azuredeploy-prereqs.json index 65d57a0d..960d4cf6 100644 --- a/azuredeploy-prereqs.json +++ b/azuredeploy-prereqs.json @@ -28,24 +28,28 @@ "deliveryIdName": "dev-d", "workflowIdName": "dev-wf", "droneSchedulerIdName": "dev-ds", + "appGatewayControllerIdName": "dev-ag", "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" }, "qa": { "deliveryIdName": "qa-d", "workflowIdName": "qa-wf", "droneSchedulerIdName": "qa-ds", + "appGatewayControllerIdName": "qa-ag", "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" }, "staging": { "deliveryIdName": "staging-d", "workflowIdName": "staging-wf", "droneSchedulerIdName": "staging-ds", + "appGatewayControllerIdName": "staging-ag", "acrResourceGroupName": "[variables('acrResourceGroupNamePrefix')]" }, "prod": { "deliveryIdName": "prod-d", "workflowIdName": "prod-wf", "droneSchedulerIdName": "prod-ds", + "appGatewayControllerIdName": "prod-ag", "acrResourceGroupName": "[concat(variables('acrResourceGroupNamePrefix'),'-',parameters('environmentName'))]" } } @@ -121,6 +125,18 @@ "app": "fabrikam-dronescheduler", "[parameters('environmentName')]": true } + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewayControllerIdName]", + "apiVersion": "2015-08-31-preview", + "location": "[parameters('resourceGroupLocation')]", + "tags": { + "displayName": "app gateway controller managed identity", + "what": "rbac", + "reason": "aad-pod-identity", + "[parameters('environmentName')]": true + } } ], "outputs": { @@ -148,6 +164,14 @@ "value": "[concat(subscription().id, '/resourceGroups/',parameters('resourceGroupName'),'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',variables('environmentSettings')[parameters('environmentName')].workflowIdName)]", "type": "string" }, + "appGatewayControllerIdName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].appGatewayControllerIdName]", + "type": "string" + }, + "appGatewayControllerPrincipalResourceId": { + "value": "[concat(subscription().id, '/resourceGroups/',parameters('resourceGroupName'),'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',variables('environmentSettings')[parameters('environmentName')].appGatewayControllerIdName)]", + "type": "string" + }, "acrResourceGroupName": { "value": "[variables('environmentSettings')[parameters('environmentName')].acrResourceGroupName]", "type": "string" diff --git a/azuredeploy.json b/azuredeploy.json index 1fa50698..997cd1f5 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -54,6 +54,18 @@ }, "type": "string" }, + "appGatewayControllerIdName": { + "metadata": { + "description": "Name of the app gateway controller managed identity" + }, + "type": "string" + }, + "appGatewayControllerPrincipalId": { + "metadata": { + "description": "Principal id for the app gateway controller managed identity" + }, + "type": "string" + }, "sshRSAPublicKey": { "type": "string", "metadata": { @@ -139,16 +151,18 @@ "acrNamePrefix": "acr", "aiNamePrefix": "ai", "acrName": "[uniqueString(variables('acrNamePrefix'),resourceGroup().id)]", + "appGatewayNamePrefix": "appg", "aksVnetAddressPrefix": "10.10.0.0/16", "aksClusterSubnetPrefix": "10.10.0.0/21", + "appGatewaySubnetPrefix": "10.10.8.0/24", "aksVnetNamePrefix": "vnet", "aksClusterSubnetNamePrefix": "subnet", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", "managedIdentityOperatorRoleObjectId": "f1a07417-d97a-45cb-824c-7a7467783830", - "networkContributorRoleObjectId": "4d97b98b-1d4f-4787-a291-c67834d212e7", + "contributorRoleObjectId": "b24988ac-6180-42a0-ab88-20f7382dd24c", "readerRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('readerRoleObjectId'))]", "managedIdentityOperatorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('managedIdentityOperatorRoleObjectId'))]", - "networkContributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('networkContributorRoleObjectId'))]", + "contributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('contributorRoleObjectId'))]", "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", "nestedACRDeploymentName": "[concat('azuredeploy-acr-',parameters('acrResourceGroupName'),parameters('environmentName'))]", "environmentSettings": { @@ -158,8 +172,12 @@ "aksMaxCount": null, "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", + "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -186,8 +204,12 @@ "aksMaxCount": null, "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", + "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -214,8 +236,12 @@ "aksMaxCount": 50, "aksEnableAutoScaling": true, "acrName": "[variables('acrName')]", + "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -242,8 +268,12 @@ "aksMaxCount": 50, "aksEnableAutoScaling": true, "acrName": "[concat(parameters('environmentName'),variables('acrName'))]", + "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", "deliveryRedisStorageName": "[variables('deliveryRedisStorageName')]", "deliveryRedisStorageId": "[resourceId('Microsoft.Storage/storageAccounts',variables('deliveryRedisStorageName'))]", @@ -333,49 +363,176 @@ "type": "Microsoft.Network/virtualNetworks", "apiVersion": "2018-06-01", "location": "[resourceGroup().location]", + "tags": { + "displayName": "Vnet for the aks cluster and the app gateways", + "environment": "shared network" + }, "properties": { "addressSpace": { "addressPrefixes": [ "[variables('aksVnetAddressPrefix')]" ] + }, + "subnets": [ + { + "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", + "tags": { + "displayName": "Subnet for the aks cluster", + "environment": "shared network" + }, + "properties": { + "addressPrefix": "[variables('aksClusterSubnetPrefix')]", + "serviceEndpoints": [ + { + "service": "Microsoft.KeyVault", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.AzureCosmosDB", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.Storage", + "locations": [ + "[resourceGroup().location]" + ] + } + ], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName]", + "tags": { + "displayName": "Subnet for the app gateways", + "environment": "shared network" + }, + "properties": { + "addressPrefix": "[variables('appGatewaySubnetPrefix')]" + } + } + ] + } + }, + { + "apiVersion":"2018-08-01", + "type":"Microsoft.Network/publicIPAddresses", + "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName]", + "location": "[resourceGroup().location]", + "tags": { + "displayName": "Public IP address for the application gateway", + "environment": "[parameters('environmentName')]" + }, + "sku": { + "name": "Standard" + }, + "properties": { + "publicIPAllocationMethod": "Static", + "dnsSettings": { + "domainNameLabel": "[variables('environmentSettings')[parameters('environmentName')].appGatewayPublicDnsName]" } + } + }, + { + "apiVersion": "2018-08-01", + "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewayName]", + "type": "Microsoft.Network/applicationGateways", + "location": "[resourceGroup().location]", + "tags": { + "displayName": "Application gateway for the app gateway ingress", + "environment": "[parameters('environmentName')]" }, - "resources": [ - { - "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", - "type": "subnets", - "apiVersion": "2018-06-01", - "location": "[resourceGroup().location]", - "dependsOn": [ - "[variables('environmentSettings')[parameters('environmentName')].aksVnetName]" - ], - "properties": { - "addressPrefix": "[variables('aksClusterSubnetPrefix')]", - "serviceEndpoints": [ - { - "service": "Microsoft.KeyVault", - "locations": [ - "[resourceGroup().location]" - ] + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName)]" + ], + "properties": { + "sku": { + "name": "Standard_v2", + "tier": "Standard_v2" + }, + "autoscaleConfiguration": { + "minCapacity": 2 + }, + "gatewayIPConfigurations": [ + { + "name": "appGatewayIpConfig", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName)]" + } + } + } + ], + "frontendIPConfigurations": [ + { + "name": "appGatewayFrontendIP", + "properties": { + "PublicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName)]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "defaultAddressPool" + } + ], + "backendHttpSettingsCollection": [ + { + "name": "defaultBackendHttpSettings", + "properties": { + "Port": "80", + "Protocol": "Http" + } + } + ], + "httpListeners": [ + { + "name": "defaultHttpListener", + "properties": { + "FrontendIpConfiguration": { + "Id": "[resourceId('Microsoft.Network/applicationGateways/frontendIPConfigurations', variables('environmentSettings')[parameters('environmentName')].appGatewayName, 'appGatewayFrontendIP')]" }, - { - "service": "Microsoft.AzureCosmosDB", - "locations": [ - "[resourceGroup().location]" - ] + "FrontendPort": { + "Id": "[resourceId('Microsoft.Network/applicationGateways/frontendPorts', variables('environmentSettings')[parameters('environmentName')].appGatewayName, 'http')]" }, - { - "service": "Microsoft.Storage", - "locations": [ - "[resourceGroup().location]" - ] + "Protocol": "Http" + } + } + ], + "requestRoutingRules": [ + { + "Name": "default", + "properties": { + "RuleType": "Basic", + "httpListener": { + "id": "[resourceId('Microsoft.Network/applicationGateways/httpListeners', variables('environmentSettings')[parameters('environmentName')].appGatewayName, 'defaultHttpListener')]" + }, + "backendAddressPool": { + "id": "[resourceId('Microsoft.Network/applicationGateways/backendAddressPools', variables('environmentSettings')[parameters('environmentName')].appGatewayName, 'defaultAddressPool')]" + }, + "backendHttpSettings": { + "id": "[resourceId('Microsoft.Network/applicationGateways/backendHttpSettingsCollection', variables('environmentSettings')[parameters('environmentName')].appGatewayName, 'defaultBackendHttpSettings')]" } - ], - "privateEndpointNetworkPolicies": "Enabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - } - ] + } + } + ], + "frontendPorts": [ + { + "name": "http", + "properties": { + "Port": "80" + } + } + ] + } }, { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", @@ -386,7 +543,7 @@ "environment": "shared cluster" }, "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "properties": { "kubernetesVersion": "[parameters('kubernetesVersion')]", @@ -512,7 +669,7 @@ "apiVersion": "2016-03-31", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Delivery Cosmos Db", @@ -543,7 +700,7 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Package Cosmos Db", @@ -567,7 +724,7 @@ "apiVersion": "2015-04-08", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "properties": { "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", @@ -651,7 +808,7 @@ "location": "[resourceGroup().location]", "dependsOn": [ "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]", - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Delivery Key Vault", @@ -765,7 +922,7 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "DroneScheduler Key Vault", @@ -911,7 +1068,7 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Workflow Key Vault", @@ -1138,6 +1295,49 @@ "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]", "principalId": "[parameters('servicePrincipalId')]" } + }, + { + "type": "Microsoft.Network/applicationGateways/providers/roleAssignments", + "name": "[concat(variables('environmentSettings')[parameters('environmentName')].appGatewayName, '/Microsoft.Authorization/', guid(concat('msi-appgatewaycontroller', 'appgateway'), resourceGroup().id, parameters('environmentName')))]", + "apiVersion": "2017-05-01", + "comments": "Grant the App gateway controller access to the app gateway", + "tags": { + "displayName": "App gateway controller RBAC Contributor", + "what": "rbac", + "to": "pod", + "identity-type": "msi", + "access": "appgateway", + "reason": "aad-pod-identity", + "flex-vol": "no" + }, + "properties": { + "roleDefinitionId": "[variables('contributorRoleId')]", + "principalId": "[parameters('appGatewayControllerPrincipalId')]", + "scope": "[resourceId('Microsoft.Network/applicationGateways', variables('environmentSettings')[parameters('environmentName')].appGatewayName)]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/applicationGateways', variables('environmentSettings')[parameters('environmentName')].appGatewayName)]" + ] + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "name": "[guid(concat('msi-appgatewaycontroller', 'resourcegroup', parameters('environmentName')), resourceGroup().id)]", + "apiVersion": "2017-05-01", + "comments": "Grant the App gateway controller access to the AKS cluster", + "tags": { + "displayName": "App gateway controller RBAC Reader", + "what": "rbac", + "to": "pod", + "identity-type": "msi", + "access": "appgateway", + "reason": "aad-pod-identity", + "flex-vol": "no" + }, + "properties": { + "roleDefinitionId": "[variables('readerRoleId')]", + "principalId": "[parameters('appGatewayControllerPrincipalId')]", + "scope": "[resourceGroup().id]" + } } ], "outputs": { @@ -1196,6 +1396,14 @@ "acrDeploymentName": { "value": "[variables('nestedACRDeploymentName')]", "type": "string" + }, + "appGatewayName": { + "value": "[variables('environmentSettings')[parameters('environmentName')].appGatewayName]", + "type": "string" + }, + "appGatewayPublicIpFqdn": { + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName)).dnsSettings.fqdn]", + "type": "string" } } } diff --git a/charts/delivery/templates/delivery-deploy.yaml b/charts/delivery/templates/delivery-deploy.yaml index 0e04a84f..f5255875 100644 --- a/charts/delivery/templates/delivery-deploy.yaml +++ b/charts/delivery/templates/delivery-deploy.yaml @@ -96,6 +96,6 @@ spec: value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 - ports: - - name: service - containerPort: 8080 + ports: + - name: service + containerPort: 8080 diff --git a/charts/delivery/templates/delivery-ingress.yaml b/charts/delivery/templates/delivery-ingress.yaml index f90a95e9..af80b627 100644 --- a/charts/delivery/templates/delivery-ingress.yaml +++ b/charts/delivery/templates/delivery-ingress.yaml @@ -15,8 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-ingress annotations: - kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} - nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries/public$1 + kubernetes.io/ingress.class: azure/application-gateway spec: {{- if .Values.ingress.tls }} tls: @@ -34,15 +33,15 @@ spec: http: paths: {{- if .path }} - - path: {{ printf "%s/%s/" .path $appversion }}api/deliveries(.*) + - path: {{ printf "%s/%s/" .path $appversion }}api/deliveries* {{- else }} - - path: {{ $defaultversionedpath }}api/deliveries(.*) + - path: {{ $defaultversionedpath }}api/deliveries* {{- end }} backend: serviceName: "{{ .serviceName }}-{{ $svcversion }}" servicePort: http {{- if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }}api/deliveries(.*) + - path: {{ default "/" .path }}api/deliveries* backend: serviceName: "{{ .serviceName }}" servicePort: http diff --git a/charts/ingestion/templates/ingestion-deploy.yaml b/charts/ingestion/templates/ingestion-deploy.yaml index f4bedcce..a7028f63 100644 --- a/charts/ingestion/templates/ingestion-deploy.yaml +++ b/charts/ingestion/templates/ingestion-deploy.yaml @@ -110,3 +110,6 @@ spec: value: {{ default "error" .Values.telemetry.level | quote }} - name: CONTAINER_NAME value: *ingestion-container_name + ports: + - name: service + containerPort: 80 diff --git a/charts/ingestion/templates/ingestion-ingress.yaml b/charts/ingestion/templates/ingestion-ingress.yaml index 8581606b..9f1cedfb 100644 --- a/charts/ingestion/templates/ingestion-ingress.yaml +++ b/charts/ingestion/templates/ingestion-ingress.yaml @@ -15,8 +15,7 @@ kind: Ingress metadata: name: {{ $relname }}-ingress annotations: - kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} - nginx.ingress.kubernetes.io/rewrite-target: /api/deliveryrequests$1 + kubernetes.io/ingress.class: azure/application-gateway spec: {{- if .Values.ingress.tls }} tls: @@ -34,15 +33,15 @@ spec: http: paths: {{ if .path }} - - path: {{ printf "%s/%s/" .path $appversion }}api/deliveryrequests(.*) + - path: {{ printf "%s/%s/" .path $appversion }}api/deliveryrequests* {{ else }} - - path: {{ $defaultversionedpath }}api/deliveryrequests(.*) + - path: {{ $defaultversionedpath }}api/deliveryrequests* {{ end }} backend: serviceName: "{{ .serviceName }}-{{ $svcversion }}" servicePort: http {{ if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }}api/deliveryrequests(.*) + - path: {{ default "/" .path }}api/deliveryrequests* backend: serviceName: "{{ .serviceName }}" servicePort: http diff --git a/deployment.md b/deployment.md index d0d16ec8..69bca717 100644 --- a/deployment.md +++ b/deployment.md @@ -87,12 +87,15 @@ export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $ export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) && \ export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowIdName.value -o tsv) && \ export WORKFLOW_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) && \ +export GATEWAY_CONTROLLER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.appGatewayControllerIdName.value -o tsv) && \ +export GATEWAY_CONTROLLER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $GATEWAY_CONTROLLER_ID_NAME --query principalId -o tsv) && \ export RESOURCE_GROUP_ACR=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.acrResourceGroupName.value -o tsv) # Wait for AAD propagation until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id ${GATEWAY_CONTROLLER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done # Export the kubernetes cluster version export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) @@ -110,6 +113,8 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template- droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \ workflowIdName=${WORKFLOW_ID_NAME} \ workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} \ + appGatewayControllerIdName=${GATEWAY_CONTROLLER_ID_NAME} \ + appGatewayControllerPrincipalId=${GATEWAY_CONTROLLER_ID_PRINCIPAL_ID} \ acrResourceGroupName=${RESOURCE_GROUP_ACR} \ acrResourceGroupLocation=$LOCATION ``` @@ -120,6 +125,7 @@ Get outputs from Azure Deploy export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) +export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) ``` Enable Azure Monitoring for the AKS cluster @@ -175,23 +181,42 @@ kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/maste kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` -## Deploy the ingress controller +## Deploy the ingress controllers ```bash -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress-dev --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-dev +# Deploy the AppGateway ingress controller +helm repo add application-gateway-kubernetes-ingress https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/ +helm repo update + +export GATEWAY_CONTROLLER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.appGatewayControllerPrincipalResourceId.value -o tsv) && \ +export GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $GATEWAY_CONTROLLER_ID_NAME --query clientId -o tsv) +export APP_GATEWAY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewayName.value -o tsv) +export APP_GATEWAY_PUBLIC_IP_FQDN=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewayPublicIpFqdn.value -o tsv) -# Obtain the load balancer ip address and assign a domain name -until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-dev-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done -export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest-dev" -export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) +helm install application-gateway-kubernetes-ingress/ingress-azure \ + --name ingress-azure-dev \ + --namespace ingress-controllers \ + --set appgw.name=$APP_GATEWAY_NAME \ + --set appgw.resourceGroup=$RESOURCE_GROUP \ + --set appgw.subscriptionId=$SUBSCRIPTION_ID \ + --set appgw.shared=false \ + --set kubernetes.watchNamespace=backend-dev \ + --set armAuth.type=aadPodIdentity \ + --set armAuth.identityResourceID=$GATEWAY_CONTROLLER_PRINCIPAL_RESOURCE_ID \ + --set armAuth.identityClientID=$GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID \ + --set rbac.enabled=true \ + --set verbosityLevel=3 \ + --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER + +# Deploy the ngnix ingress controller +helm install stable/nginx-ingress --name nginx-ingress-dev --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-dev --set controller.service.type=ClusterIP # Create a self-signed certificate for TLS +export EXTERNAL_INGEST_FQDN=$APP_GATEWAY_PUBLIC_IP_FQDN openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -out ingestion-ingress-tls.crt \ -keyout ingestion-ingress-tls.key \ - -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" + -subj "/CN=${APP_GATEWAY_PUBLIC_IP_FQDN}/O=fabrikam" ``` ## Setup cluster resource quota diff --git a/deploymentCICD.md b/deploymentCICD.md index af64df69..eb81061f 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -23,19 +23,22 @@ az deployment create \ resourceGroupLocation=$LOCATION \ environmentName=${env} -export {${ENV}_IDENTITIES_DEPLOYMENT_NAME,IDENTITIES_DEPLOYMENT_NAME}=$(az deployment show -n azuredeploy-adv-prereqs-adv-${env} --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ +export {${ENV}_IDENTITIES_DEPLOYMENT_NAME,IDENTITIES_DEPLOYMENT_NAME}=$(az deployment show -n azuredeploy-prereqs-${env} --query properties.outputs.identitiesDeploymentName.value -o tsv) && \ export {${ENV}_DELIVERY_ID_NAME,DELIVERY_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryIdName.value -o tsv) export DELIVERY_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query principalId -o tsv) export {${ENV}_DRONESCHEDULER_ID_NAME,DRONESCHEDULER_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerIdName.value -o tsv) export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) export {${ENV}_WORKFLOW_ID_NAME,WORKFLOW_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowIdName.value -o tsv) export WORKFLOW_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) +export {${ENV}_GATEWAY_CONTROLLER_ID_NAME,GATEWAY_CONTROLLER_ID_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.appGatewayControllerIdName.value -o tsv) && \ +export GATEWAY_CONTROLLER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $GATEWAY_CONTROLLER_ID_NAME --query principalId -o tsv) && \ export RESOURCE_GROUP_ACR=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.acrResourceGroupName.value -o tsv) # Wait for AAD propagation until az ad sp show --id $DELIVERY_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id $DRONESCHEDULER_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done until az ad sp show --id $WORKFLOW_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done +until az ad sp show --id $GATEWAY_CONTROLLER_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --template-file ${PROJECT_ROOT}/azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ @@ -48,6 +51,8 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa droneSchedulerIdName=$DRONESCHEDULER_ID_NAME \ droneSchedulerPrincipalId=$DRONESCHEDULER_ID_PRINCIPAL_ID \ workflowIdName=$WORKFLOW_ID_NAME \ + appGatewayControllerIdName=${GATEWAY_CONTROLLER_ID_NAME} \ + appGatewayControllerPrincipalId=${GATEWAY_CONTROLLER_ID_PRINCIPAL_ID} \ workflowPrincipalId=$WORKFLOW_ID_PRINCIPAL_ID \ acrResourceGroupName=${RESOURCE_GROUP_ACR} \ acrResourceGroupLocation=$LOCATION \ @@ -65,6 +70,7 @@ Get outputs from Azure Deploy ```bash # Shared export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) && \ +export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) ``` Enable Azure Monitoring for Containers in the AKS cluster @@ -81,6 +87,7 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces +kubectl create namespace backend-dev && \ kubectl create namespace backend-qa && \ kubectl create namespace backend-staging && \ kubectl create namespace backend @@ -200,27 +207,49 @@ export AZURE_DEVOPS_USER_ID=$(az devops user show --user ${AZURE_DEVEOPS_USER} - ### Build pipelines pre-requisites ```bash -# Create a self-signed certificate for TLS and public ip addresses -export RESOURCE_GROUP_NODE=$(az aks show -g $RESOURCE_GROUP -n $CLUSTER_NAME --query "nodeResourceGroup" -o tsv) && \ -export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest" +# Deploy the AppGateway ingress controller +helm repo add application-gateway-kubernetes-ingress https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/ +helm repo update for env in dev qa staging prod;do ENV=${env^^} -az network public-ip create --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --dns-name ${EXTERNAL_INGEST_DNS_NAME}-${env} --allocation-method static -g $RESOURCE_GROUP_NODE -EXTERNAL_INGEST_FQDN=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "dnsSettings.fqdn" -g $RESOURCE_GROUP_NODE --output tsv) -export ${ENV}_EXTERNAL_INGEST_FQDN=${EXTERNAL_INGEST_FQDN} -export ${ENV}_INGRESS_LOAD_BALANCER_IP=$(az network public-ip show --name ${EXTERNAL_INGEST_DNS_NAME}-${env}-pip --query "ipAddress" -g $RESOURCE_GROUP_NODE --output tsv) - -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress-${env} --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-${env} --set controller.service.loadBalancerIP=${ENV}_INGRESS_LOAD_BALANCER_IP - +export IDENTITIES_DEPLOYMENT_NAME_VARIABLE=${ENV}_IDENTITIES_DEPLOYMENT_NAME +export GATEWAY_CONTROLLER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n ${!IDENTITIES_DEPLOYMENT_NAME_VARIABLE} --query properties.outputs.appGatewayControllerPrincipalResourceId.value -o tsv) && \ +export GATEWAY_CONTROLLER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n ${!IDENTITIES_DEPLOYMENT_NAME_VARIABLE} --query properties.outputs.appGatewayControllerIdName.value -o tsv) && \ +export GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $GATEWAY_CONTROLLER_ID_NAME --query clientId -o tsv) + +# Deploy the App Gateway ingress controller +export APP_GATEWAY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appGatewayName.value -o tsv) +export {${ENV}_APP_GATEWAY_PUBLIC_IP_FQDN,APP_GATEWAY_PUBLIC_IP_FQDN}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appGatewayPublicIpFqdn.value -o tsv) +export ENV_NAMESPACE=$([ $env == 'prod' ] && echo 'backend' || echo "backend-$env") + +helm install application-gateway-kubernetes-ingress/ingress-azure \ + --name ingress-azure-${env} \ + --namespace ingress-controllers \ + --set appgw.name=$APP_GATEWAY_NAME \ + --set appgw.resourceGroup=$RESOURCE_GROUP \ + --set appgw.subscriptionId=$SUBSCRIPTION_ID \ + --set appgw.shared=false \ + --set kubernetes.watchNamespace=$ENV_NAMESPACE \ + --set armAuth.type=aadPodIdentity \ + --set armAuth.identityResourceID=$GATEWAY_CONTROLLER_PRINCIPAL_RESOURCE_ID \ + --set armAuth.identityClientID=$GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID \ + --set rbac.enabled=true \ + --set verbosityLevel=3 \ + --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER + +# Create a self-signed certificate for TLS for the environment +export {${ENV}_EXTERNAL_INGEST_FQDN,EXTERNAL_INGEST_FQDN}=$APP_GATEWAY_PUBLIC_IP_FQDN openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -out ingestion-ingress-tls-${env}.crt \ -keyout ingestion-ingress-tls-${env}.key \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" export "${ENV}_INGRESS_TLS_SECRET_CERT=$(echo $(cat ingestion-ingress-tls-${env}.crt) | tr '\n' "\\n")" export "${ENV}_INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls-${env}.key) | tr '\n' "\\n")" + +# Deploy the ngnix ingress controllers +helm install stable/nginx-ingress --name nginx-ingress-${env} --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-${env} --set controller.service.type=ClusterIP done # export app paths From 417d9e3b0ede7de309cc290c1fca99e405e02600 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Wed, 27 Nov 2019 14:27:40 +0000 Subject: [PATCH 206/246] Merged PR 891: bug fix subnet tags --- azuredeploy.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 997cd1f5..a54997d2 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -376,10 +376,6 @@ "subnets": [ { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", - "tags": { - "displayName": "Subnet for the aks cluster", - "environment": "shared network" - }, "properties": { "addressPrefix": "[variables('aksClusterSubnetPrefix')]", "serviceEndpoints": [ @@ -408,10 +404,6 @@ }, { "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName]", - "tags": { - "displayName": "Subnet for the app gateways", - "environment": "shared network" - }, "properties": { "addressPrefix": "[variables('appGatewaySubnetPrefix')]" } From 3f3b9d899e7a3adaade3ea792bb1d1f30e9ca00e Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Wed, 27 Nov 2019 14:34:42 +0000 Subject: [PATCH 207/246] Merged PR 890: enable monitoring via arm --- azuredeploy.json | 47 ++++++++++++++++++++++++++++++++++++++++++----- deployment.md | 5 ----- deploymentCICD.md | 5 ----- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index a54997d2..5a36df70 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -165,6 +165,7 @@ "contributorRoleId": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/', variables('contributorRoleObjectId'))]", "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", "nestedACRDeploymentName": "[concat('azuredeploy-acr-',parameters('acrResourceGroupName'),parameters('environmentName'))]", + "aksLogAnalyticsNamePrefix": "logsAnalytics", "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -196,7 +197,10 @@ "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey", "agentCount": 1, - "agentVMSize": "Standard_D2_v2" + "agentVMSize": "Standard_D2_v2", + "workspaceName": "[concat(parameters('environmentName'),'-la-', uniqueString(variables('aksLogAnalyticsNamePrefix'), resourceGroup().id))]", + "workspaceSku": "Free", + "workspaceRetentionInDays": null }, "qa": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -228,7 +232,10 @@ "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey", "agentCount": 3, - "agentVMSize": "Standard_D2_v2" + "agentVMSize": "Standard_D2_v2", + "workspaceName": "[concat(parameters('environmentName'),'-la-', uniqueString(variables('aksLogAnalyticsNamePrefix'), resourceGroup().id))]", + "workspaceSku": "Free", + "workspaceRetentionInDays": null }, "staging": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -260,7 +267,10 @@ "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey", "agentCount": 3, - "agentVMSize": "Standard_D2_v2" + "agentVMSize": "Standard_D2_v2", + "workspaceName": "[concat(parameters('environmentName'),'-la-', uniqueString(variables('aksLogAnalyticsNamePrefix'), resourceGroup().id))]", + "workspaceSku": "PerGB2018", + "workspaceRetentionInDays": 730 }, "prod": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -292,7 +302,10 @@ "workflowKeyVaultName": "[concat(parameters('environmentName'),'-wf-',uniqueString(resourceGroup().id))]", "workflowServiceAccessKey": "WorkflowServiceAccessKey", "agentCount": 3, - "agentVMSize": "Standard_D2_v2" + "agentVMSize": "Standard_D2_v2", + "workspaceName": "[concat(parameters('environmentName'),'-la-', uniqueString(variables('aksLogAnalyticsNamePrefix'), resourceGroup().id))]", + "workspaceSku": "PerGB2018", + "workspaceRetentionInDays": 730 } } }, @@ -526,6 +539,21 @@ ] } }, + { + "type": "Microsoft.OperationalInsights/workspaces", + "name": "[variables('environmentSettings')[parameters('environmentName')].workspaceName]", + "apiVersion": "2015-11-01-preview", + "location": "[resourceGroup().location]", + "properties": { + "retentionInDays": "[variables('environmentSettings')[parameters('environmentName')].workspaceRetentionInDays]", + "sku": { + "Name": "[variables('environmentSettings')[parameters('environmentName')].workspaceSku]" + }, + "features": { + "searchVersion": 1 + } + } + }, { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", @@ -535,7 +563,8 @@ "environment": "shared cluster" }, "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.OperationalInsights/workspaces', variables('environmentSettings')[parameters('environmentName')].workspaceName)]" ], "properties": { "kubernetesVersion": "[parameters('kubernetesVersion')]", @@ -569,6 +598,14 @@ "clientId": "[parameters('servicePrincipalClientId')]", "secret": "[parameters('servicePrincipalClientSecret')]" }, + "addonProfiles": { + "omsagent": { + "config": { + "logAnalyticsWorkspaceResourceID": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('environmentSettings')[parameters('environmentName')].workspaceName)]" + }, + "enabled": true + } + }, "enableRBAC": true, "networkProfile": { "networkPlugin": "azure", diff --git a/deployment.md b/deployment.md index 69bca717..027b7e7c 100644 --- a/deployment.md +++ b/deployment.md @@ -128,11 +128,6 @@ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) ``` -Enable Azure Monitoring for the AKS cluster -```bash -az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -``` - Download kubectl and create a k8s namespace ```bash # Install kubectl diff --git a/deploymentCICD.md b/deploymentCICD.md index eb81061f..1c3eae92 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -73,11 +73,6 @@ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) ``` -Enable Azure Monitoring for Containers in the AKS cluster -```bash -az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME -``` - Download kubectl and create a k8s namespace ```bash # Install kubectl From accffc3d9101c02b3bd1fb6897daae42574a45a9 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Wed, 27 Nov 2019 11:34:12 -0700 Subject: [PATCH 208/246] workaround helm regression issue introduced in 2.15.0 --- deployment.md | 7 +++++-- deploymentCICD.md | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deployment.md b/deployment.md index 027b7e7c..398fbbc1 100644 --- a/deployment.md +++ b/deployment.md @@ -5,7 +5,6 @@ - Azure subscription - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) -- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) - [JQ](https://stedolan.github.io/jq/download/) > Note: in linux systems, it is possible to run the docker command without prefacing @@ -140,9 +139,13 @@ az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME kubectl create namespace backend-dev ``` -Setup Helm in the container +Setup Helm ```bash +# install helm client side +DESIRED_VERSION=v2.14.2;curl -L https://git.io/get_helm.sh | bash + +# setup tiller in your cluster kubectl apply -f $K8S/tiller-rbac.yaml helm init --service-account tiller ``` diff --git a/deploymentCICD.md b/deploymentCICD.md index 1c3eae92..1a2e7ec0 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -5,7 +5,6 @@ - Azure subscription - [Azure CLI 2.0.49 or later](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Azure DevOps account](https://azure.microsoft.com/services/devops) -- [Helm 2.12.3 or later](https://docs.helm.sh/using_helm/#installing-helm) - [Values from deployment instructions](./deployment.md) ## Infrastructure for dev, test, staging and production @@ -88,9 +87,13 @@ kubectl create namespace backend-staging && \ kubectl create namespace backend ``` -Setup Helm in the container +Setup Helm ```bash +# install helm client side +DESIRED_VERSION=v2.14.2;curl -L https://git.io/get_helm.sh | bash + +# setup tiller in your cluster kubectl apply -f $K8S/tiller-rbac.yaml helm init --service-account tiller ``` From 827cb7273fec610f95dfba7c595ccfee77ad4e02 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 2 Dec 2019 18:54:27 +0000 Subject: [PATCH 209/246] Merged PR 895: enable cluster network policy - enable cluster network policy with azure policies - deny all non whitelisted traffic in dev, qa, staging and prod - add deployment steps for deny all traffic between pods using network policies Verification: ``` / # wget -qO- --timeout 1 https://www.microsoft.com wget: download timed out ... / # wget -qO- --timeout 1 http://delivery.backend-dev wget: download timed out ``` solved: #146525, #146046 --- azuredeploy.json | 12 ++++++ deployment.md | 6 +++ deploymentCICD.md | 6 +++ ...-deny-all-non-whitelisted-traffic-dev.yaml | 15 ++++++++ ...l-non-whitelisted-traffic-qa-stg-prod.yaml | 37 +++++++++++++++++++ 5 files changed, 76 insertions(+) create mode 100644 k8s/k8s-deny-all-non-whitelisted-traffic-dev.yaml create mode 100644 k8s/k8s-deny-all-non-whitelisted-traffic-qa-stg-prod.yaml diff --git a/azuredeploy.json b/azuredeploy.json index 5a36df70..bc0b6df3 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -144,6 +144,17 @@ "metadata": { "description": "A value that indicates whether diagnostics should be saved to the specified storage account." } + }, + "networkPolicy": { + "type": "string", + "defaultValue": "azure", + "allowedValues": [ + "azure", + "calico" + ], + "metadata": { + "description": "Network policy used for configuring Kubernetes network profile." + } } }, "variables": { @@ -608,6 +619,7 @@ }, "enableRBAC": true, "networkProfile": { + "networkPolicy": "[parameters('networkPolicy')]", "networkPlugin": "azure", "serviceCidr": "10.2.0.0/24", "dnsServiceIP": "10.2.0.10", diff --git a/deployment.md b/deployment.md index 398fbbc1..d8ed5671 100644 --- a/deployment.md +++ b/deployment.md @@ -223,6 +223,12 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ kubectl apply -f $K8S/k8s-resource-quotas-dev.yaml ``` +## Deny all ingress and egress traffic + +```bash +kubectl apply -f $K8S/k8s-deny-all-non-whitelisted-traffic-dev.yaml +``` + ## Deploy the Delivery service Extract resource details from deployment diff --git a/deploymentCICD.md b/deploymentCICD.md index 1a2e7ec0..ba8d7de2 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -124,6 +124,12 @@ kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-fl kubectl apply -f $K8S/k8s-resource-quotas-dev.yaml -f $K8S/k8s-resource-quotas-qa-stg-prod.yaml ``` +## Deny all ingress and egress traffic + +```bash +kubectl apply -f $K8S/k8s-deny-all-non-whitelisted-traffic-dev.yaml -f $K8S/k8s-deny-all-non-whitelisted-traffic-qa-stg-prod.yaml +``` + ## Setup Azure DevOps ``` diff --git a/k8s/k8s-deny-all-non-whitelisted-traffic-dev.yaml b/k8s/k8s-deny-all-non-whitelisted-traffic-dev.yaml new file mode 100644 index 00000000..487ccf19 --- /dev/null +++ b/k8s/k8s-deny-all-non-whitelisted-traffic-dev.yaml @@ -0,0 +1,15 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dev-deny-all + namespace: backend-dev +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress diff --git a/k8s/k8s-deny-all-non-whitelisted-traffic-qa-stg-prod.yaml b/k8s/k8s-deny-all-non-whitelisted-traffic-qa-stg-prod.yaml new file mode 100644 index 00000000..4ade2795 --- /dev/null +++ b/k8s/k8s-deny-all-non-whitelisted-traffic-qa-stg-prod.yaml @@ -0,0 +1,37 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: qa-deny-all + namespace: backend-qa +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: staging-deny-all + namespace: backend-staging +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: prod-deny-all + namespace: backend +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress From ab0c3fdabb8617d6639617bfc9852835c39fd011 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Tue, 3 Dec 2019 13:24:32 +0000 Subject: [PATCH 210/246] Merged PR 892: Remove the nginx ingress from advanced --- charts/delivery/envs/delivery-dev/values.yaml | 2 - .../delivery/envs/delivery-prod/values.yaml | 2 - charts/delivery/envs/delivery-qa/values.yaml | 2 - .../envs/delivery-staging/values.yaml | 2 - .../templates/delivery-internal-ingress.yaml | 42 ------------------- .../envs/dronescheduler-dev/values.yaml | 2 - .../envs/dronescheduler-prod/values.yaml | 2 - .../envs/dronescheduler-qa/values.yaml | 2 - .../envs/dronescheduler-staging/values.yaml | 2 - .../dronescheduler-internal-ingress.yaml | 42 ------------------- .../ingestion/envs/ingestion-dev/values.yaml | 2 - .../ingestion/envs/ingestion-prod/values.yaml | 2 - .../ingestion/envs/ingestion-qa/values.yaml | 2 - .../envs/ingestion-staging/values.yaml | 2 - charts/package/envs/package-dev/values.yaml | 2 - charts/package/envs/package-prod/values.yaml | 2 - charts/package/envs/package-qa/values.yaml | 2 - .../package/envs/package-staging/values.yaml | 2 - .../templates/package-internal-ingress.yaml | 42 ------------------- deployment.md | 3 -- deploymentCICD.md | 2 - 21 files changed, 163 deletions(-) delete mode 100644 charts/delivery/templates/delivery-internal-ingress.yaml delete mode 100644 charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml delete mode 100644 charts/package/templates/package-internal-ingress.yaml diff --git a/charts/delivery/envs/delivery-dev/values.yaml b/charts/delivery/envs/delivery-dev/values.yaml index 78d9f7f9..24ea476c 100644 --- a/charts/delivery/envs/delivery-dev/values.yaml +++ b/charts/delivery/envs/delivery-dev/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new dev deploy" current: true - ingress: - class: "nginx-dev" resources: requests: cpu: 80m diff --git a/charts/delivery/envs/delivery-prod/values.yaml b/charts/delivery/envs/delivery-prod/values.yaml index 0c9dd8b9..ec646e66 100644 --- a/charts/delivery/envs/delivery-prod/values.yaml +++ b/charts/delivery/envs/delivery-prod/values.yaml @@ -8,8 +8,6 @@ exports: telemetry: level: "Error" reason: "new prod deploy" - ingress: - class: "nginx-prod" resources: requests: cpu: 100m diff --git a/charts/delivery/envs/delivery-qa/values.yaml b/charts/delivery/envs/delivery-qa/values.yaml index 668e722f..66ce0c20 100644 --- a/charts/delivery/envs/delivery-qa/values.yaml +++ b/charts/delivery/envs/delivery-qa/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new qa deploy" current: true - ingress: - class: "nginx-qa" resources: requests: cpu: 100m diff --git a/charts/delivery/envs/delivery-staging/values.yaml b/charts/delivery/envs/delivery-staging/values.yaml index 091823ba..b2cbaee4 100644 --- a/charts/delivery/envs/delivery-staging/values.yaml +++ b/charts/delivery/envs/delivery-staging/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new staging deploy" current: true - ingress: - class: "nginx-staging" resources: requests: cpu: 100m diff --git a/charts/delivery/templates/delivery-internal-ingress.yaml b/charts/delivery/templates/delivery-internal-ingress.yaml deleted file mode 100644 index 417b440b..00000000 --- a/charts/delivery/templates/delivery-internal-ingress.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ingress -################################################################################################### -{{- $svcversion := .Chart.AppVersion | replace "." "" }} -{{- $appversion := .Chart.AppVersion }} -{{- $defaultversionedpath := printf "/%s/" $appversion }} -{{- $relname := .Release.Name }} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $relname }}-internal-ingress - annotations: - kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} - nginx.ingress.kubernetes.io/rewrite-target: /api/deliveries$1 - nginx.ingress.kubernetes.io/configuration-snippet: | - internal; -spec: - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .name }} - http: - paths: - {{- if .path }} - - path: {{ printf "%s/%s/" .path $appversion }}api/internal/deliveries(.*) - {{- else }} - - path: {{ $defaultversionedpath }}api/internal/deliveries(.*) - {{- end }} - backend: - serviceName: "{{ .serviceName }}-{{ $svcversion }}" - servicePort: http - {{- if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }}api/internal/deliveries(.*) - backend: - serviceName: "{{ .serviceName }}" - servicePort: http - {{- end }} - {{ end }} diff --git a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml index 87346616..b70d3353 100644 --- a/charts/dronescheduler/envs/dronescheduler-dev/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new dev deploy" current: true - ingress: - class: "nginx-dev" resources: requests: cpu: 105m diff --git a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml index 621194b5..9cd19e8f 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/values.yaml @@ -8,8 +8,6 @@ exports: telemetry: level: "Error" reason: "new prod deploy" - ingress: - class: "nginx-prod" resources: requests: cpu: 130m diff --git a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml index 5f0b9786..f9c21ec7 100644 --- a/charts/dronescheduler/envs/dronescheduler-qa/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-qa/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new qa deploy" current: true - ingress: - class: "nginx-qa" resources: requests: cpu: 130m diff --git a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml index 8a252fe0..3cd9688d 100644 --- a/charts/dronescheduler/envs/dronescheduler-staging/values.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/values.yaml @@ -9,8 +9,6 @@ exports: level: "Information" reason: "new staging deploy" current: true - ingress: - class: "nginx-staging" resources: requests: cpu: 130m diff --git a/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml b/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml deleted file mode 100644 index 99b15734..00000000 --- a/charts/dronescheduler/templates/dronescheduler-internal-ingress.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ingress -################################################################################################### -{{- $svcversion := .Chart.AppVersion | replace "." "" }} -{{- $appversion := .Chart.AppVersion }} -{{- $defaultversionedpath := printf "/%s/" $appversion }} -{{- $relname := .Release.Name }} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $relname }}-internal-ingress - annotations: - kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} - nginx.ingress.kubernetes.io/rewrite-target: /api/dronedeliveries$1 - nginx.ingress.kubernetes.io/configuration-snippet: | - internal; -spec: - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .name }} - http: - paths: - {{- if .path }} - - path: {{ printf "%s/%s/" .path $appversion }}api/internal/dronedeliveries(.*) - {{- else }} - - path: {{ $defaultversionedpath }}api/internal/dronedeliveries(.*) - {{- end }} - backend: - serviceName: "{{ .serviceName }}-{{ $svcversion }}" - servicePort: http - {{- if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }}api/internal/dronedeliveries(.*) - backend: - serviceName: "{{ .serviceName }}" - servicePort: http - {{- end }} - {{ end }} diff --git a/charts/ingestion/envs/ingestion-dev/values.yaml b/charts/ingestion/envs/ingestion-dev/values.yaml index dfaa1ba2..286f2121 100644 --- a/charts/ingestion/envs/ingestion-dev/values.yaml +++ b/charts/ingestion/envs/ingestion-dev/values.yaml @@ -9,8 +9,6 @@ exports: level: "info" reason: "new dev deploy" current: true - ingress: - class: "nginx-dev" resources: requests: cpu: 110m diff --git a/charts/ingestion/envs/ingestion-prod/values.yaml b/charts/ingestion/envs/ingestion-prod/values.yaml index eb7c3936..fd328c17 100644 --- a/charts/ingestion/envs/ingestion-prod/values.yaml +++ b/charts/ingestion/envs/ingestion-prod/values.yaml @@ -8,8 +8,6 @@ exports: telemetry: level: "error" reason: "new prod deploy" - ingress: - class: "nginx-prod" resources: requests: cpu: 150m diff --git a/charts/ingestion/envs/ingestion-qa/values.yaml b/charts/ingestion/envs/ingestion-qa/values.yaml index 40473576..490659b2 100644 --- a/charts/ingestion/envs/ingestion-qa/values.yaml +++ b/charts/ingestion/envs/ingestion-qa/values.yaml @@ -9,8 +9,6 @@ exports: level: "info" reason: "new qa deploy" current: true - ingress: - class: "nginx-qa" resources: requests: cpu: 150m diff --git a/charts/ingestion/envs/ingestion-staging/values.yaml b/charts/ingestion/envs/ingestion-staging/values.yaml index 731b4765..717acf13 100644 --- a/charts/ingestion/envs/ingestion-staging/values.yaml +++ b/charts/ingestion/envs/ingestion-staging/values.yaml @@ -8,8 +8,6 @@ exports: telemetry: level: "info" reason: "new staging deploy" - ingress: - class: "nginx-staging" current: true resources: requests: diff --git a/charts/package/envs/package-dev/values.yaml b/charts/package/envs/package-dev/values.yaml index dddcf9c1..8eebdb82 100644 --- a/charts/package/envs/package-dev/values.yaml +++ b/charts/package/envs/package-dev/values.yaml @@ -9,8 +9,6 @@ exports: level: "info" reason: "new dev deploy" current: true - ingress: - class: "nginx-dev" resources: requests: cpu: 70m diff --git a/charts/package/envs/package-prod/values.yaml b/charts/package/envs/package-prod/values.yaml index 99cbba6e..7c1c90de 100644 --- a/charts/package/envs/package-prod/values.yaml +++ b/charts/package/envs/package-prod/values.yaml @@ -8,8 +8,6 @@ exports: log: level: "error" reason: "new prod deploy" - ingress: - class: "nginx-prod" resources: requests: cpu: 90m diff --git a/charts/package/envs/package-qa/values.yaml b/charts/package/envs/package-qa/values.yaml index 439ccf81..28668aa4 100644 --- a/charts/package/envs/package-qa/values.yaml +++ b/charts/package/envs/package-qa/values.yaml @@ -9,8 +9,6 @@ exports: level: "info" reason: "new qa deploy" current: true - ingress: - class: "nginx-qa" resources: requests: cpu: 90m diff --git a/charts/package/envs/package-staging/values.yaml b/charts/package/envs/package-staging/values.yaml index 20520a6e..3421a963 100644 --- a/charts/package/envs/package-staging/values.yaml +++ b/charts/package/envs/package-staging/values.yaml @@ -9,8 +9,6 @@ exports: level: "info" reason: "new staging deploy" current: true - ingress: - class: "nginx-staging" resources: requests: cpu: 90m diff --git a/charts/package/templates/package-internal-ingress.yaml b/charts/package/templates/package-internal-ingress.yaml deleted file mode 100644 index 969b7136..00000000 --- a/charts/package/templates/package-internal-ingress.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# ------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -# ------------------------------------------------------------ - -################################################################################################### -# ingress -################################################################################################### -{{- $svcversion := .Chart.AppVersion | replace "." "" }} -{{- $appversion := .Chart.AppVersion }} -{{- $defaultversionedpath := printf "/%s/" $appversion }} -{{- $relname := .Release.Name }} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $relname }}-internal-ingress - annotations: - kubernetes.io/ingress.class: {{ required "ingress.class is required" .Values.ingress.class | quote }} - nginx.ingress.kubernetes.io/rewrite-target: /api/package$1 - nginx.ingress.kubernetes.io/configuration-snippet: | - internal; -spec: - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .name }} - http: - paths: - {{- if .path }} - - path: {{ printf "%s/%s/" .path $appversion }}api/internal/package(.*) - {{- else }} - - path: {{ $defaultversionedpath }}api/internal/package(.*) - {{- end }} - backend: - serviceName: "{{ .serviceName }}-{{ $svcversion }}" - servicePort: http - {{- if (eq $appversion "v0.1.0") }} - - path: {{ default "/" .path }}api/internal/package(.*) - backend: - serviceName: "{{ .serviceName }}" - servicePort: http - {{- end }} - {{ end }} diff --git a/deployment.md b/deployment.md index d8ed5671..030f6d2b 100644 --- a/deployment.md +++ b/deployment.md @@ -206,9 +206,6 @@ helm install application-gateway-kubernetes-ingress/ingress-azure \ --set verbosityLevel=3 \ --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER -# Deploy the ngnix ingress controller -helm install stable/nginx-ingress --name nginx-ingress-dev --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-dev --set controller.service.type=ClusterIP - # Create a self-signed certificate for TLS export EXTERNAL_INGEST_FQDN=$APP_GATEWAY_PUBLIC_IP_FQDN openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ diff --git a/deploymentCICD.md b/deploymentCICD.md index ba8d7de2..1f042ace 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -252,8 +252,6 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ export "${ENV}_INGRESS_TLS_SECRET_CERT=$(echo $(cat ingestion-ingress-tls-${env}.crt) | tr '\n' "\\n")" export "${ENV}_INGRESS_TLS_SECRET_KEY=$(echo $(cat ingestion-ingress-tls-${env}.key) | tr '\n' "\\n")" -# Deploy the ngnix ingress controllers -helm install stable/nginx-ingress --name nginx-ingress-${env} --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-${env} --set controller.service.type=ClusterIP done # export app paths From 83acf46468094274dc5ce286d02354c2f5c0b37a Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Thu, 5 Dec 2019 16:53:40 +0000 Subject: [PATCH 211/246] Merged PR 897: parameterize appg tier, make waf the default parameterize appg tier, make waf the default --- azuredeploy.json | 127 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 26 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index bc0b6df3..2d585fb9 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -3,14 +3,14 @@ "contentVersion": "1.0.0.0", "parameters": { "environmentName": { - "type": "string", - "defaultValue": "dev", - "allowedValues": [ - "dev", - "qa", - "staging", - "prod" - ] + "type": "string", + "defaultValue": "dev", + "allowedValues": [ + "dev", + "qa", + "staging", + "prod" + ] }, "acrResourceGroupName": { "type": "string" @@ -155,6 +155,67 @@ "metadata": { "description": "Network policy used for configuring Kubernetes network profile." } + }, + "applicationGatewaySize": { + "type": "string", + "allowedValues": [ + "WAF_v2", + "Standard_v2" + ], + "defaultValue": "WAF_v2", + "metadata": { + "description": "Application gateway size" + } + }, + "applicationGatewayTier": { + "type": "string", + "allowedValues": [ + "WAF_v2", + "Standard_v2" + ], + "defaultValue": "WAF_v2", + "metadata": { + "description": "Application gateway tier" + } + }, + "applicationGatewayWafEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "WAF Enabled" + } + }, + "applicationGatewayWafMode": { + "type": "string", + "allowedValues": [ + "Detection", + "Prevention" + ], + "defaultValue": "Detection", + "metadata": { + "description": "WAF Mode" + } + }, + "applicationGatewayWafRuleSetType": { + "type": "string", + "allowedValues": [ + "OWASP" + ], + "defaultValue": "OWASP", + "metadata": { + "description": "WAF Rule Set Type" + } + }, + "applicationGatewayWafRuleSetVersion": { + "type": "string", + "allowedValues": [ + "2.2.9", + "3.0" + ], + "defaultValue": "3.0", + "metadata": { + "description": "WAF Rule Set Version" + } } }, "variables": { @@ -165,7 +226,6 @@ "appGatewayNamePrefix": "appg", "aksVnetAddressPrefix": "10.10.0.0/16", "aksClusterSubnetPrefix": "10.10.0.0/21", - "appGatewaySubnetPrefix": "10.10.8.0/24", "aksVnetNamePrefix": "vnet", "aksClusterSubnetNamePrefix": "subnet", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", @@ -185,9 +245,11 @@ "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "applicationGatewayMinCapacity": 1, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewaySubnetPrefix": "10.10.8.0/24", "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -220,9 +282,11 @@ "aksEnableAutoScaling": false, "acrName": "[variables('acrName')]", "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "applicationGatewayMinCapacity": 1, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewaySubnetPrefix": "10.10.9.0/24", "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -255,9 +319,11 @@ "aksEnableAutoScaling": true, "acrName": "[variables('acrName')]", "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "applicationGatewayMinCapacity": 2, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewaySubnetPrefix": "10.10.10.0/24", "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -290,9 +356,11 @@ "aksEnableAutoScaling": true, "acrName": "[concat(parameters('environmentName'),variables('acrName'))]", "appGatewayName": "[concat(parameters('environmentName'),'-ag-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "applicationGatewayMinCapacity": 2, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[uniqueString(variables('appGatewaySubnetPrefix'), resourceGroup().id)]", + "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", + "appGatewaySubnetPrefix": "10.10.11.0/24", "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -429,15 +497,15 @@ { "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName]", "properties": { - "addressPrefix": "[variables('appGatewaySubnetPrefix')]" + "addressPrefix": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetPrefix]" } } ] } }, { - "apiVersion":"2018-08-01", - "type":"Microsoft.Network/publicIPAddresses", + "apiVersion": "2018-08-01", + "type": "Microsoft.Network/publicIPAddresses", "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName]", "location": "[resourceGroup().location]", "tags": { @@ -445,12 +513,12 @@ "environment": "[parameters('environmentName')]" }, "sku": { - "name": "Standard" + "name": "Standard" }, "properties": { - "publicIPAllocationMethod": "Static", - "dnsSettings": { - "domainNameLabel": "[variables('environmentSettings')[parameters('environmentName')].appGatewayPublicDnsName]" + "publicIPAllocationMethod": "Static", + "dnsSettings": { + "domainNameLabel": "[variables('environmentSettings')[parameters('environmentName')].appGatewayPublicDnsName]" } } }, @@ -469,11 +537,11 @@ ], "properties": { "sku": { - "name": "Standard_v2", - "tier": "Standard_v2" + "name": "[parameters('applicationGatewaySize')]", + "tier": "[parameters('applicationGatewayTier')]" }, "autoscaleConfiguration": { - "minCapacity": 2 + "minCapacity": "[variables('environmentSettings')[parameters('environmentName')].applicationGatewayMinCapacity]" }, "gatewayIPConfigurations": [ { @@ -547,7 +615,14 @@ "Port": "80" } } - ] + ], + "webApplicationFirewallConfiguration": { + "enabled": "[parameters('applicationGatewayWafEnabled')]", + "firewallMode": "[parameters('applicationGatewayWafMode')]", + "ruleSetType": "[parameters('applicationGatewayWafRuleSetType')]", + "ruleSetVersion": "[parameters('applicationGatewayWafRuleSetVersion')]", + "disabledRuleGroups": [] + } } }, { @@ -1065,7 +1140,7 @@ "value": "50" }, "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" + "[resourceId('Microsoft.KeyVault/vaults', variables('environmentSettings')[parameters('environmentName')].droneSchedulerKeyVaultName)]" ] }, { @@ -1447,4 +1522,4 @@ "type": "string" } } -} +} \ No newline at end of file From 13807ddd1b0efd1159e97f453e16396ca0b7373f Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 2 Jan 2020 14:13:49 +0000 Subject: [PATCH 212/246] Merged PR 898: Delivery Network Policies ### [delivery] allow egress traffic from delivery to coredns for dns resolution starts allowing connections being stablished from delivery to CoreDns Verification: ``` / # nslookup microsoft.com nslookup: can't resolve '(null)': Name does not resolve Name: microsoft.com Address 1: 13.77.161.179 ... / # nslookup delivery.backend-dev nslookup: can't resolve '(null)': Name does not resolve Name: delivery.backend-dev Address 1: 10.2.0.167 delivery.backend-dev.svc.cluster.local ``` Important: this might not be required as per: https://github.com/Azure/azure-container-networking/issues/450 solved: #147493 ### [delivery] allow egress traffic from delivery to keyvault, cosmosdb and app insights starts allowing connections being stablished from delivery through port 443. This way delivery can connect to Azure KeyVault, Cosmos Db and Application Insights https endpoints Verification: ``` before / # wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.vault.azure.net/ wget: download timed out / # wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.documents.azure.com:443/ wget: download timed out after / # wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.documents.azure.com:443/ wget: server returned error: HTTP/1.1 401 Unauthorized / # wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.vault.azure.net/ wget: server returned error: HTTP/1.1 403 Forbidden ``` solved: #146500, #147345, #148528 ### [delivery] allow egress traffic from delivery to redis starts allowing connections being stablished from delivery through port 6380. This way delivery can start open connections to Azure Redis over TCP ``` before wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.redis.cache.windows.net:6380 wget: download timed out after / # wget -qO- --timeout 1 https://dev-d-yv3stnpy24lxy.redis.cache.windows.net:6380 wget: bad header line: -noauth Authentication required. ``` solved: #146496 ### [delivery] allow ingress traffic from workflow to delivery starts allowing connections being stablished from workflow to delivery ``` before / # wget -qO- --timeout 1 http://delivery.backend-dev wget: download timed out after / # wget -qO- --timeout 1 http://delivery.backend-dev/swagger/v1/swagger.json {"swagger":"2.0","info":{"version":"v1","title":"Fabrikam DroneDelivery DeliveryService API"}... ``` solved: #146048 ### [delivery] allow ingress traffic from agic to delivery starts allowing connections being stablished from a subnet where AGIC happens to be provisioned to delivery - ipblock selects the app gw subnet - open port 8080 tcp solved: #147484 ### workaround npm: ipBlock not working with external all ### make app gateway subnet configurable --- azuredeploy.json | 6 +- ...ry-networkpolicy-allow-egress-traffic.yaml | 32 ++++++++ ...y-networkpolicy-allow-ingress-traffic.yaml | 44 ++++++++++ .../delivery/templates/delivery-service.yaml | 4 +- charts/delivery/values.yaml | 33 ++++++++ .../workflow/templates/workflow-deploy.yaml | 1 + charts/workflow/values.yaml | 4 + deployment.md | 5 +- deploymentCICD.md | 8 +- src/shipping/delivery/azure-pipelines-cd.json | 82 +++++++++++-------- 10 files changed, 178 insertions(+), 41 deletions(-) create mode 100644 charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml create mode 100644 charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml diff --git a/azuredeploy.json b/azuredeploy.json index 2d585fb9..ef15d4e0 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -1520,6 +1520,10 @@ "appGatewayPublicIpFqdn": { "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('environmentSettings')[parameters('environmentName')].appGatewayPublicIpName)).dnsSettings.fqdn]", "type": "string" + }, + "appGatewaySubnetPrefix": { + "value": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetPrefix]", + "type": "string" } } -} \ No newline at end of file +} diff --git a/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml b/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml new file mode 100644 index 00000000..3da8c826 --- /dev/null +++ b/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml @@ -0,0 +1,32 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery allow egress traffic +################################################################################################### + +{{- if .Values.networkPolicy.egress.enabled }} +{{- $fullname := include "delivery.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-egress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Egress +{{- if .Values.networkPolicy.egress.customSelectors }} + egress: +{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} +{{- else if .Values.networkPolicy.egress.allowAll }} + egress: [] +{{- end -}} +{{ end }} diff --git a/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml b/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml new file mode 100644 index 00000000..92bbc051 --- /dev/null +++ b/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml @@ -0,0 +1,44 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Delivery allow ingress traffic +################################################################################################### + +{{- if .Values.networkPolicy.ingress.enabled }} +{{- $fullname := include "delivery.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-ingress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "delivery.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Ingress + ingress: + - ports: + - protocol: {{ .Values.service.targetProtocol }} + port: {{ .Values.service.targetPort }} +{{- if .Values.networkPolicy.ingress.customSelectors }} + from: +{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} +{{- end }} +{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} +{{- end }} +{{- else if .Values.networkPolicy.ingress.allowAll }} + from: + - podSelector: {} + - namespaceSelector: {} + - ipBlock: {} +{{- end -}} +{{ end }} diff --git a/charts/delivery/templates/delivery-service.yaml b/charts/delivery/templates/delivery-service.yaml index 6d1d7519..7e24c7d2 100644 --- a/charts/delivery/templates/delivery-service.yaml +++ b/charts/delivery/templates/delivery-service.yaml @@ -25,7 +25,7 @@ spec: ports: - name: http port: 80 - targetPort: 8080 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} @@ -48,7 +48,7 @@ spec: ports: - name: http port: 80 - targetPort: 8080 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index b4801cca..e92076ba 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -43,3 +43,36 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: +networkPolicy: + egress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow ingress traffic from workflow + templateSelector: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-delivery: true + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +service: + targetPort: 8080 + targetProtocol: TCP diff --git a/charts/workflow/templates/workflow-deploy.yaml b/charts/workflow/templates/workflow-deploy.yaml index 8e1768aa..8e56f405 100644 --- a/charts/workflow/templates/workflow-deploy.yaml +++ b/charts/workflow/templates/workflow-deploy.yaml @@ -39,6 +39,7 @@ spec: app.kubernetes.io/part-of: dronedelivery helm.sh/chart: {{ include "workflow.chart" . }} aadpodidbinding: {{ $fullname }} +{{ toYaml .Values.workflow.customPodLabels | indent 8 }} spec: securityContext: fsGroup: 1 diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 157179d4..c8941375 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -60,3 +60,7 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: + +workflow: + customPodLabels: + dd.fabrikam.com/egress-delivery: true diff --git a/deployment.md b/deployment.md index 030f6d2b..1d6af923 100644 --- a/deployment.md +++ b/deployment.md @@ -124,7 +124,8 @@ Get outputs from Azure Deploy export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \ export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) -export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) +export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) && \ +export GATEWAY_SUBNET_PREFIX=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewaySubnetPrefix.value -o tsv) ``` Download kubectl and create a k8s namespace @@ -274,6 +275,8 @@ helm install $HELM_CHARTS/delivery/ \ --set ingress.tls.secrets[0].name=$DELIVERY_INGRESS_TLS_SECRET_NAME \ --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ + --set networkPolicy.ingress.customSelectors.argSelector={ipBlock} \ + --set networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=$GATEWAY_SUBNET_PREFIX \ --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ --set cosmosdb.id=$DATABASE_NAME \ diff --git a/deploymentCICD.md b/deploymentCICD.md index 1f042ace..2aed9564 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -61,7 +61,7 @@ export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) export {${ENV}_ACR_NAME,ACR_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.acrName.value -o tsv) export ${ENV}_ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) - +export ${ENV}_GATEWAY_SUBNET_PREFIX=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appGatewaySubnetPrefix.value -o tsv) done ``` @@ -329,6 +329,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$DEV_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL#$DEV_GATEWAY_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ @@ -341,6 +342,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$QA_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#QA_GATEWAY_SUBNET_PREFIX_VAR_VAL#$QA_GATEWAY_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -353,6 +355,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL#$STAGING_GATEWAY_SUBNET_PREFIX#g" | \ # production resources sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -366,7 +369,8 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#PROD_EXTERNAL_INGEST_FQDN_VAR_VAL#$PROD_EXTERNAL_INGEST_FQDN#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" \ + sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" \ > $DELIVERY_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index 8a5505e9..d7c13bf3 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -53,12 +53,15 @@ "INGRESS_TLS_SECRET_NAME": { "value": "DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL" }, - "ACR_SERVER": { - "value": "DEV_ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "DEV_ACR_NAME_VAR_VAL" - } + "ACR_SERVER": { + "value": "DEV_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "DEV_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -248,7 +251,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -450,12 +453,15 @@ "INGRESS_TLS_SECRET_NAME": { "value": "QA_INGRESS_TLS_SECRET_NAME_VAR_VAL" }, - "ACR_SERVER": { - "value": "QA_ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "QA_ACR_NAME_VAR_VAL" - } + "ACR_SERVER": { + "value": "QA_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "QA_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "QA_GATEWAY_SUBNET_PREFIX_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -645,7 +651,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -842,12 +848,15 @@ "INGRESS_TLS_SECRET_NAME": { "value": "STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL" }, - "ACR_SERVER": { - "value": "STAGING_ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "STAGING_ACR_NAME_VAR_VAL" - } + "ACR_SERVER": { + "value": "STAGING_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "STAGING_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -1037,7 +1046,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1233,18 +1242,21 @@ "INGRESS_TLS_SECRET_NAME": { "value": "PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL" }, - "SOURCE_ACR_SERVER": { - "value": "SOURCE_ACR_SERVER_VAR_VAL" - }, - "SOURCE_ACR_NAME": { - "value": "SOURCE_ACR_NAME_VAR_VAL" - }, - "ACR_SERVER": { - "value": "PROD_ACR_SERVER_VAR_VAL" - }, - "ACR_NAME": { - "value": "PROD_ACR_NAME_VAR_VAL" - } + "SOURCE_ACR_SERVER": { + "value": "SOURCE_ACR_SERVER_VAR_VAL" + }, + "SOURCE_ACR_NAME": { + "value": "SOURCE_ACR_NAME_VAR_VAL" + }, + "ACR_SERVER": { + "value": "PROD_ACR_SERVER_VAR_VAL" + }, + "ACR_NAME": { + "value": "PROD_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL" + } }, "variableGroups": [], "preDeployApprovals": { @@ -1462,7 +1474,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,delivery-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,delivery-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1763,7 +1775,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", From a3298ef7b00659a2078dabdb96b92bb77c9dc0b0 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 2 Jan 2020 18:35:25 +0000 Subject: [PATCH 213/246] Merged PR 899: Workflow Network Policies ### [workflow] allow egress traffic from workflow to coredns for dns resolution starts allowing connections being stablished from workflow to CoreDns Important: this might not be required as per: https://github.com/Azure/azure-container-networking/issues/450 solved: #146047 ### [workflow] allow egress traffic from workflow to keyvault and app insights starts allowing connections being stablished from workflow through port 443. This way workflow can connect to Azure KeyVault and Application Insights https endpoints Verification: ``` before / # curl -I --max-time 1 https://dev-wf-yv3stnpy24lxy.vault.azure.net:443 curl: (28) Connection timed out after 1000 milliseconds after / # curl -I https://dev-wf-yv3stnpy24lxy.vault.azure.net/ HTTP/2 403 content-length: 1233 ``` solved: #147347, #148531 ### [workflow] allow egress traffic from workflow to azure service bus AMQP starts allowing connections being stablished from workflow through port 5671. This way workflow can connect to Azure Service Bus using AMQP. For more information, https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#basic-amqp-scenarios solved: #147350 ### workaround npm: ipBlock not working with external all --- ...ow-networkpolicy-allow-egress-traffic.yaml | 32 +++++++++++++++++++ charts/workflow/values.yaml | 17 ++++++++++ 2 files changed, 49 insertions(+) create mode 100644 charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml diff --git a/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml b/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml new file mode 100644 index 00000000..6e4493b7 --- /dev/null +++ b/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml @@ -0,0 +1,32 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Workflow allow egress traffic +################################################################################################### + +{{- if .Values.networkPolicy.egress.enabled }} +{{- $fullname := include "workflow.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-egress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "workflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Egress +{{- if .Values.networkPolicy.egress.customSelectors }} + egress: +{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} +{{- else if .Values.networkPolicy.egress.allowAll }} + egress: [] +{{- end -}} +{{ end }} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index c8941375..2852603a 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -60,6 +60,23 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: +networkPolicy: + egress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP workflow: customPodLabels: From 18bc7cb06deea60205a75c9aa6c99b5b3b25175f Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 2 Jan 2020 18:37:47 +0000 Subject: [PATCH 214/246] Merged PR 900: Ingestion Network Policies ### [ingestion] allow egress traffic from ingestion to azure service bus AMQP starts allowing connections being stablished from ingestion through port 5671. This way workflow can connect to Azure Service Bus using AMQP. For more information, https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#basic-amqp-scenarios solved: #146449 ### [ingestion] allow egress traffic from ingestion to app insights starts allowing connections being stablished from ingestion through port 443. This way ingestion can connect to Azure Application Insights https endpoints solved: #148526 ### [ingestion] allow egress traffic from ingestion to coredns for dns resolution starts allowing connections being stablished from ingestion to CoreDns Important: this might not be required as per: https://github.com/Azure/azure-container-networking/issues/450 solved: #147496 ### [ingestion] allow ingress traffic from agic to ingestion starts allowing connections being stablished from a subnet where AGIC happens to be provisioned to ingestion - ipblock selects the app gw subnet - open port 80 tcp solved: #146493 ### [ingestion] workaround npm: ipBlock not working with external all ### [ingestion] make app gw subnet configurable now it is possible to pass as arg the ingress subnet to allow its ingress traffic. - add deployment steps to get and pass the gateway subnet - adapt helm chart - add new subnet varible and use it in the cd pipeline --- ...on-networkpolicy-allow-egress-traffic.yaml | 32 +++++++++++++ ...n-networkpolicy-allow-ingress-traffic.yaml | 45 +++++++++++++++++++ .../templates/ingestion-service.yaml | 4 +- charts/ingestion/values.yaml | 28 ++++++++++++ deployment.md | 2 + deploymentCICD.md | 6 ++- .../ingestion/azure-pipelines-cd.json | 22 ++++++--- 7 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml create mode 100644 charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml diff --git a/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml b/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml new file mode 100644 index 00000000..66d806db --- /dev/null +++ b/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml @@ -0,0 +1,32 @@ +# ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Ingestion allow egress traffic +################################################################################################### + +{{- if .Values.networkPolicy.egress.enabled }} +{{- $fullname := include "ingestion.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-egress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Egress +{{- if .Values.networkPolicy.egress.customSelectors }} + egress: +{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} +{{- else if .Values.networkPolicy.egress.allowAll }} + egress: [] +{{- end -}} +{{ end }} diff --git a/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml b/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml new file mode 100644 index 00000000..3de91810 --- /dev/null +++ b/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml @@ -0,0 +1,45 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Ingestion allow ingress traffic +################################################################################################### + +{{- if .Values.networkPolicy.ingress.enabled }} +{{- $fullname := include "ingestion.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-ingress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ingestion.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Ingress + ingress: + - ports: + - protocol: {{ .Values.service.targetProtocol }} + port: {{ .Values.service.targetPort }} +{{- if .Values.networkPolicy.ingress.customSelectors }} + from: +{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} +{{- end }} +{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} +{{- end }} +{{- else if .Values.networkPolicy.ingress.allowAll }} + from: + - podSelector: {} + - namespaceSelector: {} + - ipBlock: {} +{{- end -}} +{{ end }} + diff --git a/charts/ingestion/templates/ingestion-service.yaml b/charts/ingestion/templates/ingestion-service.yaml index 740aba5f..aa8cb289 100644 --- a/charts/ingestion/templates/ingestion-service.yaml +++ b/charts/ingestion/templates/ingestion-service.yaml @@ -25,7 +25,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} @@ -48,7 +48,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 992ad10f..6b705b6b 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -36,3 +36,31 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: +networkPolicy: + egress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + # customSelectors: + # templateSelector: + # argSelector: +service: + targetPort: 80 + targetProtocol: TCP diff --git a/deployment.md b/deployment.md index 1d6af923..a94e42d5 100644 --- a/deployment.md +++ b/deployment.md @@ -437,6 +437,8 @@ helm install $HELM_CHARTS/ingestion/ \ --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ + --set networkPolicy.ingress.customSelectors.argSelector={ipBlock} \ + --set networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=$GATEWAY_SUBNET_PREFIX \ --set secrets.appinsights.ikey=${AI_IKEY} \ --set secrets.queue.keyname=IngestionServiceAccessKey \ --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ diff --git a/deploymentCICD.md b/deploymentCICD.md index 2aed9564..69747dd4 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -622,6 +622,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_CERT_VAR_VAL#$DEV_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL#$DEV_GATEWAY_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ @@ -633,6 +634,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_CERT_VAR_VAL#$QA_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#QA_GATEWAY_SUBNET_PREFIX_VAR_VAL#$QA_GATEWAY_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ @@ -644,6 +646,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_CERT_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL#$STAGING_GATEWAY_SUBNET_PREFIX#g" | \ # production resources sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ @@ -656,7 +659,8 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#PROD_INGESTION_ACCESS_KEY_VALUE_VAR_VAL#$PROD_INGESTION_ACCESS_KEY_VALUE#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ - sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" \ + sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ + sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" \ > $INGESTION_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index 26506053..fa6f4af6 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -57,6 +57,9 @@ }, "ACR_NAME": { "value": "DEV_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -240,7 +243,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -440,6 +443,9 @@ }, "ACR_NAME": { "value": "QA_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "QA_GATEWAY_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -623,7 +629,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -823,6 +829,9 @@ }, "ACR_NAME": { "value": "STAGING_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1006,7 +1015,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1212,6 +1221,9 @@ }, "ACR_NAME": { "value": "PROD_ACR_NAME_VAR_VAL" + }, + "GATEWAY_SUBNET_PREFIX": { + "value": "PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1423,7 +1435,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1717,7 +1729,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", From 10410e324a4740c601510fb28eb792ba8ae737aa Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 2 Jan 2020 18:52:36 +0000 Subject: [PATCH 215/246] Merged PR 901: Dronescheduler Network Policies ### [dronescheduler] allow egress traffic from dronescheduler to coredns for dns resolution starts allowing connections being stablished from dronescheduler to CoreDns Important: this might not be required as per: https://github.com/Azure/azure-container-networking/issues/450 solved: #147497 ### [dronescheduler] allow ingress traffic from workflow to dronescheduler starts allowing connections being stablished from workflow to dronescheduler solved: #147349 ### [dronescheduler] allow egress traffic from dronescheduler to keyvault, cosmosdb and app insights starts allowing connections being stablished from delivery through port 443. This way dronescheduler can connect to Azure KeyVault, Cosmos Db and Application Insights https endpoints solved: #147346, #146498, #148530 ### [dronescheduler] workaround npm: ipBlock not working with external all ### [dronescheduler] align np ingress implementation --- ...er-networkpolicy-allow-egress-traffic.yaml | 32 ++++++++++++++ ...r-networkpolicy-allow-ingress-traffic.yaml | 44 +++++++++++++++++++ .../templates/dronescheduler-service.yaml | 4 +- charts/dronescheduler/values.yaml | 33 ++++++++++++++ charts/workflow/values.yaml | 1 + 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml create mode 100644 charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml diff --git a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml new file mode 100644 index 00000000..1afefc6a --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml @@ -0,0 +1,32 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler allow egress traffic +################################################################################################### + +{{- if .Values.networkPolicy.egress.enabled }} +{{- $fullname := include "dronescheduler.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-egress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Egress +{{- if .Values.networkPolicy.egress.customSelectors }} + egress: +{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} +{{- else if .Values.networkPolicy.egress.allowAll }} + egress: [] +{{- end -}} +{{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml new file mode 100644 index 00000000..4e68cd83 --- /dev/null +++ b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml @@ -0,0 +1,44 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Dronescheduler allow ingress traffic +################################################################################################### + +{{- if .Values.networkPolicy.ingress.enabled }} +{{- $fullname := include "dronescheduler.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-ingress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "dronescheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Ingress + ingress: + - ports: + - protocol: {{ .Values.service.targetProtocol }} + port: {{ .Values.service.targetPort }} +{{- if .Values.networkPolicy.ingress.customSelectors }} + from: +{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} +{{- end }} +{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} +{{- end }} +{{- else if .Values.networkPolicy.ingress.allowAll }} + from: + - podSelector: {} + - namespaceSelector: {} + - ipBlock: {} +{{- end -}} +{{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-service.yaml b/charts/dronescheduler/templates/dronescheduler-service.yaml index f347d1bb..9f2ae59c 100644 --- a/charts/dronescheduler/templates/dronescheduler-service.yaml +++ b/charts/dronescheduler/templates/dronescheduler-service.yaml @@ -25,7 +25,7 @@ spec: ports: - name: http port: 80 - targetPort: 8080 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} @@ -48,7 +48,7 @@ spec: ports: - name: http port: 80 - targetPort: 8080 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 6e6f2aad..e0127401 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -42,3 +42,36 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: +networkPolicy: + egress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow ingress traffic from workflow + templateSelector: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-dronescheduler: true + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +service: + targetPort: 8080 + targetProtocol: TCP diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 2852603a..9d105005 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -81,3 +81,4 @@ networkPolicy: workflow: customPodLabels: dd.fabrikam.com/egress-delivery: true + dd.fabrikam.com/egress-dronescheduler: true From 936da15a2ac6abc33ebd3eb260f3170877f9e948 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 2 Jan 2020 19:01:48 +0000 Subject: [PATCH 216/246] Merged PR 902: Package Network Policies ### [package] allow egress traffic from package to coredns for dns resolution starts allowing connections being stablished from package to CoreDns Important: this might not be required as per: https://github.com/Azure/azure-container-networking/issues/450 solved: #147494 ### [package] allow ingress traffic from workflow to package starts allowing connections being stablished from workflow to package solved: #147348 ### [package] allow egress traffic from package to app insights starts allowing connections being stablished from package through port 443. This way package can connect to Azure Application Insights https endpoints For more information: https://docs.microsoft.com/en-us/azure/azure-monitor/app/ip-addresses#outgoing-ports solved: #148529 ### [package] allow egress traffic from package to azure mongo db starts allowing connections being stablished from package through port 10255. This way workflow can connect to Azure Mongo Db. solved: #146497 ### [package] workaround npm: ipBlock not working with external all ### [package] align np ingress implementation --- ...ge-networkpolicy-allow-egress-traffic.yaml | 32 ++++++++++++++ ...e-networkpolicy-allow-ingress-traffic.yaml | 44 +++++++++++++++++++ charts/package/templates/package-service.yaml | 4 +- charts/package/values.yaml | 33 ++++++++++++++ charts/workflow/values.yaml | 1 + 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml create mode 100644 charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml diff --git a/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml b/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml new file mode 100644 index 00000000..b1c2e92c --- /dev/null +++ b/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml @@ -0,0 +1,32 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package allow egress traffic +################################################################################################### + +{{- if .Values.networkPolicy.egress.enabled }} +{{- $fullname := include "package.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-egress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Egress +{{- if .Values.networkPolicy.egress.customSelectors }} + egress: +{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} +{{- else if .Values.networkPolicy.egress.allowAll }} + egress: [] +{{- end -}} +{{ end }} diff --git a/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml b/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml new file mode 100644 index 00000000..fd439f5b --- /dev/null +++ b/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml @@ -0,0 +1,44 @@ +## ------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +# ------------------------------------------------------------ + +################################################################################################### +# Package allow ingress ingress traffic +################################################################################################### + +{{- if .Values.networkPolicy.ingress.enabled }} +{{- $fullname := include "package.fullname" . | replace "." "" }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-np-whitelist-ingress-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "package.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery + app.kubernetes.io/version: {{ .Chart.AppVersion }} + policyTypes: + - Ingress + ingress: + - ports: + - protocol: {{ .Values.service.targetProtocol }} + port: {{ .Values.service.targetPort }} +{{- if .Values.networkPolicy.ingress.customSelectors }} + from: +{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} +{{- end }} +{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} +{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} +{{- end }} +{{- else if .Values.networkPolicy.ingress.allowAll }} + from: + - podSelector: {} + - namespaceSelector: {} + - ipBlock: {} +{{- end -}} +{{ end }} diff --git a/charts/package/templates/package-service.yaml b/charts/package/templates/package-service.yaml index 7be0cde0..156cb0c8 100644 --- a/charts/package/templates/package-service.yaml +++ b/charts/package/templates/package-service.yaml @@ -25,7 +25,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} @@ -48,7 +48,7 @@ spec: ports: - name: http port: 80 - targetPort: 80 + targetPort: {{ .Values.service.targetPort }} selector: app.kubernetes.io/name: {{ $appname }} app.kubernetes.io/instance: {{ $instancename }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 9dbff564..4e5cebbc 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -37,3 +37,36 @@ autoscaling: maxReplicas: minReplicas: targetCPUUtilizationPercentage: +networkPolicy: + egress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + enabled: true + # allowAll will be ingnored if customSelectons are provided. Comment out + # customSelectors if allowAll is required + allowAll: false + customSelectors: + # allow ingress traffic from workflow + templateSelector: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-package: true + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +service: + targetPort: 80 + targetProtocol: TCP diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 9d105005..54031618 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -82,3 +82,4 @@ workflow: customPodLabels: dd.fabrikam.com/egress-delivery: true dd.fabrikam.com/egress-dronescheduler: true + dd.fabrikam.com/egress-package: true From c04cd7d50efe26f4e27a6cb42ab390d56d0a7419 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 6 Jan 2020 15:54:54 +0000 Subject: [PATCH 217/246] Merged PR 905: bug fix: escape network policy true values bug fix: escape network policy true values solved: #159501 --- charts/delivery/values.yaml | 2 +- charts/dronescheduler/values.yaml | 2 +- charts/package/values.yaml | 2 +- charts/workflow/values.yaml | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index e92076ba..d7434df8 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -70,7 +70,7 @@ networkPolicy: templateSelector: - podSelector: matchLabels: - dd.fabrikam.com/egress-delivery: true + dd.fabrikam.com/egress-delivery: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery service: diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index e0127401..1e0f5e40 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -69,7 +69,7 @@ networkPolicy: templateSelector: - podSelector: matchLabels: - dd.fabrikam.com/egress-dronescheduler: true + dd.fabrikam.com/egress-dronescheduler: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery service: diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 4e5cebbc..371da43a 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -64,7 +64,7 @@ networkPolicy: templateSelector: - podSelector: matchLabels: - dd.fabrikam.com/egress-package: true + dd.fabrikam.com/egress-package: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery service: diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 54031618..a03201a6 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -80,6 +80,6 @@ networkPolicy: workflow: customPodLabels: - dd.fabrikam.com/egress-delivery: true - dd.fabrikam.com/egress-dronescheduler: true - dd.fabrikam.com/egress-package: true + dd.fabrikam.com/egress-delivery: "true" + dd.fabrikam.com/egress-dronescheduler: "true" + dd.fabrikam.com/egress-package: "true" From c684e317a86a162a1e9e3739684cfe785eb6afbf Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 6 Jan 2020 16:32:38 +0000 Subject: [PATCH 218/246] Merged PR 903: configure cluster egress ### egress limit - add and configure firewall - add deployment instructions - create static pip for the firewall For more information, https://docs.microsoft.com/en-us/azure/aks/limit-egress-traffic#required-ports-and-addresses-for-aks-clusters solved: #132445, #146496, #146497, #146498, #146499, #146500, #147345, #147346, #147347, #147350, #148526, #148528, #148529, #148530, #148531 ### replace service tag discovery with built-in service tag fw feat. - use service tags as destination addresses - remove deployment instructions to discover service tags ### split cluster and firewall deployments - create separate arm for the firewall provisioning - create udr with firewall private ip - associate udr to cluster subnet --- azuredeploy-firewall.json | 485 ++++++++++++++++++++++++++++++++++++++ azuredeploy.json | 92 +++++++- deployment.md | 28 ++- deploymentCICD.md | 32 ++- 4 files changed, 615 insertions(+), 22 deletions(-) create mode 100644 azuredeploy-firewall.json diff --git a/azuredeploy-firewall.json b/azuredeploy-firewall.json new file mode 100644 index 00000000..d2f50fb5 --- /dev/null +++ b/azuredeploy-firewall.json @@ -0,0 +1,485 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "aksVnetName": { + "type": "string" + }, + "aksClusterSubnetName": { + "type": "string" + }, + "aksClusterSubnetPrefix": { + "type": "string" + }, + "firewallPublicIpName": { + "type": "string" + }, + "firewallSubnetName": { + "type": "string", + "defaultValue": "AzureFirewallSubnet" + }, + "serviceTagsLocation": { + "type": "string" + }, + "aksFqdns": { + "type": "array" + }, + "acrServers": { + "type": "array" + }, + "deliveryRedisHostNames": { + "type": "array" + } + }, + "variables": { + "rulePrefix": "[concat(parameters('aksVnetName'),'-',parameters('aksClusterSubnetName'),'.')]", + "regionalRulePrefix": "[concat(variables('rulePrefix'),parameters('serviceTagsLocation'),'.')]", + "egressFirewallPrefix": "egressFw", + "egressFirewallName": "[concat(variables('egressFirewallPrefix'),'-',uniqueString(resourceGroup().id))]", + "clusterEgressRouteTableName": "[concat('egressRouteTable-',uniqueString(resourceGroup().id))]", + "clusterEgressRouteName": "[concat('egressRoute-',uniqueString(resourceGroup().id))]" + }, + "resources": [ + { + "type": "Microsoft.Network/routeTables", + "apiVersion": "2019-09-01", + "name": "[variables('clusterEgressRouteTableName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/azureFirewalls', variables('egressFirewallName'))]" + ], + "properties": { + "routes": [ + { + "properties": { + "addressPrefix": "0.0.0.0/0", + "nextHopType": "VirtualAppliance", + "nextHopIpAddress": "[reference(concat('Microsoft.Network/azureFirewalls/', variables('egressFirewallName')),'2019-09-01','Full').properties.ipConfigurations[0].properties.privateIPAddress]" + }, + "name": "[variables('clusterEgressRouteName')]" + } + ] + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2019-09-01", + "name": "[concat(parameters('aksVnetName'),'/',parameters('aksClusterSubnetName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Network/routeTables', variables('clusterEgressRouteTableName'))]" + ], + "properties": { + "addressPrefix": "[parameters('aksClusterSubnetPrefix')]", + "routeTable": { + "id": "[resourceId('Microsoft.Network/routeTables', variables('clusterEgressRouteTableName'))]" + } + } + }, + { + "type": "Microsoft.Network/azureFirewalls", + "name": "[variables('egressFirewallName')]", + "dependsOn": [], + "apiVersion": "2019-09-01", + "location": "[resourceGroup().location]", + "properties": { + "threatIntelMode": "Alert", + "ipConfigurations": [ + { + "name": "clusterIpConfig", + "properties": { + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('firewallPublicIpName'))]" + }, + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('aksVnetName'),parameters('firewallSubnetName'))]" + } + } + } + ], + "networkRuleCollections": [ + { + "name": "egressNetRegionalAks", + "properties": { + "priority": 100, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('regionalRulePrefix'),'TunnelFrontCommToK8sApiServer')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "[concat('AzureCloud.',parameters('serviceTagsLocation'))]" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "9000", + "22" + ] + } + ] + } + }, + { + "name": "egressNetAks", + "properties": { + "priority": 200, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('rulePrefix'),'DnsResolution')]", + "protocols": [ + "TCP", + "UDP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "*" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "53" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'LinuxNodesTimeSync')]", + "protocols": [ + "UDP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "*" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "123" + ] + } + ] + } + }, + { + "name": "egressNetRegionalFabrikamDroneDelivery", + "properties": { + "priority": 300, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('regionalRulePrefix'),'ServiceBus')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "[concat('AzureCloud.',parameters('serviceTagsLocation'))]" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "5671" + ] + }, + { + "name": "[concat(variables('regionalRulePrefix'),'AzureCosmosDb')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "[concat('AzureCosmosDB.',parameters('serviceTagsLocation'))]" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "443" + ] + }, + { + "name": "[concat(variables('regionalRulePrefix'),'AzureMongoDb')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "[concat('AzureCosmosDB.',parameters('serviceTagsLocation'))]" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "10255" + ] + }, + { + "name": "[concat(variables('regionalRulePrefix'),'AzureKeyVault')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "[concat('AzureKeyVault.',parameters('serviceTagsLocation'))]" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "443" + ] + } + ] + } + }, + { + "name": "egressNetFabrikamDroneDelivery", + "properties": { + "priority": 400, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('rulePrefix'),'AzureMonitor')]", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "destinationAddresses": [ + "AzureMonitor" + ], + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "443" + ] + } + ] + } + } + ], + "applicationRuleCollections": [ + { + "name": "egressAppRegionalAks", + "properties": { + "priority": 100, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('regionalRulePrefix'),'K8sApiServerHcp')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": "[parameters('aksFqdns')]" + }, + { + "name": "[concat(variables('regionalRulePrefix'),'K8sApiServerTun')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "[concat('*.tun.',resourceGroup().location,'.azmk8s.io')]" + ] + } + ] + } + }, + { + "name": "egressAppAks", + "properties": { + "priority": 200, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('rulePrefix'),'ContainerRegistries')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "gcr.io", + "google.storageapis.com", + "aksrepos.azurecr.io", + "*mcr.microsoft.com", + "*.cdn.mscr.io", + "*.blob.core.windows.net" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'K8sGETPUTOperations')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "management.azure.com" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'Ubuntu')]", + "protocols": [ + { + "protocolType": "http", + "port": "80" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "*.ubuntu.com" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'AzureCNIBinaries')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "acs-mirror.azureedge.net" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'AzureActiveDirectoryAuthN')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "login.microsoftonline.com" + ] + }, + { + "name": "[concat(variables('rulePrefix'),'AzureMonitorAppInsightsQuickPulse')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": [ + "rt.services.visualstudio.com" + ] + } + ] + } + }, + { + "name": "egressAppFabrikamDroneDelivery", + "properties": { + "priority": 400, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "[concat(variables('rulePrefix'),'AzureContainerRegistries')]", + "protocols": [ + { + "protocolType": "https", + "port": "443" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": "[parameters('acrServers')]" + }, + { + "name": "[concat(variables('rulePrefix'),'AzureCacheRedis')]", + "protocols": [ + { + "protocolType": "https", + "port": "6380" + } + ], + "sourceAddresses": [ + "[parameters('aksClusterSubnetPrefix')]" + ], + "targetFqdns": "[parameters('deliveryRedisHostNames')]" + } + ] + } + } + ], + "natRuleCollections": [] + } + } + ], + "outputs": { + "azureFirewallName": { + "type": "string", + "value": "[variables('egressFirewallName')]" + } + } +} diff --git a/azuredeploy.json b/azuredeploy.json index ef15d4e0..6f88be2f 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -226,6 +226,8 @@ "appGatewayNamePrefix": "appg", "aksVnetAddressPrefix": "10.10.0.0/16", "aksClusterSubnetPrefix": "10.10.0.0/21", + "firewallSubnetPrefix": "10.10.12.0/24", + "firewallSubnetName": "AzureFirewallSubnet", "aksVnetNamePrefix": "vnet", "aksClusterSubnetNamePrefix": "subnet", "readerRoleObjectId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", @@ -237,6 +239,8 @@ "deliveryRedisStorageName": "[concat(parameters('environmentName'),'rsto',uniqueString(resourceGroup().id))]", "nestedACRDeploymentName": "[concat('azuredeploy-acr-',parameters('acrResourceGroupName'),parameters('environmentName'))]", "aksLogAnalyticsNamePrefix": "logsAnalytics", + "egressFirewallPrefix": "egressFw", + "firewallPublicIpName": "[concat('firewallPip-',uniqueString(variables('egressFirewallPrefix'), resourceGroup().id))]", "environmentSettings": { "dev": { "aksClusterName": "[uniqueString(variables('clusterNamePrefix'), resourceGroup().id)]", @@ -494,6 +498,12 @@ "privateLinkServiceNetworkPolicies": "Enabled" } }, + { + "name": "[variables('firewallSubnetName')]", + "properties": { + "addressPrefix": "[variables('firewallSubnetPrefix')]" + } + }, { "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName]", "properties": { @@ -643,7 +653,7 @@ { "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterName]", "type": "Microsoft.ContainerService/managedClusters", - "apiVersion": "2019-02-01", + "apiVersion": "2019-08-01", "location": "[resourceGroup().location]", "tags": { "environment": "shared cluster" @@ -785,7 +795,8 @@ "apiVersion": "2016-03-31", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "tags": { "displayName": "Delivery Cosmos Db", @@ -806,7 +817,8 @@ "locationName": "[resourceGroup().location]", "failoverPriority": 0 } - ] + ], + "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" } }, { @@ -816,7 +828,8 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "tags": { "displayName": "Package Cosmos Db", @@ -831,7 +844,8 @@ { "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" } - ] + ], + "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" } }, { @@ -840,7 +854,8 @@ "apiVersion": "2015-04-08", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "properties": { "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", @@ -856,7 +871,8 @@ "locationName": "[resourceGroup().location]", "failoverPriority": 0 } - ] + ], + "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" } }, { @@ -924,7 +940,8 @@ "location": "[resourceGroup().location]", "dependsOn": [ "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "tags": { "displayName": "Delivery Key Vault", @@ -944,6 +961,11 @@ { "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" } + ], + "ipRules": [ + { + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + } ] }, "accessPolicies": [ @@ -1038,7 +1060,8 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "tags": { "displayName": "DroneScheduler Key Vault", @@ -1058,6 +1081,11 @@ { "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" } + ], + "ipRules": [ + { + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + } ] }, "accessPolicies": [ @@ -1184,7 +1212,8 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" ], "tags": { "displayName": "Workflow Key Vault", @@ -1204,6 +1233,11 @@ { "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" } + ], + "ipRules": [ + { + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + } ] }, "accessPolicies": [ @@ -1454,6 +1488,20 @@ "principalId": "[parameters('appGatewayControllerPrincipalId')]", "scope": "[resourceGroup().id]" } + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2019-09-01", + "location": "[resourceGroup().location]", + "name": "[variables('firewallPublicIpName')]", + "sku": { + "name": "Standard" + }, + "properties": { + "publicIPAddressVersion": "IPv4", + "publicIPAllocationMethod": "Static", + "idleTimeoutInMinutes": 4 + } } ], "outputs": { @@ -1524,6 +1572,30 @@ "appGatewaySubnetPrefix": { "value": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetPrefix]", "type": "string" + }, + "aksVNetName": { + "type": "string", + "value": "[variables('environmentSettings')[parameters('environmentName')].aksVnetName]" + }, + "aksClusterSubnetName": { + "type": "string", + "value": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]" + }, + "aksClusterSubnetPrefix": { + "type": "string", + "value": "[variables('aksClusterSubnetPrefix')]" + }, + "firewallSubnetName": { + "type": "string", + "value": "[variables('firewallSubnetName')]" + }, + "firewallPublicIpName": { + "type": "string", + "value": "[variables('firewallPublicIpName')]" + }, + "deliveryRedisHostName": { + "value": "[reference(resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)).hostName]", + "type": "string" } } } diff --git a/deployment.md b/deployment.md index a94e42d5..dfc061fb 100644 --- a/deployment.md +++ b/deployment.md @@ -98,8 +98,9 @@ until az ad sp show --id ${GATEWAY_CONTROLLER_ID_PRINCIPAL_ID} &> /dev/null ; do # Export the kubernetes cluster version export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) +export SERVICETAGS_LOCATION=$(az account list-locations --query "[?name=='${LOCATION}'].displayName" -o tsv | sed 's/[[:space:]]//g') -# Deploy all other resources +# Deploy cluster and microservices Azure services az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template-file ${PROJECT_ROOT}/azuredeploy.json \ --parameters servicePrincipalClientId=${SP_APP_ID} \ servicePrincipalClientSecret=${SP_CLIENT_SECRET} \ @@ -116,15 +117,32 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template- appGatewayControllerPrincipalId=${GATEWAY_CONTROLLER_ID_PRINCIPAL_ID} \ acrResourceGroupName=${RESOURCE_GROUP_ACR} \ acrResourceGroupLocation=$LOCATION + +export VNET_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksVNetName.value -o tsv) && \ +export CLUSTER_SUBNET_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterSubnetName.value -o tsv) && \ +export CLUSTER_SUBNET_PREFIX=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterSubnetPrefix.value -o tsv) && \ +export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) && \ +export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) && \ +export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.firewallPublicIpName.value -o tsv) && \ +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ +export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \ +export DELIVERY_REDIS_HOSTNAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryRedisHostName.value -o tsv) + +# Restrict cluster egress traffic +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-firewall --template-file ${PROJECT_ROOT}/azuredeploy-firewall.json \ +--parameters aksVnetName=${VNET_NAME} \ + aksClusterSubnetName=${CLUSTER_SUBNET_NAME} \ + aksClusterSubnetPrefix=${CLUSTER_SUBNET_PREFIX} \ + firewallPublicIpName=${FIREWALL_PIP_NAME} \ + serviceTagsLocation=${SERVICETAGS_LOCATION} \ + aksFqdns="['${CLUSTER_SERVER}']" \ + acrServers="['${ACR_SERVER}']" \ + deliveryRedisHostNames="['${DELIVERY_REDIS_HOSTNAME}']" ``` Get outputs from Azure Deploy ```bash # Shared -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ -export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \ -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) -export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) && \ export GATEWAY_SUBNET_PREFIX=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewaySubnetPrefix.value -o tsv) ``` diff --git a/deploymentCICD.md b/deploymentCICD.md index 69747dd4..2ab763bc 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -12,6 +12,7 @@ ```bash # Export the kubernetes cluster version and deploy export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv) && \ +export SERVICETAGS_LOCATION=$(az account list-locations --query "[?name=='${LOCATION}'].displayName" -o tsv | sed 's/[[:space:]]//g') for env in dev qa staging prod; do ENV=${env^^} az deployment create \ @@ -60,16 +61,33 @@ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-${env} --templa export {${ENV}_AI_NAME,AI_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appInsightsName.value -o tsv) export ${ENV}_AI_IKEY=$(az resource show -g $RESOURCE_GROUP -n $AI_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv) export {${ENV}_ACR_NAME,ACR_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.acrName.value -o tsv) -export ${ENV}_ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) export ${ENV}_GATEWAY_SUBNET_PREFIX=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appGatewaySubnetPrefix.value -o tsv) +export {${ENV}_VNET_NAME,VNET_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.aksVNetName.value -o tsv) +export {${ENV}_CLUSTER_SUBNET_NAME,CLUSTER_SUBNET_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.aksClusterSubnetName.value -o tsv) +export {${ENV}_CLUSTER_SUBNET_PREFIX,CLUSTER_SUBNET_PREFIX}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.aksClusterSubnetPrefix.value -o tsv) +export {${ENV}_CLUSTER_FQDN,CLUSTER_FQDN}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.aksFqdn.value -o tsv) +export {${ENV}_CLUSTER_NAME,CLUSTER_NAME}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.aksClusterName.value -o tsv) && \ +export {${ENV}_CLUSTER_SERVER,CLUSTER_SERVER}=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) +export CLUSTER_SERVERS=${CLUSTER_SERVERS}\'${CLUSTER_SERVER}\', +export {${ENV}_ACR_SERVER,ACR_SERVER}=$(az acr show -n $ACR_NAME --query loginServer -o tsv) +export ACR_SERVERS=${ACR_SERVERS}\'${ACR_SERVER}\', +export DELIVERY_REDIS_HOSTNAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryRedisHostName.value -o tsv) +export DELIVERY_REDIS_HOSTNAMES=${DELIVERY_REDIS_HOSTNAMES}\'${DELIVERY_REDIS_HOSTNAME}\', done -``` -Get outputs from Azure Deploy -```bash -# Shared -export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv) && \ -export CLUSTER_SERVER=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP --query fqdn -o tsv) +# Restrict cluster egress traffic +export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-prod --query properties.outputs.firewallPublicIpName.value -o tsv) && \ + +export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ +az group deployment create -g $RESOURCE_GROUP --name azuredeploy-firewall --template-file ${PROJECT_ROOT}/azuredeploy-firewall.json \ +--parameters aksVnetName=${VNET_NAME} \ + aksClusterSubnetName=${CLUSTER_SUBNET_NAME} \ + aksClusterSubnetPrefix=${CLUSTER_SUBNET_PREFIX} \ + firewallPublicIpName=${FIREWALL_PIP_NAME} \ + serviceTagsLocation=${SERVICETAGS_LOCATION} \ + aksFqdns="[${CLUSTER_SERVERS%?}]" \ + acrServers="[${ACR_SERVERS%?}]" \ + deliveryRedisHostNames="[${DELIVERY_REDIS_HOSTNAMES%?}]" ``` Download kubectl and create a k8s namespace From 378819f025731119ccde4d36952c0aa5f5b6019a Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Fri, 10 Jan 2020 19:29:00 +0000 Subject: [PATCH 219/246] Merged PR 908: Enable Service Endpoints from the Firewall Subnet ### Enable Service Endpoints from the Firewall Subnet now Azure Service (Azure CosmosDb and Azure KeyVault) allow inbound traffic from the Azure Firewall subnet. Based on this new configuration, we have an extra hop from the cluster subnet to the firewall subnet. That being said, the traffic reaching Azure Services are using private Ip(s) from the Azure Firewall subnet address space. For more information, please take a look at the possible configuration approaches below (in bold the implemented one): | Enable Service Endpoints from the Cluster Subnet | Enable Service Endpoints from the Firewall Subnet | Requires Azure Firewall Rule | Requires Azure Service Firewall Rule for the Public Firewall Ip (Ip Range) | Requires Azure Service Firewall Rule for the Cluster Subnet | Requires Azure Service Firewall Rule for the Firewall Subnet | IP | Secure (lower is better, but highest isn't bad) | Note | |---|--------------------------------------------------|------------------------------|----------------------------------------------------------------------------|-------------------------------------------------------------|--------------------------------------------------------------|---------------------|-------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | 0 | yes | yes | no | no | Public FW Ip | 3 | work without service endpoints + azure firewall rules + azure service firewall (ip range) | | **0** | **1** | **yes** | **no** | **no** | **yes** | **CIDR FW Subnet** | **1** | **work with service endpoints + azure firewall rules + azure services firewall rules + extra hop from cluster subnet to fw subnet. ** | | 1 | 0 | no | no | yes | no | CIDR Cluster Subnet | 2 | work with service endpoints + azure services firewall rules | | 1 | 1 | no | no | yes | no | CIDR Cluster Subnet | 2 | work with service endpoints + azure services firewall rules | solved: #161219 ### remove ip range filters Azure Services that allow inbound traffic from Azure Firewall subnet do not require to have an ip range filter for the Azure Firewall Ip. --- azuredeploy.json | 72 ++++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 6f88be2f..04840566 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -474,6 +474,14 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", "properties": { "addressPrefix": "[variables('aksClusterSubnetPrefix')]", + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "name": "[variables('firewallSubnetName')]", + "properties": { + "addressPrefix": "[variables('firewallSubnetPrefix')]", "serviceEndpoints": [ { "service": "Microsoft.KeyVault", @@ -493,15 +501,7 @@ "[resourceGroup().location]" ] } - ], - "privateEndpointNetworkPolicies": "Enabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - }, - { - "name": "[variables('firewallSubnetName')]", - "properties": { - "addressPrefix": "[variables('firewallSubnetPrefix')]" + ] } }, { @@ -795,8 +795,7 @@ "apiVersion": "2016-03-31", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Delivery Cosmos Db", @@ -809,7 +808,7 @@ "isVirtualNetworkFilterEnabled": true, "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } ], "locations": [ @@ -817,8 +816,7 @@ "locationName": "[resourceGroup().location]", "failoverPriority": 0 } - ], - "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + ] } }, { @@ -828,8 +826,7 @@ "name": "[variables('environmentSettings')[parameters('environmentName')].packageMongoDbName]", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Package Cosmos Db", @@ -842,10 +839,9 @@ "isVirtualNetworkFilterEnabled": true, "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } - ], - "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + ] } }, { @@ -854,8 +850,7 @@ "apiVersion": "2015-04-08", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "properties": { "name": "[variables('environmentSettings')[parameters('environmentName')].droneSchedulerCosmosDbName]", @@ -863,7 +858,7 @@ "isVirtualNetworkFilterEnabled": true, "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } ], "locations": [ @@ -871,8 +866,7 @@ "locationName": "[resourceGroup().location]", "failoverPriority": 0 } - ], - "ipRangeFilter": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + ] } }, { @@ -940,8 +934,7 @@ "location": "[resourceGroup().location]", "dependsOn": [ "[variables('environmentSettings')[parameters('environmentName')].deliveryRedisStorageId]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Delivery Key Vault", @@ -959,12 +952,7 @@ "defaultAction": "Deny", "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" - } - ], - "ipRules": [ - { - "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } ] }, @@ -1060,8 +1048,7 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "DroneScheduler Key Vault", @@ -1079,12 +1066,7 @@ "defaultAction": "Deny", "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" - } - ], - "ipRules": [ - { - "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } ] }, @@ -1212,8 +1194,7 @@ "apiVersion": "2016-10-01", "location": "[resourceGroup().location]", "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" ], "tags": { "displayName": "Workflow Key Vault", @@ -1231,12 +1212,7 @@ "defaultAction": "Deny", "virtualNetworkRules": [ { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName)]" - } - ], - "ipRules": [ - { - "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" } ] }, From c7e4ef6f15e943af685d80452bac34ce19d1cc97 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Mon, 13 Jan 2020 17:22:32 +0000 Subject: [PATCH 220/246] Merged PR 910: fix goolgleapis url from the fw rule fix goolgleapis url in the fw rule solved: #162272 --- azuredeploy-firewall.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azuredeploy-firewall.json b/azuredeploy-firewall.json index d2f50fb5..fec9fa78 100644 --- a/azuredeploy-firewall.json +++ b/azuredeploy-firewall.json @@ -349,7 +349,7 @@ ], "targetFqdns": [ "gcr.io", - "google.storageapis.com", + "storage.googleapis.com", "aksrepos.azurecr.io", "*mcr.microsoft.com", "*.cdn.mscr.io", From aee38a73c2d8b8c19857bb221b548cf8a5eace52 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 14 Jan 2020 17:19:51 +0000 Subject: [PATCH 221/246] Merged PR 911: allow ubuntu motd add 443 for ubuntu so motd doesn't get denied. --- azuredeploy-firewall.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azuredeploy-firewall.json b/azuredeploy-firewall.json index fec9fa78..836a9543 100644 --- a/azuredeploy-firewall.json +++ b/azuredeploy-firewall.json @@ -377,6 +377,10 @@ { "protocolType": "http", "port": "80" + }, + { + "protocolType": "https", + "port": "443" } ], "sourceAddresses": [ From 8b259a10f89453d83db9d13feb4ababa9f6154f0 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 21 Jan 2020 19:22:05 +0000 Subject: [PATCH 222/246] Merged PR 912: allow Azure Cloud per region on 443 with lowest prio at Network Rules level allow Azure Cloud per region on 443 with lowest prio at Network Rules level this is the alternative workaround for the more restrictive one approach which is allow outbound traffic for the AKS Api Server Public Ip. drawbacks: 1. less restrictive egress limit, we open more than needed having less control over the outbound traffic 2. most of - if not all - the Application Rules (FQDNs) are now being preceded/overrided by this new Network Rule, which is not by design only to be consider, if official doc for #160145 is not available. solved: #163212 --- azuredeploy-firewall.json | 61 ++++++++++++--------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/azuredeploy-firewall.json b/azuredeploy-firewall.json index 836a9543..ad5d220b 100644 --- a/azuredeploy-firewall.json +++ b/azuredeploy-firewall.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "aksVnetName": { - "type": "string" + "type": "string" }, "aksClusterSubnetName": { "type": "string" @@ -286,52 +286,42 @@ } ] } - } - ], - "applicationRuleCollections": [ + }, { - "name": "egressAppRegionalAks", + "name": "egressNetRegionalTCP443AzureCloud", "properties": { - "priority": 100, + "priority": 65000, "action": { "type": "Allow" }, "rules": [ { - "name": "[concat(variables('regionalRulePrefix'),'K8sApiServerHcp')]", + "name": "[concat(variables('regionalRulePrefix'),'AADPodIdentityWorkaround')]", "protocols": [ - { - "protocolType": "https", - "port": "443" - } + "TCP" ], "sourceAddresses": [ "[parameters('aksClusterSubnetPrefix')]" ], - "targetFqdns": "[parameters('aksFqdns')]" - }, - { - "name": "[concat(variables('regionalRulePrefix'),'K8sApiServerTun')]", - "protocols": [ - { - "protocolType": "https", - "port": "443" - } - ], - "sourceAddresses": [ - "[parameters('aksClusterSubnetPrefix')]" + "destinationAddresses": [ + "[concat('AzureCloud.',parameters('serviceTagsLocation'))]" ], - "targetFqdns": [ - "[concat('*.tun.',resourceGroup().location,'.azmk8s.io')]" + "sourceIpGroups": [], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "443" ] } ] } - }, + } + ], + "applicationRuleCollections": [ { "name": "egressAppAks", "properties": { - "priority": 200, + "priority": 100, "action": { "type": "Allow" }, @@ -419,21 +409,6 @@ "targetFqdns": [ "login.microsoftonline.com" ] - }, - { - "name": "[concat(variables('rulePrefix'),'AzureMonitorAppInsightsQuickPulse')]", - "protocols": [ - { - "protocolType": "https", - "port": "443" - } - ], - "sourceAddresses": [ - "[parameters('aksClusterSubnetPrefix')]" - ], - "targetFqdns": [ - "rt.services.visualstudio.com" - ] } ] } @@ -441,7 +416,7 @@ { "name": "egressAppFabrikamDroneDelivery", "properties": { - "priority": 400, + "priority": 200, "action": { "type": "Allow" }, From df7b739f4a0be10944bd71577e6fb0cd50aba327 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Fri, 24 Jan 2020 16:57:11 +0000 Subject: [PATCH 223/246] Merged PR 913: migrate Fabrikam.DroneDelivery.DeliveryService to ASP.NET Core 3.1 migrate Fabrikam.DroneDelivery.DeliveryService to ASP.NET Core 3.1 - migrate Fabrikam.DroneDelivery.Common to .NET Core 3.1 - migrate Fabrikam.DroneDelivery.DeliveryService to .NET Core 3.1 - remove view support - migrate Fabrikam.DroneDelivery.DeliveryService.Tests to .NET Core 3.1 - migrate TestServer to .NET Core 3.1 - change assert from NotFound to MethodNotAllowed. For more information: https://github.com/dotnet/aspnetcore/issues/11260#issuecomment-505614935 - fix unsupported deserialization in the new System.Text.Json. For more information: https://github.com/dotnet/corefx/issues/38569#issuecomment-502957651 - fix global logger middleware tests. For more information: https://github.com/moq/moq4/issues/918 solved: #160880 --- src/shipping/delivery/Dockerfile | 6 ++-- .../Fabrikam.DroneDelivery.Common.csproj | 6 ++-- .../Fabrikam.DroneDelivery.Common/Location.cs | 8 +++-- .../UserAccount.cs | 6 ++-- .../DeliveriesControllerRoutingFixture.cs | 14 ++++++--- ...DroneDelivery.DeliveryService.Tests.csproj | 27 ++++++++-------- .../GlobalLoggerMiddlewareFixture.cs | 31 +++++++++---------- .../Controllers/DeliveriesController.cs | 3 +- ...rikam.DroneDelivery.DeliveryService.csproj | 8 ++--- .../Models/Delivery.cs | 19 ++++++------ .../Program.cs | 27 +++++++++------- .../Startup.cs | 26 ++++++++-------- 12 files changed, 96 insertions(+), 85 deletions(-) diff --git a/src/shipping/delivery/Dockerfile b/src/shipping/delivery/Dockerfile index 0496458a..2ac03568 100644 --- a/src/shipping/delivery/Dockerfile +++ b/src/shipping/delivery/Dockerfile @@ -1,8 +1,8 @@ -FROM microsoft/dotnet:2.2-aspnetcore-runtime as base +FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 as base WORKDIR /app EXPOSE 8080 -FROM microsoft/dotnet:2.2-sdk AS build +FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build WORKDIR /app COPY Fabrikam.DroneDelivery.Common/*.csproj ./Fabrikam.DroneDelivery.Common/ @@ -31,7 +31,7 @@ FROM build AS publish MAINTAINER Fernando Antivero (https://github.com/ferantivero) WORKDIR /app -RUN dotnet publish /app/Fabrikam.DroneDelivery.DeliveryService/ -c Release -o ../out +RUN dotnet publish /app/Fabrikam.DroneDelivery.DeliveryService/ -c release -o ./out --no-restore FROM base AS runtime diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj index e7b6e6a5..db0c8a56 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Fabrikam.DroneDelivery.Common.csproj @@ -1,13 +1,13 @@ - netcoreapp2.2 + netcoreapp3.1 - + - + \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs index ebde932b..21fd6888 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/Location.cs @@ -7,14 +7,16 @@ namespace Fabrikam.DroneDelivery.Common { public class Location { + public Location() {} + public Location(double altitude, double latitude, double longitude) { Altitude = altitude; Latitude = latitude; Longitude = longitude; } - public double Altitude { get; } - public double Latitude { get; } - public double Longitude { get; } + public double Altitude { get; set; } + public double Latitude { get; set; } + public double Longitude { get; set; } } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs index 6f9d49a4..b407236b 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.Common/UserAccount.cs @@ -7,12 +7,14 @@ namespace Fabrikam.DroneDelivery.Common { public class UserAccount { + public UserAccount() {} + public UserAccount(string userid, string accountid) { UserId = userid; AccountId = accountid; } - public string UserId { get; } - public string AccountId { get; } + public string UserId { get; set;} + public string AccountId { get; set;} } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs index a69b6ee6..9b99f947 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/DeliveriesControllerRoutingFixture.cs @@ -39,14 +39,20 @@ public DeliveriesControllerRoutingFixture() _testServer = new TestServer( new WebHostBuilder() + .UseTestServer() .Configure(builder => { builder.UseGlobalExceptionHandler(); - builder.UseMvc(); + + builder.UseRouting(); + builder.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); }) .ConfigureServices(builder => { - builder.AddMvc(); + builder.AddControllers(); builder.AddSingleton(_deliveryRepositoryMock.Object); builder.AddSingleton(_notifyMeRequestRepository.Object); @@ -136,7 +142,7 @@ public async Task PutDelivery_GetsResponse() } [TestMethod] - public async Task PutDeliveryThroughPublicRoute_GetsNotFoundResponse() + public async Task PutDeliveryThroughPublicRoute_GetsMethodNotAllowedResponse() { var deliveryId = Guid.NewGuid().ToString(); var delivery = new Delivery(deliveryId, new UserAccount("user", "accound"), null, null, null, false, ConfirmationType.None, null); @@ -144,7 +150,7 @@ public async Task PutDeliveryThroughPublicRoute_GetsNotFoundResponse() using (var client = _testServer.CreateClient()) { var response = await client.PutAsJsonAsync($"http://localhost/api/deliveries/public/{deliveryId}", delivery); - Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); + Assert.AreEqual(HttpStatusCode.MethodNotAllowed, response.StatusCode); } _deliveryRepositoryMock.VerifyAll(); diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj index 9e2c8428..dcf23957 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/Fabrikam.DroneDelivery.DeliveryService.Tests.csproj @@ -1,23 +1,22 @@ - + - - netcoreapp2.2 + + netcoreapp3.1 + false - - - - - - - - - - - + + + + + + + + + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs index 172529e0..17acf30c 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService.Tests/GlobalLoggerMiddlewareFixture.cs @@ -70,12 +70,12 @@ public async Task IfHandledInternalServerError_ItLogsError() await logRequestMiddleware.Invoke(contextMock.Object); // Assert - loggerMock.Verify(l => l.Log( + loggerMock.Verify(l => l.Log( LogLevel.Error, It.IsAny(), - It.Is(fV => fV.ToString().Equals(($"An internal handled exception has occurred: {exMessage}"))), + It.Is((object fV, Type _) => fV.ToString().Equals(($"An internal handled exception has occurred: {exMessage}"))), It.IsAny(), - It.IsAny>())); + (Func)It.IsAny())); } [TestMethod] @@ -117,12 +117,12 @@ public async Task IfHandledClientError_ItLogsError() await logRequestMiddleware.Invoke(contextMock.Object); // Assert - loggerMock.Verify(l => l.Log( + loggerMock.Verify(l => l.Log( LogLevel.Error, It.IsAny(), - It.Is(fV => fV.ToString().Equals(($"An error has occurred: 499"))), + It.Is((object fV, Type _) => fV.ToString().Equals(($"An error has occurred: 499"))), null, - It.IsAny>())); + (Func)It.IsAny())); } [TestMethod] @@ -164,12 +164,12 @@ public async Task IfTooManyRequestIsHandled_ItLogsError() await logRequestMiddleware.Invoke(contextMock.Object); // Assert - loggerMock.Verify(l => l.Log( + loggerMock.Verify(l => l.Log( LogLevel.Error, It.IsAny(), - It.Is(fV => fV.ToString().Equals(($"An error has occurred: 429"))), + It.Is((object fV, Type _) => fV.ToString().Equals(($"An error has occurred: 429"))), null, - It.IsAny>())); + (Func)It.IsAny())); } [TestMethod] @@ -213,24 +213,23 @@ public async Task IfUnhandledExceptionWhileResponding_ItLogsErrorPlusWarningAndR try { await logRequestMiddleware.Invoke(contextMock.Object); - } // Assert catch (Exception) { - loggerMock.Verify(l => l.Log( + loggerMock.Verify(l => l.Log( LogLevel.Error, It.IsAny(), - It.Is(fV => fV.ToString().Equals(($"An exception was thrown attempting to execute the global internal server error handler: {exMessage}"))), + It.Is((object fV, Type _) => fV.ToString().Equals(($"An exception was thrown attempting to execute the global internal server error handler: {exMessage}"))), It.IsAny(), - It.IsAny>())); + (Func)It.IsAny())); - loggerMock.Verify(l => l.Log( + loggerMock.Verify(l => l.Log( LogLevel.Warning, It.IsAny(), - It.Is(fV => fV.ToString().Equals(("The response has already started, the error handler will not be executed."))), + It.Is((object fV, Type _) => fV.ToString().Equals(("The response has already started, the error handler will not be executed."))), null, - It.IsAny>())); + (Func)It.IsAny())); // re-throw the actual re-throw throw; } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs index 13bd3f72..59107998 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Controllers/DeliveriesController.cs @@ -16,8 +16,9 @@ namespace Fabrikam.DroneDelivery.DeliveryService.Controllers { + [ApiController] [Route("api/[controller]")] - public class DeliveriesController : Controller + public class DeliveriesController : ControllerBase { private readonly IDeliveryRepository deliveryRepository; private readonly INotifyMeRequestRepository notifyMeRequestRepository; diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj index 70a3a033..9d0d3031 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Fabrikam.DroneDelivery.DeliveryService.csproj @@ -1,8 +1,7 @@  - netcoreapp2.2 - ..\docker-compose.dcproj + netcoreapp3.1 @@ -11,9 +10,8 @@ - + - @@ -23,7 +21,7 @@ - + diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs index e9b9e749..f2f4cfc3 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Models/Delivery.cs @@ -3,13 +3,14 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using System.Collections.ObjectModel; using Fabrikam.DroneDelivery.Common; namespace Fabrikam.DroneDelivery.DeliveryService.Models { public class Delivery { + public Delivery() {} + public Delivery(string id, UserAccount owner, Location pickup, @@ -29,13 +30,13 @@ public Delivery(string id, DroneId = droneid; } - public string Id { get; } - public UserAccount Owner { get; } - public Location Pickup { get; } - public Location Dropoff { get; } - public string Deadline { get; } - public bool Expedited { get; } - public ConfirmationType ConfirmationRequired { get; } - public string DroneId { get; } + public string Id { get; set;} + public UserAccount Owner { get; set;} + public Location Pickup { get; set;} + public Location Dropoff { get; set;} + public string Deadline { get; set;} + public bool Expedited { get; set;} + public ConfirmationType ConfirmationRequired { get; set;} + public string DroneId { get; set;} } } diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs index 35ca56da..c592c4c7 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Program.cs @@ -3,10 +3,9 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; @@ -16,21 +15,25 @@ public class Program { public static void Main(string[] args) { - var host = CreateWebHostBuilder(args).Build(); + var host = CreateHostBuilder(args).Build(); var logger = host.Services.GetRequiredService>(); logger.LogInformation("Fabrikam Delivery Service is starting."); host.Run(); } - public static IWebHostBuilder CreateWebHostBuilder(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup() - .ConfigureLogging((hostingContext, loggingBuilder) => + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => { - loggingBuilder.AddApplicationInsights(); - loggingBuilder.AddSerilog(dispose: true); - }) - .UseUrls("http://0.0.0.0:8080"); + webBuilder + .UseStartup() + .ConfigureLogging((hostingContext, loggingBuilder) => + { + loggingBuilder.AddApplicationInsights(); + loggingBuilder.AddSerilog(dispose: true); + }) + .UseUrls("http://0.0.0.0:8080"); + }); } -} +} \ No newline at end of file diff --git a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs index 57d4e325..c604aa30 100644 --- a/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs +++ b/src/shipping/delivery/Fabrikam.DroneDelivery.DeliveryService/Startup.cs @@ -4,15 +4,14 @@ // ------------------------------------------------------------ using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Swashbuckle.AspNetCore.Swagger; +using Microsoft.OpenApi.Models; using Fabrikam.DroneDelivery.DeliveryService.Models; using Fabrikam.DroneDelivery.DeliveryService.Services; using Fabrikam.DroneDelivery.DeliveryService.Middlewares.Builder; @@ -25,7 +24,7 @@ public class Startup { private const string HealCheckName = "ReadinessLiveness"; - public Startup(IHostingEnvironment env) + public Startup(IWebHostEnvironment env) { var builder = new ConfigurationBuilder() .SetBasePath(env.ContentRootPath) @@ -54,18 +53,17 @@ public void ConfigureServices(IServiceCollection services) services.AddApplicationInsightsKubernetesEnricher(); services.AddApplicationInsightsTelemetry(Configuration); - // Add framework services. - services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); - // Add health check services.AddHealthChecks().AddCheck( HealCheckName, () => HealthCheckResult.Healthy("OK")); + services.AddControllers(); + // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => { - c.SwaggerDoc("v1", new Info { Title = "Fabrikam DroneDelivery DeliveryService API", Version = "v1" }); + c.SwaggerDoc("v1", new OpenApiInfo { Title = "Fabrikam DroneDelivery DeliveryService API", Version = "v1" }); }); services.AddSingleton(); @@ -75,7 +73,7 @@ public void ConfigureServices(IServiceCollection services) } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) + public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) { Log.Logger = new LoggerConfiguration() .WriteTo.Console(new CompactJsonFormatter()) @@ -88,12 +86,14 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF // Important: it has to be second: Enable global exception, error handling app.UseGlobalExceptionHandler(); - // Map health checks - app.UseHealthChecks("/healthz"); - // TODO: Add middleware AuthZ here - app.UseMvc(); + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapHealthChecks("/healthz"); + endpoints.MapControllers(); + }); // Enable middleware to serve generated Swagger as a JSON endpoint. app.UseSwagger(); From c113277e8e3f0a71d188d6ddaa9c79e7fd5be601 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Fri, 24 Jan 2020 16:58:05 +0000 Subject: [PATCH 224/246] Merged PR 914: migrate Fabrikam.DroneDelivery.DroneSchedulerService to ASP.NET Core 3.1 - migrate Fabrikam.DroneDelivery.DroneSchedulerService to .NET Core 3.1 - upgrade Swashbuckle AspNetCore to 5.0.0. For more information, https://github.com/dotnet/aspnetcore/issues/7220 - migrate Fabrikam.DroneDelivery.DroneSchedulerService.Tests to .NET Core 3.1 - migrate Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport to .NET Core 3.1 solved: #160882 --- src/shipping/dronescheduler/Dockerfile | 6 +-- ...ry.DroneSchedulerService.BulkImport.csproj | 2 +- .../CustomWebApplicationFactory.cs | 7 ++-- ...elivery.DroneSchedulerService.Tests.csproj | 34 ++++----------- .../Controllers/DroneDeliveriesController.cs | 5 ++- ...DroneDelivery.DroneSchedulerService.csproj | 10 ++--- .../Program.cs | 41 ++++++++++--------- .../Startup.cs | 21 +++++----- 8 files changed, 55 insertions(+), 71 deletions(-) diff --git a/src/shipping/dronescheduler/Dockerfile b/src/shipping/dronescheduler/Dockerfile index 5ee44111..d0dcb15a 100644 --- a/src/shipping/dronescheduler/Dockerfile +++ b/src/shipping/dronescheduler/Dockerfile @@ -1,8 +1,8 @@ -FROM microsoft/dotnet:2.2-aspnetcore-runtime as base +FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 as base WORKDIR /app EXPOSE 80 -FROM microsoft/dotnet:2.2-sdk AS build +FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build MAINTAINER Fernando Antivero (https://github.com/ferantivero) @@ -22,7 +22,7 @@ FROM build AS publish MAINTAINER Fernando Antivero (https://github.com/ferantivero) WORKDIR /app -RUN dotnet publish /app/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ -c Release -o ../../out +RUN dotnet publish /app/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/ -c release -o ./out --no-restore FROM base AS runtime diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj index 013cdf20..e847f7ff 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport/Fabrikam.DroneDelivery.DroneSchedulerService.BulkImport.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.2 + netcoreapp3.1 7.1 diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs index 83cd6968..eccdba2e 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/CustomWebApplicationFactory.cs @@ -7,16 +7,17 @@ using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Fabrikam.DroneDelivery.DroneSchedulerService.Tests { public class CustomWebApplicationFactory - : WebApplicationFactory + : WebApplicationFactory { protected override void ConfigureWebHost(IWebHostBuilder builder) - { - builder + { + builder .UseContentRoot(".") .UseEnvironment("Test") .ConfigureTestServices(s => diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj index 7f10d231..180e86d3 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService.Tests/Fabrikam.DroneDelivery.DroneSchedulerService.Tests.csproj @@ -1,37 +1,19 @@ - + - netcoreapp2.2 + netcoreapp3.1 false - - - - - - - - - - - - + + + + + - - - - - PreserveNewest - - - - - - PreserveNewest - + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs index 607ac985..5c8a78ca 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Controllers/DroneDeliveriesController.cs @@ -13,8 +13,9 @@ namespace Fabrikam.DroneDelivery.DroneSchedulerService.Controllers { + [ApiController] [Route("api/[controller]")] - public class DroneDeliveriesController : Controller + public class DroneDeliveriesController : ControllerBase { private readonly ILogger logger; private readonly IInvoicingRepository _invoicingRepository; @@ -76,4 +77,4 @@ public async Task GetDroneUtilization([FromQuery] string ownerId, }); } } -} \ No newline at end of file +} diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj index 03919a13..5e5645a5 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Fabrikam.DroneDelivery.DroneSchedulerService.csproj @@ -1,8 +1,7 @@  - netcoreapp2.2 - ..\docker-compose.dcproj + netcoreapp3.1 Fabrikam.DroneDelivery.DroneSchedulerService Fabrikam.DroneDelivery.DroneSchedulerService @@ -13,22 +12,21 @@ - + - - + - + diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs index 2ed62f07..a53f62f4 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Program.cs @@ -3,10 +3,10 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; @@ -16,31 +16,34 @@ public class Program { public static void Main(string[] args) { - var host = CreateWebHostBuilder(args).Build(); + var host = CreateHostBuilder(args).Build(); var logger = host.Services.GetRequiredService>(); logger.LogInformation("Fabrikam Dronescheduler Service is starting."); host.Run(); } - public static IWebHostBuilder CreateWebHostBuilder(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup() - .ConfigureAppConfiguration(configurationBuilder => + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => { - var buildConfig = configurationBuilder.Build(); - - if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) - { - configurationBuilder.AddAzureKeyVault(keyVaultUri); - } - }) - .ConfigureLogging((hostingContext, loggingBuilder) => - { - loggingBuilder.AddApplicationInsights(); - loggingBuilder.AddSerilog(dispose: true); - }) - .UseUrls("http://0.0.0.0:8080"); + webBuilder + .UseStartup() + .ConfigureAppConfiguration(configurationBuilder => + { + var buildConfig = configurationBuilder.Build(); + if (buildConfig["KEY_VAULT_URI"] is var keyVaultUri && !string.IsNullOrEmpty(keyVaultUri)) + { + configurationBuilder.AddAzureKeyVault(keyVaultUri); + } + }) + .ConfigureLogging((hostingContext, loggingBuilder) => + { + loggingBuilder.AddApplicationInsights(); + loggingBuilder.AddSerilog(dispose: true); + }) + .UseUrls("http://0.0.0.0:8080"); + }); } } diff --git a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs index a127d59b..b1cd4c34 100644 --- a/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs +++ b/src/shipping/dronescheduler/Fabrikam.DroneDelivery.DroneSchedulerService/Startup.cs @@ -4,20 +4,17 @@ // ------------------------------------------------------------ using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Diagnostics.HealthChecks; -using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; using Microsoft.FeatureManagement; +using Microsoft.OpenApi.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Models; using Fabrikam.DroneDelivery.DroneSchedulerService.Services; using Serilog; using Serilog.Formatting.Compact; -using Swashbuckle.AspNetCore.Swagger; namespace Fabrikam.DroneDelivery.DroneSchedulerService { @@ -44,7 +41,7 @@ public void ConfigureServices(IServiceCollection services) services.AddApplicationInsightsTelemetry(Configuration); // Add framework services. - services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); + services.AddControllers(); // Add health check services.AddHealthChecks().AddCheck( @@ -54,7 +51,7 @@ public void ConfigureServices(IServiceCollection services) // Register the Swagger generator, defining one or more Swagger documents services.AddSwaggerGen(c => { - c.SwaggerDoc("v1", new Info { Title = "Mock DroneScheduler API", Version = "v1" }); + c.SwaggerDoc("v1", new OpenApiInfo { Title = "Mock DroneScheduler API", Version = "v1" }); }); services.AddSingleton(); @@ -64,17 +61,19 @@ public void ConfigureServices(IServiceCollection services) } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) + public void Configure(IApplicationBuilder app, ILoggerFactory loggerFactory, IHttpContextAccessor httpContextAccessor) { Log.Logger = new LoggerConfiguration() .WriteTo.Console(new CompactJsonFormatter()) .ReadFrom.Configuration(Configuration) .CreateLogger(); - // Map health checks - app.UseHealthChecks("/healthz"); - - app.UseMvc(); + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapHealthChecks("/healthz"); + endpoints.MapControllers(); + }); // Enable middleware to serve generated Swagger as a JSON endpoint. app.UseSwagger(); From 0af8ca1c07ff618e0930f7f56cb6e03810b72f40 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Fri, 24 Jan 2020 16:58:56 +0000 Subject: [PATCH 225/246] Merged PR 915: migrate Fabrikam.DroneDelivery.Workflow.Service to .NET Core 3.1 - migrate Fabrikam.DroneDelivery.Workflow.Service to .NET Core 3.1 - migrate Fabrikam.DroneDelivery.Workflow.Service.Tests to .NET Core 3.1 - allow sync calls using testserver for Netwonsoft deserialization - update docker images + cli commands - solve InvalidOperationException, Reading is not allowed after reader was completed. solved: #160881 --- src/shipping/workflow/Dockerfile | 8 ++--- .../DeliveryServiceCallerIntegrationTests.cs | 11 +++---- ...eSchedulerServiceCallerIntegrationTests.cs | 7 +++-- .../Fabrikam.Workflow.Service.Tests.csproj | 18 +++++------ .../PackageServiceCallerIntegrationTests.cs | 22 +++++++------- .../RequestProcessorIntegrationTests.cs | 7 +++-- .../Fabrikam.Workflow.Service.csproj | 30 +++++++++---------- 7 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/shipping/workflow/Dockerfile b/src/shipping/workflow/Dockerfile index b3f6f0b1..55945e01 100644 --- a/src/shipping/workflow/Dockerfile +++ b/src/shipping/workflow/Dockerfile @@ -1,14 +1,14 @@ -FROM microsoft/dotnet:2.2-runtime AS base +FROM mcr.microsoft.com/dotnet/core/runtime:3.1 AS base WORKDIR /app -FROM microsoft/dotnet:2.2-sdk AS build +FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build WORKDIR /src/Fabrikam.Workflow.Service COPY Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj . RUN dotnet restore Fabrikam.Workflow.Service.csproj COPY Fabrikam.Workflow.Service/. . -RUN dotnet build Fabrikam.Workflow.Service.csproj -c Release -o /app +RUN dotnet build Fabrikam.Workflow.Service.csproj -c release -o /app --no-restore FROM build AS testrunner WORKDIR /src/tests @@ -25,4 +25,4 @@ RUN dotnet publish Fabrikam.Workflow.Service.csproj -c Release -o /app FROM base AS final WORKDIR /app COPY --from=publish /app . -ENTRYPOINT ["dotnet", "Fabrikam.Workflow.Service.dll"] +ENTRYPOINT ["dotnet", "Fabrikam.Workflow.Service.dll"] \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs index bfbe43f4..92f095f5 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DeliveryServiceCallerIntegrationTests.cs @@ -47,7 +47,7 @@ public DeliveryServiceCallerIntegrationTests() .AddEnvironmentVariables() .Build(); context.HostingEnvironment = - Mock.Of(e => e.EnvironmentName == "Test"); + Mock.Of(e => e.EnvironmentName == "Test"); var serviceCollection = new ServiceCollection(); ServiceStartup.ConfigureServices(context, serviceCollection); @@ -55,16 +55,17 @@ public DeliveryServiceCallerIntegrationTests() _testServer = new TestServer( - new WebHostBuilder() + new WebHostBuilder() + .UseTestServer() .Configure(builder => { - builder.UseMvc(); builder.Run(ctx => _handleHttpRequest(ctx)); }) .ConfigureServices(builder => { - builder.AddMvc(); + builder.AddControllers(); })); + _testServer.AllowSynchronousIO = true; serviceCollection.Replace( ServiceDescriptor.Transient( @@ -179,4 +180,4 @@ public async Task WhenPackageAPIDoesNotReturnOK_ThenThrows() await Assert.ThrowsAsync(() => _caller.ScheduleDeliveryAsync(delivery, "someDroneId")); } } -} \ No newline at end of file +} diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs index 4e3a54a1..c462c6f4 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/DroneSchedulerServiceCallerIntegrationTests.cs @@ -47,7 +47,7 @@ public DroneSchedulerServiceCallerIntegrationTests() .AddEnvironmentVariables() .Build(); context.HostingEnvironment = - Mock.Of(e => e.EnvironmentName == "Test"); + Mock.Of(e => e.EnvironmentName == "Test"); var serviceCollection = new ServiceCollection(); ServiceStartup.ConfigureServices(context, serviceCollection); @@ -56,15 +56,16 @@ public DroneSchedulerServiceCallerIntegrationTests() _testServer = new TestServer( new WebHostBuilder() + .UseTestServer() .Configure(builder => { - builder.UseMvc(); builder.Run(ctx => _handleHttpRequest(ctx)); }) .ConfigureServices(builder => { - builder.AddMvc(); + builder.AddControllers(); })); + _testServer.AllowSynchronousIO = true; serviceCollection.Replace( ServiceDescriptor.Transient( diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj index bceb3d78..7324d1b6 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/Fabrikam.Workflow.Service.Tests.csproj @@ -1,27 +1,25 @@ - + - netcoreapp2.2 - + netcoreapp3.1 false - - - - - - + + + + all runtime; build; native; contentfiles; analyzers + - + \ No newline at end of file diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs index 539dc038..8bc7cc9f 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/PackageServiceCallerIntegrationTests.cs @@ -49,7 +49,7 @@ public PackageServiceCallerIntegrationTests() .AddEnvironmentVariables() .Build(); context.HostingEnvironment = - Mock.Of(e => e.EnvironmentName == "Test"); + Mock.Of(e => e.EnvironmentName == "Test"); var serviceCollection = new ServiceCollection(); ServiceStartup.ConfigureServices(context, serviceCollection); @@ -58,15 +58,16 @@ public PackageServiceCallerIntegrationTests() _testServer = new TestServer( new WebHostBuilder() + .UseTestServer() .Configure(builder => { - builder.UseMvc(); builder.Run(ctx => _handleHttpRequest(ctx)); }) .ConfigureServices(builder => { - builder.AddMvc(); + builder.AddControllers(); })); + _testServer.AllowSynchronousIO = true; serviceCollection.Replace( ServiceDescriptor.Transient( @@ -121,26 +122,25 @@ public async Task WhenUpdatingPackage_ThenInvokesDroneSchedulerAPI() // Arrange string actualPackageId = null; PackageGen actualPackage = null; - Stream body = null; _handleHttpRequest = ctx => { if (ctx.Request.Host.Host == PackageHost && ctx.Request.Method.Equals("PUT")) { ctx.Response.StatusCode = StatusCodes.Status204NoContent; - body = ctx.Request.Body; - } - else if (ctx.Request.Host.Host == PackageHost && - ctx.Request.Method.Equals("GET")) - { - actualPackageId = ctx.Request.Path; actualPackage = new JsonSerializer() .Deserialize( new JsonTextReader( new StreamReader( - body, + ctx.Request.Body, Encoding.UTF8))); + + } + else if (ctx.Request.Host.Host == PackageHost && + ctx.Request.Method.Equals("GET")) + { + actualPackageId = ctx.Request.Path; } else { diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs index a07fc772..d4ac23d9 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs +++ b/src/shipping/workflow/Fabrikam.Workflow.Service.Tests/RequestProcessorIntegrationTests.cs @@ -56,7 +56,7 @@ public RequestProcessorIntegrationTests() .AddEnvironmentVariables() .Build(); context.HostingEnvironment = - Mock.Of(e => e.EnvironmentName == "Test"); + Mock.Of(e => e.EnvironmentName == "Test"); var serviceCollection = new ServiceCollection(); ServiceStartup.ConfigureServices(context, serviceCollection); @@ -65,15 +65,16 @@ public RequestProcessorIntegrationTests() _testServer = new TestServer( new WebHostBuilder() + .UseTestServer() .Configure(builder => { - builder.UseMvc(); builder.Run(ctx => _handleHttpRequest(ctx)); }) .ConfigureServices(builder => { - builder.AddMvc(); + builder.AddControllers(); })); + _testServer.AllowSynchronousIO = true; serviceCollection.Replace( ServiceDescriptor.Transient( diff --git a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj index ce5a6eae..ce944744 100644 --- a/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj +++ b/src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj @@ -2,26 +2,26 @@ Exe - netcoreapp2.2 + netcoreapp3.1 7.1 - - + + - - - - - - - + + + + + + + @@ -29,10 +29,10 @@ - - - PreserveNewest - - + + + PreserveNewest + + From 56537dcbc6b90c5272467717644731e15553f5a6 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 24 Jan 2020 10:05:13 -0700 Subject: [PATCH 226/246] add comparison table between advanced and basic versions solved: #163250 --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b75d75e2..21b531c9 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,21 @@ Microsoft patterns & practices This reference implementation shows a set of best practices for building and running a microservices architecture on Microsoft Azure, using Kubernetes. +| | [Basic](https://github.com/mspnp/microservices-reference-implementation/tree/basic) | [Advanced](https://github.com/mspnp/microservices-reference-implementation/) | +|-------------------------------------|-------|----------| +| Distributed Monitoring | ✅ | ✅ | +| Ingress Controller | ✅ | ✅ | +| Azure Active Directory Pod Identity | ✅ | ✅ | +| CI/CD | ✅ | ✅ | +| Helm charts | ✅ | ✅ | +| Resource Limits | ✅ | ✅ | +| Readiness/Liveness Probes | ✅ | ✅ | +| Horizontal Pod Autoscaling | ❌ | ✅ | +| Cluster Autoscaling | ❌ | ✅ | +| Advanced Networking | ❌ | ✅ | +| Network Policies | ❌ | ✅ | +| Egress Lockdown | ❌ | ✅ | + ## Guidance This project has a companion set of articles that describe challenges, design patterns, and best practices for building microservices architecture. You can find these articles on the Azure Architecture Center: @@ -49,7 +64,7 @@ The Drone Delivery application has been tested up to 2000 messages/sec: 1. Serve: Visual Studio Load Test Throughout Request/Sec 2. Ingest: Azure Service Bus metrics Incoming Messages/Sec 3. Egress: Azure Service Bus metrics Outgoing Messages/Sec -4. Complete: AI Service Bus Complete dependencies +4. Complete: AI Service Bus Complete dependencies 5. Avg/50th/95th/99th: AI dependencies 6. CPU/Mem: Azure Monitor for Containers From 4a3a6ab754100326e0680f45232affb112f78af0 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Mon, 27 Jan 2020 06:25:19 -0700 Subject: [PATCH 227/246] adress team's feedback - list service endpoints - imporove egress description - improve ci/cd --- README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 21b531c9..e6e0992c 100644 --- a/README.md +++ b/README.md @@ -3,20 +3,21 @@ Microsoft patterns & practices This reference implementation shows a set of best practices for building and running a microservices architecture on Microsoft Azure, using Kubernetes. -| | [Basic](https://github.com/mspnp/microservices-reference-implementation/tree/basic) | [Advanced](https://github.com/mspnp/microservices-reference-implementation/) | -|-------------------------------------|-------|----------| -| Distributed Monitoring | ✅ | ✅ | -| Ingress Controller | ✅ | ✅ | -| Azure Active Directory Pod Identity | ✅ | ✅ | -| CI/CD | ✅ | ✅ | -| Helm charts | ✅ | ✅ | -| Resource Limits | ✅ | ✅ | -| Readiness/Liveness Probes | ✅ | ✅ | -| Horizontal Pod Autoscaling | ❌ | ✅ | -| Cluster Autoscaling | ❌ | ✅ | -| Advanced Networking | ❌ | ✅ | -| Network Policies | ❌ | ✅ | -| Egress Lockdown | ❌ | ✅ | +| | [Basic](https://github.com/mspnp/microservices-reference-implementation/tree/basic) | [Advanced](https://github.com/mspnp/microservices-reference-implementation/) | +|-----------------------------------------|-------|----------| +| Distributed Monitoring | ✅ | ✅ | +| Ingress Controller | ✅ | ✅ | +| Azure Active Directory Pod Identity | ✅ | ✅ | +| CI/CD using Azure Pipelines | ✅ | ✅ | +| Helm charts | ✅ | ✅ | +| Resource Limits | ✅ | ✅ | +| Readiness/Liveness Probes | ✅ | ✅ | +| Horizontal Pod Autoscaling | ❌ | ✅ | +| Cluster Autoscaling | ❌ | ✅ | +| Advanced Networking | ❌ | ✅ | +| Service Endpoints | ❌ | ✅ | +| Network Policies | ❌ | ✅ | +| Egress restriction using Azure Firewall | ❌ | ✅ | ## Guidance From 69a8f634a35e77f7b055830e505cb1dd222d55ca Mon Sep 17 00:00:00 2001 From: Michael Maxwell Date: Wed, 29 Jan 2020 09:14:05 -0800 Subject: [PATCH 228/246] Fixed typo (#149) --- src/shipping/delivery/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml index fad33c12..406fe058 100644 --- a/src/shipping/delivery/azure-pipelines.yml +++ b/src/shipping/delivery/azure-pipelines.yml @@ -25,7 +25,7 @@ trigger: include: # for new release to production: release flow strategy - release/delivery/v* - - refs/relelase/delivery/v* + - refs/release/delivery/v* - master - feature/delivery/* - topic/delivery/* From 186b94a7f20124224167b100b2cb86ae0d1f7f01 Mon Sep 17 00:00:00 2001 From: Fernando Simonazzi Date: Fri, 31 Jan 2020 14:25:34 +0000 Subject: [PATCH 229/246] Merged PR 909: change service bus to premium, set up firewalls for service bus and redis cache Changes service bus SKU to premium to support service endpoints. https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-service-endpoints Enables service endpoints for ServiceBus in the Azure Firewall subnet, and allows traffic from that subnet in the namespace's network rule set. Removes partitioning for the service bus queue as it's not supported in Premium https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-partitioning Narrows down the firewall rule for port 5671 to the ServiceBus tag instead of AzureCloud, as it's only available for premium SKU https://docs.microsoft.com/en-us/azure/virtual-network/service-tags-overview#available-service-tags Sets up firewall rules in the REDIS cache to allow traffic from the Azure Firewall public IP address only. --- azuredeploy-firewall.json | 2 +- azuredeploy.json | 46 +++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/azuredeploy-firewall.json b/azuredeploy-firewall.json index ad5d220b..754152df 100644 --- a/azuredeploy-firewall.json +++ b/azuredeploy-firewall.json @@ -192,7 +192,7 @@ "[parameters('aksClusterSubnetPrefix')]" ], "destinationAddresses": [ - "[concat('AzureCloud.',parameters('serviceTagsLocation'))]" + "[concat('ServiceBus.',parameters('serviceTagsLocation'))]" ], "sourceIpGroups": [], "destinationIpGroups": [], diff --git a/azuredeploy.json b/azuredeploy.json index 04840566..d69ee5bc 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -267,6 +267,8 @@ "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespaceSKU": "Premium", + "ingestionSBNamespaceTier": "Premium", "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", @@ -304,6 +306,8 @@ "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespaceSKU": "Premium", + "ingestionSBNamespaceTier": "Premium", "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", @@ -341,6 +345,8 @@ "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespaceSKU": "Premium", + "ingestionSBNamespaceTier": "Premium", "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", @@ -378,6 +384,8 @@ "deliveryKeyVaultName": "[concat(parameters('environmentName'),'-d-',uniqueString(resourceGroup().id))]", "packageMongoDbName": "[concat(parameters('environmentName'),'-p-',uniqueString(resourceGroup().id))]", "ingestionSBNamespace": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", + "ingestionSBNamespaceSKU": "Premium", + "ingestionSBNamespaceTier": "Premium", "ingestionSBName": "[concat(parameters('environmentName'),'-i-',uniqueString(resourceGroup().id))]", "ingestionServiceAccessKey": "IngestionServiceAccessKey", "droneSchedulerKeyVaultName": "[concat(parameters('environmentName'),'-ds-',uniqueString(resourceGroup().id))]", @@ -500,6 +508,12 @@ "locations": [ "[resourceGroup().location]" ] + }, + { + "service": "Microsoft.ServiceBus", + "locations": [ + "[resourceGroup().location]" + ] } ] } @@ -764,6 +778,19 @@ } }, "resources": [ + { + "name": "firewallAccess", + "type": "firewallRules", + "apiVersion": "2018-03-01", + "dependsOn": [ + "[resourceId('Microsoft.Cache/Redis', variables('environmentSettings')[parameters('environmentName')].deliveryRedisName)]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))]" + ], + "properties": { + "startIP": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]", + "endIP": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('firewallPublicIpName'))).ipAddress]" + } + }, { "apiVersion": "2017-05-01-preview", "type": "Microsoft.Cache/redis/providers/diagnosticsettings", @@ -875,8 +902,8 @@ "apiVersion": "2017-04-01", "location": "[resourceGroup().location]", "sku": { - "name": "Standard", - "tier": "Standard" + "name": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespaceSKU]", + "tier": "[variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespaceTier]" }, "tags": { "displayName": "Ingestion and Workflow Service Bus", @@ -895,8 +922,7 @@ ], "properties": { "lockDuration": "PT5M", - "maxSizeInMegabytes": "1024", - "enablePartitioning": "true" + "maxSizeInMegabytes": "1024" } }, { @@ -924,6 +950,18 @@ "dependsOn": [ "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]" ] + }, + { + "name": "firewall", + "type": "VirtualNetworkRules", + "apiVersion": "2018-01-01-preview", + "properties": { + "virtualNetworkSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('firewallSubnetName'))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.ServiceBus/namespaces', variables('environmentSettings')[parameters('environmentName')].ingestionSBNamespace)]", + "[resourceId('Microsoft.Network/virtualNetworks', variables('environmentSettings')[parameters('environmentName')].aksVnetName)]" + ] } ] }, From c643e284e50804a7476d64c35cb7dc77eef0adb2 Mon Sep 17 00:00:00 2001 From: ferantivero Date: Fri, 31 Jan 2020 07:41:11 -0700 Subject: [PATCH 230/246] remove perf results, since are not valid for advanced --- README.md | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/README.md b/README.md index e6e0992c..2bdcc2c1 100644 --- a/README.md +++ b/README.md @@ -45,31 +45,6 @@ The Drone Delivery application is a sample application that consists of several ![](./architecture.png) -## Test results and metrics -The Drone Delivery application has been tested up to 2000 messages/sec: - - -| | Replicas | ~Max CPU (mc) | ~Max Mem (MB) | Avg. Throughput*  | Max. Throughput* | Avg (ms) | 50th (ms) | 95th (ms) | 99th (ms) | -|------------------------------------------|----------|---------------|---------------|-------------------------|-------------------------|----------|-----------|-----------|-----------| -| Nginx | 1 | N/A | N/A | serve: 1595 reqs/sec | serve: 1923 reqs/sec | N/A | N/A | N/A | N/A | -| Ingestion | 10 | 474 | 488 | ingest: 1275 msgs/sec | ingest: 1710 msgs/sec | 251 | 50.1 | 1560 | 2540 | -| Workflow (receive messages) | 35 | 1445 | 79 | egress: 1275 msgs/sec | egress: 1710 msgs/sec | 81.5 | 0 | 25.7 | 121 | -| Workflow (call backend services + mark message as complete) | 35 | 1445 | 79 | complete: 1100 msgs/sec | complete: 1322 msgs/sec | 561.8 | 447 | 1350 | 2540 | -| Package | 50 | 213 | 78 | N/A | N/A | 67.5 | 53.9 | 165 | 306 | -| Delivery | 50 | 328 | 334 | N/A | N/A | 93.8 | 82.4 | 200 | 304 | -| Dronescheduler | 50 | 402 | 301 | N/A | N/A | 85.9 | 72.6 | 203 | 308 | - - - -*sources: -1. Serve: Visual Studio Load Test Throughout Request/Sec -2. Ingest: Azure Service Bus metrics Incoming Messages/Sec -3. Egress: Azure Service Bus metrics Outgoing Messages/Sec -4. Complete: AI Service Bus Complete dependencies -5. Avg/50th/95th/99th: AI dependencies -6. CPU/Mem: Azure Monitor for Containers - - ## Deployment To deploy the solution, follow the steps listed [here](./deployment.md). From d34b1d29e8943ed8decedc576bee1f4df2213c51 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Wed, 26 Feb 2020 16:28:18 +0000 Subject: [PATCH 231/246] Merged PR 21314: helm 3 migration - install latest helm version 3 and remove tiller instructions - add clean up for Fabrikam Drone Delivery Apps - migrate delivery chart - migrate drone chart - migrate package chart - migrate ingestion chart - migrate workflow chart solved: #160868, #160869, #160871, #160873, #160872, #160870 --- charts/delivery/.helmignore | 22 +++++ charts/delivery/Chart.yaml | 36 ++++++++ charts/delivery/envs/delivery-dev/Chart.yaml | 2 + charts/delivery/envs/delivery-prod/Chart.yaml | 2 + charts/delivery/envs/delivery-qa/Chart.yaml | 2 + .../delivery/envs/delivery-staging/Chart.yaml | 2 + charts/delivery/requirements.lock | 15 ---- charts/delivery/requirements.yaml | 32 ------- charts/delivery/templates/NOTES.txt | 6 +- charts/delivery/templates/_helpers.tpl | 21 +++++ charts/delivery/values.yaml | 34 +++++++- charts/dronescheduler/.helmignore | 22 +++++ charts/dronescheduler/Chart.yaml | 38 ++++++++ .../envs/dronescheduler-dev/Chart.yaml | 3 +- .../envs/dronescheduler-prod/Chart.yaml | 2 + .../envs/dronescheduler-qa/Chart.yaml | 2 + .../envs/dronescheduler-staging/Chart.yaml | 2 + charts/dronescheduler/requirements.lock | 15 ---- charts/dronescheduler/requirements.yaml | 32 ------- charts/dronescheduler/templates/NOTES.txt | 6 +- charts/dronescheduler/templates/_helpers.tpl | 21 +++++ .../templates/dronescheduler-deploy.yaml | 6 +- charts/dronescheduler/values.yaml | 36 +++++++- charts/ingestion/.helmignore | 22 +++++ charts/ingestion/Chart.yaml | 36 ++++++++ .../ingestion/envs/ingestion-dev/Chart.yaml | 2 + .../ingestion/envs/ingestion-prod/Chart.yaml | 2 + charts/ingestion/envs/ingestion-qa/Chart.yaml | 2 + .../envs/ingestion-staging/Chart.yaml | 2 + charts/ingestion/requirements.lock | 15 ---- charts/ingestion/requirements.yaml | 32 ------- charts/ingestion/templates/NOTES.txt | 6 +- charts/ingestion/templates/_helpers.tpl | 21 +++++ charts/ingestion/values.yaml | 28 +++++- charts/package/.helmignore | 22 +++++ charts/package/Chart.yaml | 36 ++++++++ charts/package/envs/package-dev/Chart.yaml | 2 + charts/package/envs/package-prod/Chart.yaml | 2 + charts/package/envs/package-qa/Chart.yaml | 2 + .../package/envs/package-staging/Chart.yaml | 2 + charts/package/requirements.lock | 15 ---- charts/package/requirements.yaml | 32 ------- charts/package/templates/NOTES.txt | 6 +- charts/package/templates/_helpers.tpl | 20 +++++ charts/package/values.yaml | 32 ++++++- charts/workflow/.helmignore | 22 +++++ charts/workflow/Chart.yaml | 36 ++++++++ charts/workflow/envs/workflow-dev/Chart.yaml | 2 + charts/workflow/envs/workflow-prod/Chart.yaml | 2 + charts/workflow/envs/workflow-qa/Chart.yaml | 2 + .../workflow/envs/workflow-staging/Chart.yaml | 2 + charts/workflow/requirements.lock | 15 ---- charts/workflow/requirements.yaml | 32 ------- charts/workflow/templates/NOTES.txt | 2 +- charts/workflow/templates/_helpers.tpl | 20 +++++ charts/workflow/values.yaml | 34 +++++++- deployment.md | 86 ++++++++++--------- deploymentCICD.md | 27 +++--- 58 files changed, 666 insertions(+), 314 deletions(-) create mode 100644 charts/delivery/.helmignore delete mode 100644 charts/delivery/requirements.lock delete mode 100644 charts/delivery/requirements.yaml create mode 100644 charts/dronescheduler/.helmignore delete mode 100644 charts/dronescheduler/requirements.lock delete mode 100644 charts/dronescheduler/requirements.yaml create mode 100644 charts/ingestion/.helmignore delete mode 100644 charts/ingestion/requirements.lock delete mode 100644 charts/ingestion/requirements.yaml create mode 100644 charts/package/.helmignore delete mode 100644 charts/package/requirements.lock delete mode 100644 charts/package/requirements.yaml create mode 100644 charts/workflow/.helmignore delete mode 100644 charts/workflow/requirements.lock delete mode 100644 charts/workflow/requirements.yaml diff --git a/charts/delivery/.helmignore b/charts/delivery/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/delivery/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/delivery/Chart.yaml b/charts/delivery/Chart.yaml index dd231e35..98047987 100644 --- a/charts/delivery/Chart.yaml +++ b/charts/delivery/Chart.yaml @@ -1,4 +1,40 @@ +apiVersion: v2 name: delivery version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Service +type: application +home: https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/microservices/aks +sources: + - https://github.com/mspnp/microservices-reference-implementation +dependencies: + - name: delivery-dev + repository: "file://envs/delivery-dev" + version: v0.1.0 + condition: envs.dev + import-values: + - data + + - name: delivery-prod + repository: "file://envs/delivery-prod" + version: v0.1.0 + condition: envs.prod + import-values: + - data + + - name: delivery-qa + repository: "file://envs/delivery-qa" + version: v0.1.0 + condition: envs.qa + import-values: + - data + + - name: delivery-staging + repository: "file://envs/delivery-staging" + version: v0.1.0 + condition: envs.staging + import-values: + - data +maintainers: + - email: v-fean@microsoft.com + name: ferantivero diff --git a/charts/delivery/envs/delivery-dev/Chart.yaml b/charts/delivery/envs/delivery-dev/Chart.yaml index 7ad76f0b..821ec15b 100644 --- a/charts/delivery/envs/delivery-dev/Chart.yaml +++ b/charts/delivery/envs/delivery-dev/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: delivery-dev version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Service +type: application diff --git a/charts/delivery/envs/delivery-prod/Chart.yaml b/charts/delivery/envs/delivery-prod/Chart.yaml index 98682ddf..349560de 100644 --- a/charts/delivery/envs/delivery-prod/Chart.yaml +++ b/charts/delivery/envs/delivery-prod/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: delivery-prod version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Service +type: application diff --git a/charts/delivery/envs/delivery-qa/Chart.yaml b/charts/delivery/envs/delivery-qa/Chart.yaml index 8174cf05..f872f4cd 100644 --- a/charts/delivery/envs/delivery-qa/Chart.yaml +++ b/charts/delivery/envs/delivery-qa/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: delivery-qa version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Service +type: application diff --git a/charts/delivery/envs/delivery-staging/Chart.yaml b/charts/delivery/envs/delivery-staging/Chart.yaml index 69219341..ff47ca2c 100644 --- a/charts/delivery/envs/delivery-staging/Chart.yaml +++ b/charts/delivery/envs/delivery-staging/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: delivery-staging version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Service +type: application diff --git a/charts/delivery/requirements.lock b/charts/delivery/requirements.lock deleted file mode 100644 index 2b8be7d5..00000000 --- a/charts/delivery/requirements.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: delivery-dev - repository: file://envs/delivery-dev - version: v0.1.0 -- name: delivery-prod - repository: file://envs/delivery-prod - version: v0.1.0 -- name: delivery-qa - repository: file://envs/delivery-qa - version: v0.1.0 -- name: delivery-staging - repository: file://envs/delivery-staging - version: v0.1.0 -digest: sha256:8f25310facbaa5f8791424964158ceddd0f5da6994d316422eab26dd02e1e3ec -generated: "2019-10-22T07:45:01.368035081-06:00" diff --git a/charts/delivery/requirements.yaml b/charts/delivery/requirements.yaml deleted file mode 100644 index c73bd374..00000000 --- a/charts/delivery/requirements.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dependencies: - - name: delivery-dev - repository: "file://envs/delivery-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data - - - name: delivery-prod - repository: "file://envs/delivery-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data - - - name: delivery-qa - repository: "file://envs/delivery-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data - - - name: delivery-staging - repository: "file://envs/delivery-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data diff --git a/charts/delivery/templates/NOTES.txt b/charts/delivery/templates/NOTES.txt index c5dbfc98..68d120cf 100644 --- a/charts/delivery/templates/NOTES.txt +++ b/charts/delivery/templates/NOTES.txt @@ -2,9 +2,9 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -All the objects were created in the namespace {{ .Values.namespace }} +All the objects were created in the namespace {{ .Release.Namespace }} To learn more about the release, try: - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} + $ helm status {{ .Release.Name }} --namespace {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/charts/delivery/templates/_helpers.tpl b/charts/delivery/templates/_helpers.tpl index d02ec5d1..432c9ac8 100644 --- a/charts/delivery/templates/_helpers.tpl +++ b/charts/delivery/templates/_helpers.tpl @@ -38,3 +38,24 @@ Create chart name and version as used by the chart label. {{- define "delivery.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "delivery.labels" -}} +helm.sh/chart: {{ include "delivery.chart" . }} +{{ include "delivery.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "delivery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "delivery.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index d7434df8..d2b50d2b 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -1,20 +1,34 @@ # Default values for delivery. nameOverride: delivery + + replicaCount: 1 + + identity: clientid: resourceid: + + dockerregistrynamespace: dockerregistry: + + image: repository: tag: pullPolicy: IfNotPresent + + cosmosdb: id: collectionid: + keyvault: uri: + + +# probes readinessProbe: httpGet: path: /healthz @@ -29,20 +43,36 @@ livenessProbe: port: 8080 initialDelaySeconds: 50 periodSeconds: 15 + + telemetry: level: "Error" + + +# specify an installation/upgrade reason reason: unknown -tags: + + +# indicate what environment is meant to be installed/upgraded +envs: dev: false prod: false qa: false staging: false + + current: false + + +# Horizontal Pod Autoscaling autoscaling: enabled: false maxReplicas: minReplicas: targetCPUUtilizationPercentage: + + +# Pod-to-pod traffic: east-west networkPolicy: egress: enabled: true @@ -73,6 +103,8 @@ networkPolicy: dd.fabrikam.com/egress-delivery: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery + + service: targetPort: 8080 targetProtocol: TCP diff --git a/charts/dronescheduler/.helmignore b/charts/dronescheduler/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/dronescheduler/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/dronescheduler/Chart.yaml b/charts/dronescheduler/Chart.yaml index 0df49fbf..41b0228f 100644 --- a/charts/dronescheduler/Chart.yaml +++ b/charts/dronescheduler/Chart.yaml @@ -1,4 +1,42 @@ +apiVersion: v2 name: dronescheduler version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Scheduler Service +type: application +home: https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/microservices/aks +sources: + - https://github.com/mspnp/microservices-reference-implementation +dependencies: + - name: dronescheduler-dev + repository: "file://envs/dronescheduler-dev" + version: v0.1.0 + condition: envs.dev + tags: + - dev + import-values: + - data + + - name: dronescheduler-prod + repository: "file://envs/dronescheduler-prod" + version: v0.1.0 + condition: envs.prod + import-values: + - data + + - name: dronescheduler-qa + repository: "file://envs/dronescheduler-qa" + version: v0.1.0 + condition: envs.qa + import-values: + - data + + - name: dronescheduler-staging + repository: "file://envs/dronescheduler-staging" + version: v0.1.0 + condition: envs.staging + import-values: + - data +maintainers: + - email: v-fean@microsoft.com + name: ferantivero diff --git a/charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml index d9adf749..43231385 100644 --- a/charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml +++ b/charts/dronescheduler/envs/dronescheduler-dev/Chart.yaml @@ -1,5 +1,6 @@ +apiVersion: v2 name: dronescheduler-dev -appName: dronescheduler version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Scheduler Service +type: application diff --git a/charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml index e61d1d8f..80404b46 100644 --- a/charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml +++ b/charts/dronescheduler/envs/dronescheduler-prod/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: dronescheduler-prod version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Scheduler Service +type: application diff --git a/charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml index 1629a692..e83a8bf4 100644 --- a/charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml +++ b/charts/dronescheduler/envs/dronescheduler-qa/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: dronescheduler-qa version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Scheduler Service +type: application diff --git a/charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml b/charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml index fb63d4ea..26581923 100644 --- a/charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml +++ b/charts/dronescheduler/envs/dronescheduler-staging/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: dronescheduler-staging version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Scheduler Service +type: application diff --git a/charts/dronescheduler/requirements.lock b/charts/dronescheduler/requirements.lock deleted file mode 100644 index a08805eb..00000000 --- a/charts/dronescheduler/requirements.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: dronescheduler-dev - repository: file://envs/dronescheduler-dev - version: v0.1.0 -- name: dronescheduler-prod - repository: file://envs/dronescheduler-prod - version: v0.1.0 -- name: dronescheduler-qa - repository: file://envs/dronescheduler-qa - version: v0.1.0 -- name: dronescheduler-staging - repository: file://envs/dronescheduler-staging - version: v0.1.0 -digest: sha256:37064a63155187c00aa56a0083ee9a241f69d0c0cdbdce487b3bcbb40c81b179 -generated: "2019-10-16T15:49:58.129570048-06:00" diff --git a/charts/dronescheduler/requirements.yaml b/charts/dronescheduler/requirements.yaml deleted file mode 100644 index dc951129..00000000 --- a/charts/dronescheduler/requirements.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dependencies: - - name: dronescheduler-dev - repository: "file://envs/dronescheduler-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data - - - name: dronescheduler-prod - repository: "file://envs/dronescheduler-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data - - - name: dronescheduler-qa - repository: "file://envs/dronescheduler-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data - - - name: dronescheduler-staging - repository: "file://envs/dronescheduler-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data diff --git a/charts/dronescheduler/templates/NOTES.txt b/charts/dronescheduler/templates/NOTES.txt index c5dbfc98..68d120cf 100644 --- a/charts/dronescheduler/templates/NOTES.txt +++ b/charts/dronescheduler/templates/NOTES.txt @@ -2,9 +2,9 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -All the objects were created in the namespace {{ .Values.namespace }} +All the objects were created in the namespace {{ .Release.Namespace }} To learn more about the release, try: - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} + $ helm status {{ .Release.Name }} --namespace {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/charts/dronescheduler/templates/_helpers.tpl b/charts/dronescheduler/templates/_helpers.tpl index caf6b7ac..94bf529a 100644 --- a/charts/dronescheduler/templates/_helpers.tpl +++ b/charts/dronescheduler/templates/_helpers.tpl @@ -38,3 +38,24 @@ Create chart name and version as used by the chart label. {{- define "dronescheduler.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "dronescheduler.labels" -}} +helm.sh/chart: {{ include "dronescheduler.chart" . }} +{{ include "dronescheduler.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "dronescheduler.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dronescheduler.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + diff --git a/charts/dronescheduler/templates/dronescheduler-deploy.yaml b/charts/dronescheduler/templates/dronescheduler-deploy.yaml index c607520a..caa28797 100644 --- a/charts/dronescheduler/templates/dronescheduler-deploy.yaml +++ b/charts/dronescheduler/templates/dronescheduler-deploy.yaml @@ -96,6 +96,6 @@ spec: value: {{ default "Error" .Values.telemetry.level | quote }} - name: no_proxy value: 169.254.169.254 - ports: - - name: service - containerPort: 8080 + ports: + - name: service + containerPort: 8080 diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 1e0f5e40..6710612d 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -1,19 +1,31 @@ # Default values for dronescheduler. replicaCount: 1 + + identity: clientid: resourceid: + + dockerregistrynamespace: dockerregistry: + + image: repository: tag: pullPolicy: IfNotPresent -keyvault: - uri: + + cosmosdb: id: collectionid: + +keyvault: + uri: + + +# probes readinessProbe: httpGet: path: /healthz @@ -28,20 +40,36 @@ livenessProbe: port: 8080 initialDelaySeconds: 50 periodSeconds: 15 + + +# specify an installation/upgrade reason reason: unknown + + telemetry: level: "Error" -tags: + + +# indicate what environment is meant to be installed/upgraded +envs: dev: false prod: false qa: false staging: false + + current: false + + +# Horizontal Pod Autoscaling autoscaling: enabled: false maxReplicas: minReplicas: targetCPUUtilizationPercentage: + + +# Pod-to-pod traffic: east-west networkPolicy: egress: enabled: true @@ -72,6 +100,8 @@ networkPolicy: dd.fabrikam.com/egress-dronescheduler: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery + + service: targetPort: 8080 targetProtocol: TCP diff --git a/charts/ingestion/.helmignore b/charts/ingestion/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/ingestion/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/ingestion/Chart.yaml b/charts/ingestion/Chart.yaml index aa6d03d4..e3e8e168 100644 --- a/charts/ingestion/Chart.yaml +++ b/charts/ingestion/Chart.yaml @@ -1,4 +1,40 @@ +apiVersion: v2 name: ingestion version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Ingestion Service +type: application +home: https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/microservices/aks +sources: + - https://github.com/mspnp/microservices-reference-implementation +dependencies: +- name: ingestion-dev + repository: "file://envs/ingestion-dev" + version: v0.1.0 + condition: envs.dev + import-values: + - data + +- name: ingestion-prod + repository: "file://envs/ingestion-prod" + version: v0.1.0 + condition: envs.prod + import-values: + - data + +- name: ingestion-qa + repository: "file://envs/ingestion-qa" + version: v0.1.0 + condition: envs.qa + import-values: + - data + +- name: ingestion-staging + repository: "file://envs/ingestion-staging" + version: v0.1.0 + condition: envs.staging + import-values: + - data +maintainers: + - email: v-fean@microsoft.com + name: ferantivero diff --git a/charts/ingestion/envs/ingestion-dev/Chart.yaml b/charts/ingestion/envs/ingestion-dev/Chart.yaml index 30475d3f..64ab4ad2 100644 --- a/charts/ingestion/envs/ingestion-dev/Chart.yaml +++ b/charts/ingestion/envs/ingestion-dev/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: ingestion-dev version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Ingestion Service +type: application diff --git a/charts/ingestion/envs/ingestion-prod/Chart.yaml b/charts/ingestion/envs/ingestion-prod/Chart.yaml index 288b4ec7..a12f1d49 100644 --- a/charts/ingestion/envs/ingestion-prod/Chart.yaml +++ b/charts/ingestion/envs/ingestion-prod/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: ingestion-prod version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Ingestion Service +type: application diff --git a/charts/ingestion/envs/ingestion-qa/Chart.yaml b/charts/ingestion/envs/ingestion-qa/Chart.yaml index 0cdac0fa..95c8212c 100644 --- a/charts/ingestion/envs/ingestion-qa/Chart.yaml +++ b/charts/ingestion/envs/ingestion-qa/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: ingestion-qa version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Ingestion Service +type: application diff --git a/charts/ingestion/envs/ingestion-staging/Chart.yaml b/charts/ingestion/envs/ingestion-staging/Chart.yaml index c5f33a04..d5522948 100644 --- a/charts/ingestion/envs/ingestion-staging/Chart.yaml +++ b/charts/ingestion/envs/ingestion-staging/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: ingestion-staging version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Ingestion Service +type: application diff --git a/charts/ingestion/requirements.lock b/charts/ingestion/requirements.lock deleted file mode 100644 index 7e7ca650..00000000 --- a/charts/ingestion/requirements.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: ingestion-dev - repository: file://envs/ingestion-dev - version: v0.1.0 -- name: ingestion-prod - repository: file://envs/ingestion-prod - version: v0.1.0 -- name: ingestion-qa - repository: file://envs/ingestion-qa - version: v0.1.0 -- name: ingestion-staging - repository: file://envs/ingestion-staging - version: v0.1.0 -digest: sha256:feee2c2842b5de8b8e7fda4a388b62ff5e85438633a480b40518c1c3371adcc5 -generated: "2019-10-25T14:37:00.76647856-06:00" diff --git a/charts/ingestion/requirements.yaml b/charts/ingestion/requirements.yaml deleted file mode 100644 index b09f4aff..00000000 --- a/charts/ingestion/requirements.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dependencies: -- name: ingestion-dev - repository: "file://envs/ingestion-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data - -- name: ingestion-prod - repository: "file://envs/ingestion-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data - -- name: ingestion-qa - repository: "file://envs/ingestion-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data - -- name: ingestion-staging - repository: "file://envs/ingestion-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data diff --git a/charts/ingestion/templates/NOTES.txt b/charts/ingestion/templates/NOTES.txt index c5dbfc98..68d120cf 100644 --- a/charts/ingestion/templates/NOTES.txt +++ b/charts/ingestion/templates/NOTES.txt @@ -2,9 +2,9 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -All the objects were created in the namespace {{ .Values.namespace }} +All the objects were created in the namespace {{ .Release.Namespace }} To learn more about the release, try: - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} + $ helm status {{ .Release.Name }} --namespace {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/charts/ingestion/templates/_helpers.tpl b/charts/ingestion/templates/_helpers.tpl index 353284bc..2047d700 100644 --- a/charts/ingestion/templates/_helpers.tpl +++ b/charts/ingestion/templates/_helpers.tpl @@ -38,3 +38,24 @@ Create chart name and version as used by the chart label. {{- define "ingestion.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "ingestion.labels" -}} +helm.sh/chart: {{ include "ingestion.chart" . }} +{{ include "ingestion.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "ingestion.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ingestion.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 6b705b6b..79cccdf7 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -1,12 +1,22 @@ # Default values for ingestion. replicaCount: 1 + + dockerregistrynamespace: dockerregistry: + + image: repository: tag: pullPolicy: IfNotPresent + + +# specify an installation/upgrade reason reason: unknown + + +# probes livenessProbe: httpGet: path: /actuator/health @@ -15,6 +25,7 @@ livenessProbe: periodSeconds: 30 successThreshold: 1 failureThreshold: 5 + readinessProbe: httpGet: path: /api/probe @@ -23,19 +34,32 @@ readinessProbe: periodSeconds: 30 successThreshold: 1 failureThreshold: 5 + + telemetry: level: "error" -tags: + + +# indicate what environment is meant to be installed/upgraded +envs: dev: false prod: false qa: false staging: false + + current: false + + +# Horizontal Pod Autoscaling autoscaling: enabled: false maxReplicas: minReplicas: targetCPUUtilizationPercentage: + + +# Pod-to-pod traffic: east-west networkPolicy: egress: enabled: true @@ -61,6 +85,8 @@ networkPolicy: # customSelectors: # templateSelector: # argSelector: + + service: targetPort: 80 targetProtocol: TCP diff --git a/charts/package/.helmignore b/charts/package/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/package/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/package/Chart.yaml b/charts/package/Chart.yaml index d4d20804..d8e8c5c4 100644 --- a/charts/package/Chart.yaml +++ b/charts/package/Chart.yaml @@ -1,4 +1,40 @@ +apiVersion: v2 name: package version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Package Service +type: application +home: https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/microservices/aks +sources: + - https://github.com/mspnp/microservices-reference-implementation +dependencies: + - name: package-dev + repository: "file://envs/package-dev" + version: v0.1.0 + condition: envs.dev + import-values: + - data + + - name: package-prod + repository: "file://envs/package-prod" + version: v0.1.0 + condition: envs.prod + import-values: + - data + + - name: package-qa + repository: "file://envs/package-qa" + version: v0.1.0 + condition: envs.qa + import-values: + - data + + - name: package-staging + repository: "file://envs/package-staging" + version: v0.1.0 + condition: envs.staging + import-values: + - data +maintainers: + - email: v-fean@microsoft.com + name: ferantivero diff --git a/charts/package/envs/package-dev/Chart.yaml b/charts/package/envs/package-dev/Chart.yaml index 9b9e3a5d..87e0686a 100644 --- a/charts/package/envs/package-dev/Chart.yaml +++ b/charts/package/envs/package-dev/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: package-dev version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Package Service +type: application diff --git a/charts/package/envs/package-prod/Chart.yaml b/charts/package/envs/package-prod/Chart.yaml index 141e274d..edcc14da 100644 --- a/charts/package/envs/package-prod/Chart.yaml +++ b/charts/package/envs/package-prod/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: package-prod version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Package Service +type: application diff --git a/charts/package/envs/package-qa/Chart.yaml b/charts/package/envs/package-qa/Chart.yaml index 43e1f25a..b75ef3c3 100644 --- a/charts/package/envs/package-qa/Chart.yaml +++ b/charts/package/envs/package-qa/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: package-qa version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Package Service +type: application diff --git a/charts/package/envs/package-staging/Chart.yaml b/charts/package/envs/package-staging/Chart.yaml index 60024b05..7111cfd9 100644 --- a/charts/package/envs/package-staging/Chart.yaml +++ b/charts/package/envs/package-staging/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: package-staging version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Package Service +type: application diff --git a/charts/package/requirements.lock b/charts/package/requirements.lock deleted file mode 100644 index 74e81899..00000000 --- a/charts/package/requirements.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: package-dev - repository: file://envs/package-dev - version: v0.1.0 -- name: package-prod - repository: file://envs/package-prod - version: v0.1.0 -- name: package-qa - repository: file://envs/package-qa - version: v0.1.0 -- name: package-staging - repository: file://envs/package-staging - version: v0.1.0 -digest: sha256:058409196b33c962ddb1c1a554013e7840c7ccfc1ddf3a98e12559aeeef05b91 -generated: "2019-10-22T08:05:25.923366271-06:00" diff --git a/charts/package/requirements.yaml b/charts/package/requirements.yaml deleted file mode 100644 index fbee7424..00000000 --- a/charts/package/requirements.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dependencies: - - name: package-dev - repository: "file://envs/package-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data - - - name: package-prod - repository: "file://envs/package-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data - - - name: package-qa - repository: "file://envs/package-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data - - - name: package-staging - repository: "file://envs/package-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data diff --git a/charts/package/templates/NOTES.txt b/charts/package/templates/NOTES.txt index c5dbfc98..68d120cf 100644 --- a/charts/package/templates/NOTES.txt +++ b/charts/package/templates/NOTES.txt @@ -2,9 +2,9 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -All the objects were created in the namespace {{ .Values.namespace }} +All the objects were created in the namespace {{ .Release.Namespace }} To learn more about the release, try: - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} + $ helm status {{ .Release.Name }} --namespace {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/charts/package/templates/_helpers.tpl b/charts/package/templates/_helpers.tpl index c8a2b8a8..62a1b6df 100644 --- a/charts/package/templates/_helpers.tpl +++ b/charts/package/templates/_helpers.tpl @@ -38,3 +38,23 @@ Create chart name and version as used by the chart label. {{- define "package.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "package.labels" -}} +helm.sh/chart: {{ include "package.chart" . }} +{{ include "package.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "package.selectorLabels" -}} +app.kubernetes.io/name: {{ include "package.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index 371da43a..f63629c0 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -1,13 +1,25 @@ # Default values for package service. nameOverride: package + + replicaCount: 1 + + dockerregistrynamespace: dockerregistry: + + image: repository: tag: pullPolicy: IfNotPresent + + +# specify an installation/upgrade reason reason: unknown + + +# probes readinessProbe: httpGet: path: /healthz @@ -16,27 +28,43 @@ readinessProbe: periodSeconds: 15 timeoutSeconds: 2 failureThreshold: 5 + livenessProbe: httpGet: path: /healthz port: 80 initialDelaySeconds: 50 periodSeconds: 15 + + log: level: error + + cosmosDb: collectionName: -tags: + + +# indicate what environment is meant to be installed/upgraded +envs: dev: false prod: false qa: false staging: false + + current: false + + +# Horizontal Pod Autoscaling autoscaling: enabled: false maxReplicas: minReplicas: targetCPUUtilizationPercentage: + + +# Pod-to-pod traffic: east-west networkPolicy: egress: enabled: true @@ -67,6 +95,8 @@ networkPolicy: dd.fabrikam.com/egress-package: "true" app.kubernetes.io/component: backend app.kubernetes.io/part-of: dronedelivery + + service: targetPort: 80 targetProtocol: TCP diff --git a/charts/workflow/.helmignore b/charts/workflow/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/workflow/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/workflow/Chart.yaml b/charts/workflow/Chart.yaml index 15d87905..c4c226c7 100644 --- a/charts/workflow/Chart.yaml +++ b/charts/workflow/Chart.yaml @@ -1,4 +1,40 @@ +apiVersion: v2 name: workflow version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Workflow Service +type: application +home: https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/microservices/aks +sources: + - https://github.com/mspnp/microservices-reference-implementation +dependencies: + - name: workflow-dev + repository: "file://envs/workflow-dev" + version: v0.1.0 + condition: envs.dev + import-values: + - data + + - name: workflow-prod + repository: "file://envs/workflow-prod" + version: v0.1.0 + condition: envs.prod + import-values: + - data + + - name: workflow-qa + repository: "file://envs/workflow-qa" + version: v0.1.0 + condition: envs.qa + import-values: + - data + + - name: workflow-staging + repository: "file://envs/workflow-staging" + version: v0.1.0 + condition: envs.staging + import-values: + - data +maintainers: + - email: v-fean@microsoft.com + name: ferantivero diff --git a/charts/workflow/envs/workflow-dev/Chart.yaml b/charts/workflow/envs/workflow-dev/Chart.yaml index 2a132cae..efaec960 100644 --- a/charts/workflow/envs/workflow-dev/Chart.yaml +++ b/charts/workflow/envs/workflow-dev/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: workflow-dev version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Workflow Service +type: application diff --git a/charts/workflow/envs/workflow-prod/Chart.yaml b/charts/workflow/envs/workflow-prod/Chart.yaml index 8de90dcc..bea10f6f 100644 --- a/charts/workflow/envs/workflow-prod/Chart.yaml +++ b/charts/workflow/envs/workflow-prod/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: workflow-prod version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Workflow Service +type: application diff --git a/charts/workflow/envs/workflow-qa/Chart.yaml b/charts/workflow/envs/workflow-qa/Chart.yaml index 52c2c9fc..3e519c1a 100644 --- a/charts/workflow/envs/workflow-qa/Chart.yaml +++ b/charts/workflow/envs/workflow-qa/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: workflow-qa version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Workflow Service +type: application diff --git a/charts/workflow/envs/workflow-staging/Chart.yaml b/charts/workflow/envs/workflow-staging/Chart.yaml index 47f70e49..938d091b 100644 --- a/charts/workflow/envs/workflow-staging/Chart.yaml +++ b/charts/workflow/envs/workflow-staging/Chart.yaml @@ -1,4 +1,6 @@ +apiVersion: v2 name: workflow-staging version: v0.1.0 appVersion: v0.1.0 description: Fabrikam Drone Delivery Workflow Service +type: application diff --git a/charts/workflow/requirements.lock b/charts/workflow/requirements.lock deleted file mode 100644 index a7674a08..00000000 --- a/charts/workflow/requirements.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: workflow-dev - repository: file://envs/workflow-dev - version: v0.1.0 -- name: workflow-prod - repository: file://envs/workflow-prod - version: v0.1.0 -- name: workflow-qa - repository: file://envs/workflow-qa - version: v0.1.0 -- name: workflow-staging - repository: file://envs/workflow-staging - version: v0.1.0 -digest: sha256:55b157e79d9bdb5a1a768545096e255d5b3fec4cc5a47a622983ebfa62b04b39 -generated: "2019-10-22T08:12:09.57949386-06:00" diff --git a/charts/workflow/requirements.yaml b/charts/workflow/requirements.yaml deleted file mode 100644 index 5971dee1..00000000 --- a/charts/workflow/requirements.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dependencies: - - name: workflow-dev - repository: "file://envs/workflow-dev" - version: ">= 0.0.1" - tags: - - dev - import-values: - - data - - - name: workflow-prod - repository: "file://envs/workflow-prod" - version: ">= 0.0.1" - tags: - - prod - import-values: - - data - - - name: workflow-qa - repository: "file://envs/workflow-qa" - version: ">= 0.0.1" - tags: - - qa - import-values: - - data - - - name: workflow-staging - repository: "file://envs/workflow-staging" - version: ">= 0.0.1" - tags: - - staging - import-values: - - data diff --git a/charts/workflow/templates/NOTES.txt b/charts/workflow/templates/NOTES.txt index 0dd034e8..c863060c 100644 --- a/charts/workflow/templates/NOTES.txt +++ b/charts/workflow/templates/NOTES.txt @@ -7,4 +7,4 @@ All the objects were created in the namespace {{ .Values.namespace }} To learn more about the release, try: $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} + $ helm get all {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/charts/workflow/templates/_helpers.tpl b/charts/workflow/templates/_helpers.tpl index aeefc2b1..82ff0779 100644 --- a/charts/workflow/templates/_helpers.tpl +++ b/charts/workflow/templates/_helpers.tpl @@ -38,3 +38,23 @@ Create chart name and version as used by the chart label. {{- define "workflow.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "workflow.labels" -}} +helm.sh/chart: {{ include "workflow.chart" . }} +{{ include "workflow.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "workflow.selectorLabels" -}} +app.kubernetes.io/name: {{ include "workflow.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index a03201a6..46870104 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -1,20 +1,34 @@ # Default values for workflow. nameOverride: workflow + + replicaCount: 1 + + dockerregistrynamespace: dockerregistry: + + identity: clientid: resourceid: + + image: repository: tag: pullPolicy: IfNotPresent + + +# specify an installation/upgrade reason reason: unknown + + serviceuri: delivery: http://delivery/api/Deliveries/ drone: http://dronescheduler/api/DroneDeliveries/ package: http://package/api/packages/ + servicerequest: maxretries: 3 circuitbreakerthreshold: 0.5 @@ -23,6 +37,9 @@ servicerequest: circuitbreakerbreakduration: 30 maxbulkheadsize: 100 maxbulkheadqueuesize: 25 + + +# probes healthcheck: delay: readinessProbe: @@ -34,6 +51,7 @@ readinessProbe: periodSeconds: 15 timeoutSeconds: 2 failureThreshold: 5 + livenessProbe: exec: command: @@ -43,23 +61,36 @@ livenessProbe: - -1 initialDelaySeconds: 50 periodSeconds: 30 + + keyvault: name: resourcegroup: subscriptionid: tenantid: + + telemetry: level: "Error" -tags: + + +# indicate what environment is meant to be installed/upgraded +envs: dev: false prod: false qa: false staging: false + + +# Horizontal Pod Autoscaling autoscaling: enabled: false maxReplicas: minReplicas: targetCPUUtilizationPercentage: + + +# Pod-to-pod traffic: east-west networkPolicy: egress: enabled: true @@ -78,6 +109,7 @@ networkPolicy: - port: 53 protocol: TCP +# indicate pods this app will attemp to establish a connection with workflow: customPodLabels: dd.fabrikam.com/egress-delivery: "true" diff --git a/deployment.md b/deployment.md index dfc061fb..25836850 100644 --- a/deployment.md +++ b/deployment.md @@ -155,18 +155,15 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces -kubectl create namespace backend-dev +kubectl create namespace backend-dev && \ +kubectl create namespace ingress-controllers ``` Setup Helm ```bash -# install helm client side -DESIRED_VERSION=v2.14.2;curl -L https://git.io/get_helm.sh | bash - -# setup tiller in your cluster -kubectl apply -f $K8S/tiller-rbac.yaml -helm init --service-account tiller +# install helm CLI +curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash ``` Integrate Application Insights instance @@ -210,8 +207,7 @@ export GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GR export APP_GATEWAY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewayName.value -o tsv) export APP_GATEWAY_PUBLIC_IP_FQDN=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewayPublicIpFqdn.value -o tsv) -helm install application-gateway-kubernetes-ingress/ingress-azure \ - --name ingress-azure-dev \ +helm install ingress-azure-dev application-gateway-kubernetes-ingress/ingress-azure \ --namespace ingress-controllers \ --set appgw.name=$APP_GATEWAY_NAME \ --set appgw.resourceGroup=$RESOURCE_GROUP \ @@ -223,7 +219,8 @@ helm install application-gateway-kubernetes-ingress/ingress-azure \ --set armAuth.identityClientID=$GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID \ --set rbac.enabled=true \ --set verbosityLevel=3 \ - --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER + --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER \ + --set appgw.usePrivateIP=false # Create a self-signed certificate for TLS export EXTERNAL_INGEST_FQDN=$APP_GATEWAY_PUBLIC_IP_FQDN @@ -282,7 +279,8 @@ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DE export DELIVERY_INGRESS_TLS_SECRET_NAME=delivery-ingress-tls # Deploy the service -helm install $HELM_CHARTS/delivery/ \ +helm package $HELM_CHARTS/delivery/ -u && \ +helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=delivery \ --set dockerregistry=$ACR_SERVER \ @@ -301,13 +299,11 @@ helm install $HELM_CHARTS/delivery/ \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ --set reason="Initial deployment" \ - --set tags.dev=true \ - --namespace backend-dev \ - --name delivery-v0.1.0-dev \ - --dep-up + --set envs.dev=true \ + --namespace backend-dev # Verify the pod is created -helm status delivery-v0.1.0-dev +helm status delivery-v0.1.0-dev --namespace backend-dev ``` ## Deploy the Package service @@ -340,7 +336,8 @@ export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSD export COSMOSDB_COL_NAME=packages # Deploy service -helm install $HELM_CHARTS/package/ \ +helm package $HELM_CHARTS/package/ -u && \ +helm install package-v0.1.0-dev package-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=package \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ @@ -351,13 +348,11 @@ helm install $HELM_CHARTS/package/ \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ - --set tags.dev=true \ - --namespace backend-dev \ - --name package-v0.1.0-dev \ - --dep-up + --set envs.dev=true \ + --namespace backend-dev # Verify the pod is created -helm status package-v0.1.0-dev +helm status package-v0.1.0-dev --namespace backend-dev ``` ## Deploy the Workflow service @@ -393,7 +388,8 @@ Deploy the Workflow service: ```bash # Deploy the service -helm install $HELM_CHARTS/workflow/ \ +helm package $HELM_CHARTS/workflow/ -u && \ +helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=workflow \ --set dockerregistry=$ACR_SERVER \ @@ -404,13 +400,11 @@ helm install $HELM_CHARTS/workflow/ \ --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ --set keyvault.tenantid=$TENANT_ID \ --set reason="Initial deployment" \ - --set tags.dev=true \ - --namespace backend-dev \ - --name workflow-v0.1.0-dev \ - --dep-up + --set envs.dev=true \ + --namespace backend-dev # Verify the pod is created -helm status workflow-v0.1.0-dev +helm status workflow-v0.1.0-dev --namespace backend-dev ``` ## Deploy the Ingestion service @@ -444,7 +438,8 @@ Deploy the Ingestion service export INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls # Deploy service -helm install $HELM_CHARTS/ingestion/ \ +helm package $HELM_CHARTS/ingestion/ -u && \ +helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=ingestion \ --set dockerregistry=$ACR_SERVER \ @@ -463,13 +458,11 @@ helm install $HELM_CHARTS/ingestion/ \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ --set reason="Initial deployment" \ - --set tags.dev=true \ - --namespace backend-dev \ - --name ingestion-v0.1.0-dev \ - --dep-up + --set envs.dev=true \ + --namespace backend-dev # Verify the pod is created -helm status ingestion-v0.1.0-dev +helm status ingestion-v0.1.0-dev --namespace backend-dev ``` ## Deploy DroneScheduler service @@ -513,7 +506,8 @@ docker push $ACR_SERVER/dronescheduler:0.1.0 Deploy the dronescheduler service: ```bash # Deploy the service -helm install $HELM_CHARTS/dronescheduler/ \ +helm package $HELM_CHARTS/dronescheduler/ -u && \ +helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=dronescheduler \ --set dockerregistry=$ACR_SERVER \ @@ -526,13 +520,11 @@ helm install $HELM_CHARTS/dronescheduler/ \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set reason="Initial deployment" \ - --set tags.dev=true \ - --namespace backend-dev \ - --name dronescheduler-v0.1.0-dev \ - --dep-up + --set envs.dev=true \ + --namespace backend-dev # Verify the pod is created -helm status dronescheduler-v0.1.0-dev +helm status dronescheduler-v0.1.0-dev --namespace backend-dev ``` ## Validate the application is running @@ -601,3 +593,17 @@ sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PA sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \ kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml ``` + +## Clean up + +Follow the steps below to remove the Fabrikam Drone Delivery app from your cluster + +```bash +helm uninstall $(helm ls --all --short -n ingress-controllers) -n ingress-controllers && \ +helm uninstall $(helm ls --all --short -n backend-dev) -n backend-dev + +# if you've choosen the CI/CD path +helm uninstall $(helm ls --all --short -n backend-qa) -n backend-qa && \ +helm uninstall $(helm ls --all --short -n backend-staging) -n backend-staging && \ +helm uninstall $(helm ls --all --short -n backend-prod) -n backend-prod +``` diff --git a/deploymentCICD.md b/deploymentCICD.md index 2ab763bc..5c95624b 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -102,18 +102,15 @@ az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME kubectl create namespace backend-dev && \ kubectl create namespace backend-qa && \ kubectl create namespace backend-staging && \ -kubectl create namespace backend +kubectl create namespace backend && \ +kubectl create namespace ingress-controllers ``` Setup Helm ```bash -# install helm client side -DESIRED_VERSION=v2.14.2;curl -L https://git.io/get_helm.sh | bash - -# setup tiller in your cluster -kubectl apply -f $K8S/tiller-rbac.yaml -helm init --service-account tiller +# install helm CLI +curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash ``` ## Integrate Application Insights instance @@ -246,8 +243,7 @@ export APP_GATEWAY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azurede export {${ENV}_APP_GATEWAY_PUBLIC_IP_FQDN,APP_GATEWAY_PUBLIC_IP_FQDN}=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.appGatewayPublicIpFqdn.value -o tsv) export ENV_NAMESPACE=$([ $env == 'prod' ] && echo 'backend' || echo "backend-$env") -helm install application-gateway-kubernetes-ingress/ingress-azure \ - --name ingress-azure-${env} \ +helm install ingress-azure-${env} application-gateway-kubernetes-ingress/ingress-azure \ --namespace ingress-controllers \ --set appgw.name=$APP_GATEWAY_NAME \ --set appgw.resourceGroup=$RESOURCE_GROUP \ @@ -259,7 +255,8 @@ helm install application-gateway-kubernetes-ingress/ingress-azure \ --set armAuth.identityClientID=$GATEWAY_CONTROLLER_PRINCIPAL_CLIENT_ID \ --set rbac.enabled=true \ --set verbosityLevel=3 \ - --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER + --set aksClusterConfiguration.apiServerAddress=$CLUSTER_SERVER \ + --set appgw.usePrivateIP=false # Create a self-signed certificate for TLS for the environment export {${ENV}_EXTERNAL_INGEST_FQDN,EXTERNAL_INGEST_FQDN}=$APP_GATEWAY_PUBLIC_IP_FQDN @@ -408,7 +405,7 @@ git push newremote release/delivery/v0.1.0 Verify delivery was deployed ```bash -helm status delivery-v0.1.0 +helm status delivery-v0.1.0 --namespace backend-dev ``` ## Add Package CI/CD @@ -491,7 +488,7 @@ git push newremote release/package/v0.1.0 Verify package was deployed ```bash -helm status package-v0.1.0 +helm status package-v0.1.0 --namespace backend-dev ``` ## Add Workflow CI/CD @@ -587,7 +584,7 @@ git push newremote release/workflow/v0.1.0 Verify workflow was deployed ```bash -helm status workflow-v0.1.0 +helm status workflow-v0.1.0 --namespace backend-dev ``` ## Add Ingestion CI/CD @@ -698,7 +695,7 @@ git push newremote release/ingestion/v0.1.0 Verify ingestion was deployed ```bash -helm status ingestion-v0.1.0 +helm status ingestion-v0.1.0 --namespace backend-dev ``` ## Add DroneScheduler CI/CD @@ -783,7 +780,7 @@ git push newremote release/dronescheduler/v0.1.0 Verify dronescheduler was deployed ```bash -helm status dronescheduler-v0.1.0 +helm status dronescheduler-v0.1.0 --namespace backend-dev ``` ## Validate the application is running From c189f43ab39ff5e67b50e109eac503cd145b4a3c Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Wed, 4 Mar 2020 18:27:02 +0000 Subject: [PATCH 232/246] Merged PR 21620: deploy app gateway for multi-env - bug fix: deploy app gateway for multi-env - other minor bug fixes solved: #182863 --- azuredeploy.json | 125 ++++++++++++++++++++++++---------------------- deploymentCICD.md | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index d69ee5bc..78fb8766 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -226,6 +226,12 @@ "appGatewayNamePrefix": "appg", "aksVnetAddressPrefix": "10.10.0.0/16", "aksClusterSubnetPrefix": "10.10.0.0/21", + "appGatewaySubnetPrefixes": [ + "10.10.8.0/24", + "10.10.9.0/24", + "10.10.10.0/24", + "10.10.11.0/24" + ], "firewallSubnetPrefix": "10.10.12.0/24", "firewallSubnetName": "AzureFirewallSubnet", "aksVnetNamePrefix": "vnet", @@ -252,8 +258,7 @@ "applicationGatewayMinCapacity": 1, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", - "appGatewaySubnetPrefix": "10.10.8.0/24", + "appGatewaySubnetIndex": 0, "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -291,8 +296,7 @@ "applicationGatewayMinCapacity": 1, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", - "appGatewaySubnetPrefix": "10.10.9.0/24", + "appGatewaySubnetIndex": 1, "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -330,8 +334,7 @@ "applicationGatewayMinCapacity": 2, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", - "appGatewaySubnetPrefix": "10.10.10.0/24", + "appGatewaySubnetIndex": 2, "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -369,8 +372,7 @@ "applicationGatewayMinCapacity": 2, "aksVnetName": "[uniqueString(variables('aksVnetNamePrefix'), resourceGroup().id)]", "aksClusterSubnetName": "[uniqueString(variables('aksClusterSubnetNamePrefix'), resourceGroup().id)]", - "appGatewaySubnetName": "[concat(parameters('environmentName'),'-agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", - "appGatewaySubnetPrefix": "10.10.11.0/24", + "appGatewaySubnetIndex": 3, "appGatewayPublicIpName": "[concat(parameters('environmentName'),'-agip-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appGatewayPublicDnsName": "[concat(parameters('environmentName'),'-ingest-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id))]", "appInsightsName": "[concat(parameters('environmentName'),uniqueString(variables('aiNamePrefix'),resourceGroup().id))]", @@ -398,7 +400,59 @@ "workspaceSku": "PerGB2018", "workspaceRetentionInDays": 730 } - } + }, + "aksClusterSubnet": { + "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", + "properties": { + "addressPrefix": "[variables('aksClusterSubnetPrefix')]", + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + "firewallSubnet": { + "name": "[variables('firewallSubnetName')]", + "properties": { + "addressPrefix": "[variables('firewallSubnetPrefix')]", + "serviceEndpoints": [ + { + "service": "Microsoft.KeyVault", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.AzureCosmosDB", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.Storage", + "locations": [ + "[resourceGroup().location]" + ] + }, + { + "service": "Microsoft.ServiceBus", + "locations": [ + "[resourceGroup().location]" + ] + } + ] + } + }, + "copy": [ + { + "name": "appGatewaySubnetsLoop", + "count": "[length(variables('appGatewaySubnetPrefixes'))]", + "input": { + "name": "[concat('agsn-', uniqueString(variables('appGatewayNamePrefix'), resourceGroup().id), copyIndex('appGatewaySubnetsLoop'))]", + "properties": { + "addressPrefix": "[variables('appGatewaySubnetPrefixes')[copyIndex('appGatewaySubnetsLoop')]]" + } + } + } + ] }, "resources": [ { @@ -477,54 +531,7 @@ "[variables('aksVnetAddressPrefix')]" ] }, - "subnets": [ - { - "name": "[variables('environmentSettings')[parameters('environmentName')].aksClusterSubnetName]", - "properties": { - "addressPrefix": "[variables('aksClusterSubnetPrefix')]", - "privateEndpointNetworkPolicies": "Enabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - }, - { - "name": "[variables('firewallSubnetName')]", - "properties": { - "addressPrefix": "[variables('firewallSubnetPrefix')]", - "serviceEndpoints": [ - { - "service": "Microsoft.KeyVault", - "locations": [ - "[resourceGroup().location]" - ] - }, - { - "service": "Microsoft.AzureCosmosDB", - "locations": [ - "[resourceGroup().location]" - ] - }, - { - "service": "Microsoft.Storage", - "locations": [ - "[resourceGroup().location]" - ] - }, - { - "service": "Microsoft.ServiceBus", - "locations": [ - "[resourceGroup().location]" - ] - } - ] - } - }, - { - "name": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName]", - "properties": { - "addressPrefix": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetPrefix]" - } - } - ] + "subnets": "[concat(createArray(variables('aksClusterSubnet'), variables('firewallSubnet')), variables('appGatewaySubnetsLoop'))]" } }, { @@ -572,7 +579,7 @@ "name": "appGatewayIpConfig", "properties": { "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetName)]" + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('environmentSettings')[parameters('environmentName')].aksVnetName, variables('appGatewaySubnetsLoop')[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetIndex].name)]" } } } @@ -1584,7 +1591,7 @@ "type": "string" }, "appGatewaySubnetPrefix": { - "value": "[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetPrefix]", + "value": "[variables('appGatewaySubnetsLoop')[variables('environmentSettings')[parameters('environmentName')].appGatewaySubnetIndex].properties.addressPrefix]", "type": "string" }, "aksVNetName": { diff --git a/deploymentCICD.md b/deploymentCICD.md index 5c95624b..d42af9a7 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -76,9 +76,7 @@ export DELIVERY_REDIS_HOSTNAMES=${DELIVERY_REDIS_HOSTNAMES}\'${DELIVERY_REDIS_HO done # Restrict cluster egress traffic -export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-prod --query properties.outputs.firewallPublicIpName.value -o tsv) && \ - -export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \ +export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-{env} --query properties.outputs.firewallPublicIpName.value -o tsv) && \ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-firewall --template-file ${PROJECT_ROOT}/azuredeploy-firewall.json \ --parameters aksVnetName=${VNET_NAME} \ aksClusterSubnetName=${CLUSTER_SUBNET_NAME} \ From a2f3cdc7fbb3111e03ae651c3021d7b1151924f8 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Thu, 5 Mar 2020 17:03:54 +0000 Subject: [PATCH 233/246] Merged PR 21636: fix delivery redis host name env var fix delivery redis host name env var --- deploymentCICD.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploymentCICD.md b/deploymentCICD.md index d42af9a7..9c542069 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -71,12 +71,12 @@ export {${ENV}_CLUSTER_SERVER,CLUSTER_SERVER}=$(az aks show -n $CLUSTER_NAME -g export CLUSTER_SERVERS=${CLUSTER_SERVERS}\'${CLUSTER_SERVER}\', export {${ENV}_ACR_SERVER,ACR_SERVER}=$(az acr show -n $ACR_NAME --query loginServer -o tsv) export ACR_SERVERS=${ACR_SERVERS}\'${ACR_SERVER}\', -export DELIVERY_REDIS_HOSTNAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryRedisHostName.value -o tsv) +export DELIVERY_REDIS_HOSTNAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.deliveryRedisHostName.value -o tsv) export DELIVERY_REDIS_HOSTNAMES=${DELIVERY_REDIS_HOSTNAMES}\'${DELIVERY_REDIS_HOSTNAME}\', done # Restrict cluster egress traffic -export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-{env} --query properties.outputs.firewallPublicIpName.value -o tsv) && \ +export FIREWALL_PIP_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.firewallPublicIpName.value -o tsv) && \ az group deployment create -g $RESOURCE_GROUP --name azuredeploy-firewall --template-file ${PROJECT_ROOT}/azuredeploy-firewall.json \ --parameters aksVnetName=${VNET_NAME} \ aksClusterSubnetName=${CLUSTER_SUBNET_NAME} \ From 6d83a5d885903e5c0f08a40f4810e461a30ee348 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 10 Mar 2020 13:05:38 +0000 Subject: [PATCH 234/246] Merged PR 21690: delivery CICD migration to helm3 now delivery can build and deploy using helm3 - modify the helm installation version to 3.0.3 - update package step - remove helm init steps as they are no longer required in ver >=3 - replace tags with envs - remove invalid tabs - workaround helm 3 known issue. For more information: https://github.com/helm/helm/issues/7497 solved: #160874 --- src/shipping/delivery/azure-pipelines-cd.json | 275 ++---------------- src/shipping/delivery/azure-pipelines.yml | 12 +- 2 files changed, 35 insertions(+), 252 deletions(-) diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index d7c13bf3..c0eaff6e 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -137,7 +137,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -147,59 +147,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -251,7 +205,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -268,7 +222,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -537,7 +492,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -547,59 +502,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -651,7 +560,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -668,7 +577,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -932,7 +842,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -942,59 +852,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1046,7 +910,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1063,7 +927,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1360,7 +1225,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1370,59 +1235,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1474,7 +1293,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,delivery-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,delivery-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1491,7 +1310,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1613,7 +1433,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1623,59 +1443,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1775,7 +1549,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1792,7 +1566,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] diff --git a/src/shipping/delivery/azure-pipelines.yml b/src/shipping/delivery/azure-pipelines.yml index fad33c12..03e14fb4 100644 --- a/src/shipping/delivery/azure-pipelines.yml +++ b/src/shipping/delivery/azure-pipelines.yml @@ -120,8 +120,12 @@ jobs: - task: HelmInstaller@0 condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' + displayName: 'Install Helm 3.0.3' inputs: + helmVersion: 3.0.3 + + checkLatestHelmVersion: false + kubectlVersion: 1.12.4 checkLatestKubectl: false @@ -136,7 +140,11 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName) --dependency-update' + updateDependency: true + + save: false + + arguments: '--app-version $(Build.SourceBranchName)' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From 122e03fa95a43c57190b02a788d9deac80f667ed Mon Sep 17 00:00:00 2001 From: Ralph Squillace Date: Tue, 10 Mar 2020 09:37:17 -0400 Subject: [PATCH 235/246] fixing the testing section so that it works (#154) the previous approach assumed that you could set the delivery_id, which you can't. As a result, the confirmation listing call fails with a 404. this captures the delivery_id and then uses that value to see the others. Fixes -- or addresses -- #132 --- deployment.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deployment.md b/deployment.md index dfc061fb..134b1b09 100644 --- a/deployment.md +++ b/deployment.md @@ -544,10 +544,9 @@ You can send delivery requests and check their statuses using curl. Since the certificate used for TLS is self-signed, the request disables TLS validation using the '-k' option. ```bash -curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -i -d '{ +curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -d '{ "confirmationRequired": "None", "deadline": "", - "deliveryId": "mydelivery", "dropOffLocation": "drop off", "expedited": true, "ownerId": "myowner", @@ -559,13 +558,13 @@ curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Cont }, "pickupLocation": "my pickup", "pickupTime": "2019-05-08T20:00:00.000Z" - }' + }' > deliveryresponse.json ``` ### Check the request status - ```bash -curl "https://$EXTERNAL_INGEST_FQDN/api/deliveries/mydelivery" --header 'Accept: application/json' -k -i +DELIVERY_ID=$(cat deliveryresponse.json | jq -r .deliveryId) +curl "https://$EXTERNAL_INGEST_FQDN/api/deliveries/$DELIVERY_ID" --header 'Accept: application/json' -k ``` ## Optional steps From b8e2875cb1b16bb540baa3de378d1596e1969ac3 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 10 Mar 2020 15:20:09 +0000 Subject: [PATCH 236/246] Merged PR 21693: workflow CICD migration to helm3 - modify the helm installation version to 3.0.3 - update package step - remove helm init steps as they are no longer required in ver >=3 - workaround helm 3 known issue for more information: https://github.com/helm/helm/issues/7497 --- src/shipping/workflow/azure-pipelines-cd.json | 220 ++---------------- src/shipping/workflow/azure-pipelines.yml | 12 +- 2 files changed, 30 insertions(+), 202 deletions(-) diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json index 69827538..c2eb4bed 100644 --- a/src/shipping/workflow/azure-pipelines-cd.json +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -114,7 +114,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -124,59 +124,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -228,7 +182,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -245,7 +199,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -488,7 +443,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -498,59 +453,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -602,7 +511,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -619,7 +528,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -862,7 +772,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -872,59 +782,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -976,7 +840,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -993,7 +857,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1278,7 +1143,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1288,59 +1153,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1392,7 +1211,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",tags.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1409,7 +1228,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] diff --git a/src/shipping/workflow/azure-pipelines.yml b/src/shipping/workflow/azure-pipelines.yml index ebfc3e27..5f67d0ac 100644 --- a/src/shipping/workflow/azure-pipelines.yml +++ b/src/shipping/workflow/azure-pipelines.yml @@ -120,8 +120,12 @@ jobs: - task: HelmInstaller@0 condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' + displayName: 'Install Helm 3.0.3' inputs: + helmVersion: 3.0.3 + + checkLatestHelmVersion: false + kubectlVersion: 1.12.4 checkLatestKubectl: false @@ -136,7 +140,11 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName) --dependency-update' + updateDependency: true + + save: false + + arguments: '--app-version $(Build.SourceBranchName)' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From 3846fc3594941db0980da1b40009015d9ef0aa57 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 10 Mar 2020 15:20:19 +0000 Subject: [PATCH 237/246] Merged PR 21695: package CICD migration to helm3 - modify the helm installation version to 3.0.3 - update package step - remove helm init steps as they are no longer required in ver >=3 - replace tags with envs - workaround helm 3 known issue for more information: https://github.com/helm/helm/issues/7497 --- src/shipping/package/azure-pipelines-cd.json | 275 ++----------------- src/shipping/package/azure-pipelines.yml | 12 +- 2 files changed, 35 insertions(+), 252 deletions(-) diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 66b74068..9160822d 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -114,7 +114,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -124,59 +124,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -228,7 +182,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -245,7 +199,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -493,7 +448,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -503,59 +458,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -607,7 +516,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -624,7 +533,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -867,7 +777,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -877,59 +787,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -981,7 +845,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -998,7 +862,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1276,7 +1141,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1286,59 +1151,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1390,7 +1209,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true,package-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,package-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1407,7 +1226,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1529,7 +1349,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1539,59 +1359,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1691,7 +1465,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1708,7 +1482,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] diff --git a/src/shipping/package/azure-pipelines.yml b/src/shipping/package/azure-pipelines.yml index 12f5809b..7c9f8ffe 100644 --- a/src/shipping/package/azure-pipelines.yml +++ b/src/shipping/package/azure-pipelines.yml @@ -79,8 +79,12 @@ jobs: - task: HelmInstaller@0 condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' + displayName: 'Install Helm 3.0.3' inputs: + helmVersion: 3.0.3 + + checkLatestHelmVersion: false + kubectlVersion: 1.12.4 checkLatestKubectl: false @@ -95,7 +99,11 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName) --dependency-update' + updateDependency: true + + save: false + + arguments: '--app-version $(Build.SourceBranchName)' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From 8b2bbff754d0e144fc40e16bd2ba1620cbebafb7 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 10 Mar 2020 15:20:34 +0000 Subject: [PATCH 238/246] Merged PR 21697: ingestion CICD migration to helm3 - modify the helm installation version to 3.0.3 - update package step - remove helm init steps as they are no longer required in ver >=3 - replace tags with envs - workaround helm 3 known issue for more information: https://github.com/helm/helm/issues/7497 --- .../ingestion/azure-pipelines-cd.json | 275 ++---------------- src/shipping/ingestion/azure-pipelines.yml | 12 +- 2 files changed, 35 insertions(+), 252 deletions(-) diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index fa6f4af6..f1074f1e 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -129,7 +129,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -139,59 +139,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -243,7 +197,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -260,7 +214,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -515,7 +470,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -525,59 +480,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -629,7 +538,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -646,7 +555,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -901,7 +811,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -911,59 +821,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1015,7 +879,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1032,7 +896,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1321,7 +1186,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1331,59 +1196,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1435,7 +1254,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,ingestion-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,ingestion-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1452,7 +1271,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1567,7 +1387,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1577,59 +1397,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1729,7 +1503,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1746,7 +1520,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] diff --git a/src/shipping/ingestion/azure-pipelines.yml b/src/shipping/ingestion/azure-pipelines.yml index 7a4040cc..28fb672c 100644 --- a/src/shipping/ingestion/azure-pipelines.yml +++ b/src/shipping/ingestion/azure-pipelines.yml @@ -120,8 +120,12 @@ jobs: - task: HelmInstaller@0 condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' + displayName: 'Install Helm 3.0.3' inputs: + helmVersion: 3.0.3 + + checkLatestHelmVersion: false + kubectlVersion: 1.12.4 checkLatestKubectl: false @@ -136,7 +140,11 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName) --dependency-update' + updateDependency: true + + save: false + + arguments: '--app-version $(Build.SourceBranchName)' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From 7576871897ae011649b3f6f75e80a1a51349a096 Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 17 Mar 2020 21:35:00 +0000 Subject: [PATCH 239/246] Merged PR 21692: drone CICD migration to helm3 - modify the helm installation version to 3.0.3 - update package step - remove helm init steps as they are no longer required in ver >=3 - replace tags with envs - workaround helm 3 known issue. For more information: https://github.com/helm/helm/issues/7497 --- deploymentCICD.md | 14 +- .../dronescheduler/azure-pipelines-cd.json | 299 +++--------------- .../dronescheduler/azure-pipelines.yml | 12 +- 3 files changed, 71 insertions(+), 254 deletions(-) diff --git a/deploymentCICD.md b/deploymentCICD.md index 9c542069..e21656e3 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -720,7 +720,9 @@ envIdentitiesDeploymentName="${ENV}_IDENTITIES_DEPLOYMENT_NAME" export ${ENV}_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n ${!envIdentitiesDeploymentName} --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) envDroneSchedulerIdName="${ENV}_DRONESCHEDULER_ID_NAME" export ${ENV}_DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n ${!envDroneSchedulerIdName} --query clientId -o tsv) -export ${ENV}_DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) +export ${ENV}_DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-${env} --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) && \ +export ${ENV}_COSMOSDB_DATABASEID="${env}_invoicing" && \ +export ${ENV}_COSMOSDB_COLLECTIONID="${env}_utilization" done # add relese definition @@ -739,18 +741,24 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$DEV_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DEV_DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#DEV_COSMOSDB_DATABASEID_VAR_VAL#$DEV_COSMOSDB_DATABASEID#g" | \ + sed "s#DEV_COSMOSDB_COLLECTIONID_VAR_VAL#$DEV_COSMOSDB_COLLECTIONID#g" | \ # qa resources sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$QA_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$QA_DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#QA_COSMOSDB_DATABASEID_VAR_VAL#$QA_COSMOSDB_DATABASEID#g" | \ + sed "s#QA_COSMOSDB_COLLECTIONID_VAR_VAL#$QA_COSMOSDB_COLLECTIONID#g" | \ # staging resources sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$STAGING_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$STAGING_DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#STAGING_COSMOSDB_DATABASEID_VAR_VAL#$STAGING_COSMOSDB_DATABASEID#g" | \ + sed "s#STAGING_COSMOSDB_COLLECTIONID_VAR_VAL#$STAGING_COSMOSDB_COLLECTIONID#g" | \ # production resources sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -758,7 +766,9 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_CLIENT_ID#g" | \ sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ - sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$PROD_DRONESCHEDULER_KEYVAULT_URI#g" \ + sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$PROD_DRONESCHEDULER_KEYVAULT_URI#g" | \ + sed "s#PROD_COSMOSDB_DATABASEID_VAR_VAL#$PROD_COSMOSDB_DATABASEID#g" | \ + sed "s#PROD_COSMOSDB_COLLECTIONID_VAR_VAL#$PROD_COSMOSDB_COLLECTIONID#g" \ > $DRONE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index 29418217..6e9376d4 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -36,6 +36,12 @@ }, "ACR_NAME": { "value": "DEV_ACR_NAME_VAR_VAL" + }, + "COSMOSDB_DATABASEID": { + "value": "DEV_COSMOSDB_DATABASEID_VAR_VAL" + }, + "COSMOSDB_COLLECTIONID": { + "value": "DEV_COSMOSDB_COLLECTIONID_VAR_VAL" } }, "variableGroups": [], @@ -112,7 +118,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -122,59 +128,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -226,7 +186,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -243,7 +203,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -408,6 +369,12 @@ }, "ACR_NAME": { "value": "QA_ACR_NAME_VAR_VAL" + }, + "COSMOSDB_DATABASEID": { + "value": "QA_COSMOSDB_DATABASEID_VAR_VAL" + }, + "COSMOSDB_COLLECTIONID": { + "value": "QA_COSMOSDB_COLLECTIONID_VAR_VAL" } }, "variableGroups": [], @@ -484,7 +451,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -494,59 +461,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -598,7 +519,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -615,7 +536,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -780,6 +702,12 @@ }, "ACR_NAME": { "value": "STAGING_ACR_NAME_VAR_VAL" + }, + "COSMOSDB_DATABASEID": { + "value": "STAGING_COSMOSDB_DATABASEID_VAR_VAL" + }, + "COSMOSDB_COLLECTIONID": { + "value": "STAGING_COSMOSDB_COLLECTIONID_VAR_VAL" } }, "variableGroups": [], @@ -856,7 +784,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -866,59 +794,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -970,7 +852,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -987,7 +869,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1158,6 +1041,12 @@ }, "ACR_NAME": { "value": "PROD_ACR_NAME_VAR_VAL" + }, + "COSMOSDB_DATABASEID": { + "value": "PROD_COSMOSDB_DATABASEID_VAR_VAL" + }, + "COSMOSDB_COLLECTIONID": { + "value": "PROD_COSMOSDB_COLLECTIONID_VAR_VAL" } }, "variableGroups": [], @@ -1262,7 +1151,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1272,59 +1161,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1376,7 +1219,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,dronescheduler-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.prod=true,dronescheduler-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1393,7 +1236,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] @@ -1515,7 +1359,7 @@ "environment": {}, "taskId": "068d5909-43e6-48c5-9e01-7c8a94816220", "version": "0.*", - "name": "install helm 2.12.3", + "name": "install helm 3.0.3", "refName": "", "enabled": true, "alwaysRun": false, @@ -1525,59 +1369,13 @@ "overrideInputs": {}, "condition": "succeeded()", "inputs": { - "helmVersion": "2.12.3", + "helmVersion": "3.0.3", "checkLatestHelmVersion": "false", "installKubeCtl": "true", "kubectlVersion": "1.12.4", "checkLatestKubeCtl": "false" } }, - { - "environment": {}, - "taskId": "afa7d54d-537b-4dc8-b60a-e0eeea2c9a87", - "version": "0.*", - "name": "helm init", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": null, - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "connectionType": "$(Parameters.connectionType)", - "azureSubscriptionEndpoint": "$(Parameters.azureSubscriptionEndpoint)", - "azureResourceGroup": "$(Parameters.azureResourceGroup)", - "kubernetesCluster": "$(Parameters.kubernetesCluster)", - "kubernetesServiceEndpoint": "$(Parameters.kubernetesServiceEndpoint)", - "namespace": "", - "command": "init", - "chartType": "Name", - "chartName": "", - "chartPath": "", - "version": "", - "releaseName": "", - "overrideValues": "", - "valueFile": "", - "destination": "$(Build.ArtifactStagingDirectory)", - "canaryimage": "false", - "upgradetiller": "true", - "updatedependency": "false", - "save": "true", - "install": "true", - "recreate": "false", - "resetValues": "false", - "force": "false", - "waitForExecution": "true", - "arguments": "--service-account tiller", - "enableTls": "false", - "caCert": "", - "certificate": "", - "privatekey": "", - "tillernamespace": "" - } - }, { "environment": {}, "taskId": "46e4be58-730b-4389-8a2f-ea10b3e5e815", @@ -1677,7 +1475,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),reason=\"$(REASON)\",tags.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1694,7 +1492,8 @@ "caCert": "", "certificate": "", "privatekey": "", - "tillernamespace": "" + "tillernamespace": "", + "failOnStderr": "false" } } ] diff --git a/src/shipping/dronescheduler/azure-pipelines.yml b/src/shipping/dronescheduler/azure-pipelines.yml index a3619bb9..a79a6615 100644 --- a/src/shipping/dronescheduler/azure-pipelines.yml +++ b/src/shipping/dronescheduler/azure-pipelines.yml @@ -84,8 +84,12 @@ jobs: - task: HelmInstaller@0 condition: eq(variables['fullCI'],True) - displayName: 'Install Helm 2.12.3' + displayName: 'Install Helm 3.0.3' inputs: + helmVersion: 3.0.3 + + checkLatestHelmVersion: false + kubectlVersion: 1.12.4 checkLatestKubectl: false @@ -100,7 +104,11 @@ jobs: chartVersion: $(Build.SourceBranchName) - arguments: '--app-version $(Build.SourceBranchName) --dependency-update' + updateDependency: true + + save: false + + arguments: '--app-version $(Build.SourceBranchName)' - task: AzureCLI@1 condition: eq(variables['fullCI'],True) From 16b72296ff217e5cfc74e2d362e2e5de4ca22a9d Mon Sep 17 00:00:00 2001 From: "Fernando Antivero (CLARIUS CONSULTING SA)" Date: Tue, 17 Mar 2020 21:35:12 +0000 Subject: [PATCH 240/246] Merged PR 21891: fix acr source fix acr source solved: #162725 --- deploymentCICD.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploymentCICD.md b/deploymentCICD.md index e21656e3..bbe57a76 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -277,8 +277,8 @@ export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler # configure build YAML definitions for pipelinePath in $DELIVERY_PATH $PACKAGE_PATH $WORKFLOW_PATH $INGESTION_PATH $DRONE_PATH; do sed -i \ - -e "s#ACR_SERVER_VAR_VAL#$ACR_SERVER#g" \ - -e "s#ACR_NAME_VAR_VAL#$ACR_NAME#g" \ + -e "s#ACR_SERVER_VAR_VAL#$DEV_ACR_SERVER#g" \ + -e "s#ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" \ -e "s#AZURE_PIPELINES_SERVICE_CONN_NAME_VAR_VAL#$AZURE_PIPELINES_SERVICE_CONN_NAME#g" \ ${pipelinePath}/azure-pipelines.yml done From 78b73ccd596e8dadb4e4980c783511fa600d528c Mon Sep 17 00:00:00 2001 From: Fer Antivero Date: Mon, 30 Mar 2020 19:25:18 +0000 Subject: [PATCH 241/246] Merged PR 22358: adapt Network Policies to work with npm v1.0.32 ### adapt network policies to npm v1.0.32 - change egress and ingress network policies - remove flexible helm values for simpler fixed np rules - adapt CD pipelines to NPM v1.0.32 For more information https://github.com/Azure/azure-container-networking/issues/528#issuecomment-600796166 solved: #186709 --- ...ry-networkpolicy-allow-egress-traffic.yaml | 23 ++++++++++++-- ...y-networkpolicy-allow-ingress-traffic.yaml | 28 ++++++++--------- charts/delivery/values.yaml | 29 ++++-------------- ...er-networkpolicy-allow-egress-traffic.yaml | 25 +++++++++++++--- ...r-networkpolicy-allow-ingress-traffic.yaml | 28 ++++++++--------- charts/dronescheduler/values.yaml | 29 ++++-------------- ...on-networkpolicy-allow-egress-traffic.yaml | 23 ++++++++++++-- ...n-networkpolicy-allow-ingress-traffic.yaml | 24 +++++---------- charts/ingestion/values.yaml | 25 ++++------------ ...ge-networkpolicy-allow-egress-traffic.yaml | 25 +++++++++++++--- ...e-networkpolicy-allow-ingress-traffic.yaml | 30 +++++++++---------- charts/package/values.yaml | 29 ++++-------------- ...ow-networkpolicy-allow-egress-traffic.yaml | 23 ++++++++++++-- charts/workflow/values.yaml | 16 ++-------- deployment.md | 18 ++++++++--- deploymentCICD.md | 30 +++++++++++++++---- src/shipping/delivery/azure-pipelines-cd.json | 22 ++++++++++---- .../dronescheduler/azure-pipelines-cd.json | 22 ++++++++++---- .../ingestion/azure-pipelines-cd.json | 22 ++++++++++---- src/shipping/package/azure-pipelines-cd.json | 22 ++++++++++---- src/shipping/workflow/azure-pipelines-cd.json | 20 ++++++++++--- 21 files changed, 300 insertions(+), 213 deletions(-) diff --git a/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml b/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml index 3da8c826..4c7e5fd0 100644 --- a/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml +++ b/charts/delivery/templates/delivery-networkpolicy-allow-egress-traffic.yaml @@ -23,10 +23,27 @@ spec: app.kubernetes.io/version: {{ .Chart.AppVersion }} policyTypes: - Egress -{{- if .Values.networkPolicy.egress.customSelectors }} egress: -{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + namespaceSelector: {} + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- if .Values.networkPolicy.egress.external.enabled }} + # allow egress traffic to all external resources except pods within the + # cluster subnet + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - {{ required "networkPolicy.egress.external.clusterSubnetPrefix is required to enable external traffic" .Values.networkPolicy.egress.external.clusterSubnetPrefix }} {{- else if .Values.networkPolicy.egress.allowAll }} - egress: [] + - to: [] {{- end -}} {{ end }} diff --git a/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml b/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml index 92bbc051..66b9d533 100644 --- a/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml +++ b/charts/delivery/templates/delivery-networkpolicy-allow-ingress-traffic.yaml @@ -24,21 +24,19 @@ spec: policyTypes: - Ingress ingress: - - ports: + - from: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-delivery: "true" + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +{{- if .Values.networkPolicy.ingress.externalSubnet.enabled }} + - ipBlock: + cidr: {{ required "networkPolicy.ingress.externalSubnet.subnetPrefix is required to enable allow traffic from" .Values.networkPolicy.ingress.externalSubnet.subnetPrefix }} +{{- else if .Values.networkPolicy.egress.allowAll }} + - {} +{{- end }} + ports: - protocol: {{ .Values.service.targetProtocol }} port: {{ .Values.service.targetPort }} -{{- if .Values.networkPolicy.ingress.customSelectors }} - from: -{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} -{{- end }} -{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} {{- end }} -{{- else if .Values.networkPolicy.ingress.allowAll }} - from: - - podSelector: {} - - namespaceSelector: {} - - ipBlock: {} -{{- end -}} -{{ end }} diff --git a/charts/delivery/values.yaml b/charts/delivery/values.yaml index d2b50d2b..a4f2a167 100644 --- a/charts/delivery/values.yaml +++ b/charts/delivery/values.yaml @@ -76,33 +76,16 @@ autoscaling: networkPolicy: egress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow egress traffic to kubedns - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + external: + enabled: false + clusterSubnetPrefix: ingress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow ingress traffic from workflow - templateSelector: - - podSelector: - matchLabels: - dd.fabrikam.com/egress-delivery: "true" - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery + externalSubnet: + enabled: false + subnetPrefix: service: diff --git a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml index 1afefc6a..34b94547 100644 --- a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml +++ b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-egress-traffic.yaml @@ -23,10 +23,27 @@ spec: app.kubernetes.io/version: {{ .Chart.AppVersion }} policyTypes: - Egress -{{- if .Values.networkPolicy.egress.customSelectors }} egress: -{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + namespaceSelector: {} + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- if .Values.networkPolicy.egress.external.enabled }} + # allow egress traffic to all external resources except pods within the + # cluster subnet + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - {{ required "networkPolicy.egress.external.clusterSubnetPrefix is required to enable external traffic" .Values.networkPolicy.egress.external.clusterSubnetPrefix }} {{- else if .Values.networkPolicy.egress.allowAll }} - egress: [] -{{- end -}} + - to: [] +{{- end }} {{ end }} diff --git a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml index 4e68cd83..908002f1 100644 --- a/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml +++ b/charts/dronescheduler/templates/dronescheduler-networkpolicy-allow-ingress-traffic.yaml @@ -24,21 +24,19 @@ spec: policyTypes: - Ingress ingress: - - ports: + - from: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-dronescheduler: "true" + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +{{- if .Values.networkPolicy.ingress.externalSubnet.enabled }} + - ipBlock: + cidr: {{ required "networkPolicy.ingress.externalSubnet.subnetPrefix is required to enable allow traffic from" .Values.networkPolicy.ingress.externalSubnet.subnetPrefix }} +{{- else if .Values.networkPolicy.egress.allowAll }} + - {} +{{- end }} + ports: - protocol: {{ .Values.service.targetProtocol }} port: {{ .Values.service.targetPort }} -{{- if .Values.networkPolicy.ingress.customSelectors }} - from: -{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} -{{- end }} -{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} {{- end }} -{{- else if .Values.networkPolicy.ingress.allowAll }} - from: - - podSelector: {} - - namespaceSelector: {} - - ipBlock: {} -{{- end -}} -{{ end }} diff --git a/charts/dronescheduler/values.yaml b/charts/dronescheduler/values.yaml index 6710612d..205d0c7d 100644 --- a/charts/dronescheduler/values.yaml +++ b/charts/dronescheduler/values.yaml @@ -73,33 +73,16 @@ autoscaling: networkPolicy: egress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow egress traffic to kubedns - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + external: + enabled: false + clusterSubnetPrefix: ingress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow ingress traffic from workflow - templateSelector: - - podSelector: - matchLabels: - dd.fabrikam.com/egress-dronescheduler: "true" - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery + externalSubnet: + enabled: false + subnetPrefix: service: diff --git a/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml b/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml index 66d806db..5c120d38 100644 --- a/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml +++ b/charts/ingestion/templates/ingestion-networkpolicy-allow-egress-traffic.yaml @@ -23,10 +23,27 @@ spec: app.kubernetes.io/version: {{ .Chart.AppVersion }} policyTypes: - Egress -{{- if .Values.networkPolicy.egress.customSelectors }} egress: -{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + namespaceSelector: {} + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- if .Values.networkPolicy.egress.external.enabled }} + # allow egress traffic to all external resources except pods within the + # cluster subnet + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - {{ required "networkPolicy.egress.external.clusterSubnetPrefix is required to enable external traffic" .Values.networkPolicy.egress.external.clusterSubnetPrefix }} {{- else if .Values.networkPolicy.egress.allowAll }} - egress: [] + - to: [] {{- end -}} {{ end }} diff --git a/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml b/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml index 3de91810..ff866dd4 100644 --- a/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml +++ b/charts/ingestion/templates/ingestion-networkpolicy-allow-ingress-traffic.yaml @@ -24,22 +24,14 @@ spec: policyTypes: - Ingress ingress: - - ports: + - from: +{{- if .Values.networkPolicy.ingress.externalSubnet.enabled }} + - ipBlock: + cidr: {{ required "networkPolicy.ingress.externalSubnet.subnetPrefix is required to enable allow traffic from" .Values.networkPolicy.ingress.externalSubnet.subnetPrefix }} +{{- else if .Values.networkPolicy.egress.allowAll }} + - {} +{{- end }} + ports: - protocol: {{ .Values.service.targetProtocol }} port: {{ .Values.service.targetPort }} -{{- if .Values.networkPolicy.ingress.customSelectors }} - from: -{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} -{{- end }} -{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} {{- end }} -{{- else if .Values.networkPolicy.ingress.allowAll }} - from: - - podSelector: {} - - namespaceSelector: {} - - ipBlock: {} -{{- end -}} -{{ end }} - diff --git a/charts/ingestion/values.yaml b/charts/ingestion/values.yaml index 79cccdf7..102aaadb 100644 --- a/charts/ingestion/values.yaml +++ b/charts/ingestion/values.yaml @@ -63,29 +63,16 @@ autoscaling: networkPolicy: egress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow egress traffic to kubedns - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + external: + enabled: false + clusterSubnetPrefix: ingress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - # customSelectors: - # templateSelector: - # argSelector: - + externalSubnet: + enabled: false + subnetPrefix: service: targetPort: 80 diff --git a/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml b/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml index b1c2e92c..3f25a5f0 100644 --- a/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml +++ b/charts/package/templates/package-networkpolicy-allow-egress-traffic.yaml @@ -23,10 +23,27 @@ spec: app.kubernetes.io/version: {{ .Chart.AppVersion }} policyTypes: - Egress -{{- if .Values.networkPolicy.egress.customSelectors }} egress: -{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + namespaceSelector: {} + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- if .Values.networkPolicy.egress.external.enabled }} + # allow egress traffic to all external resources except pods within the + # cluster subnet + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - {{ required "networkPolicy.egress.external.clusterSubnetPrefix is required to enable external traffic" .Values.networkPolicy.egress.external.clusterSubnetPrefix }} {{- else if .Values.networkPolicy.egress.allowAll }} - egress: [] -{{- end -}} + - to: [] +{{- end }} {{ end }} diff --git a/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml b/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml index fd439f5b..ac5f1ffc 100644 --- a/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml +++ b/charts/package/templates/package-networkpolicy-allow-ingress-traffic.yaml @@ -4,7 +4,7 @@ # ------------------------------------------------------------ ################################################################################################### -# Package allow ingress ingress traffic +# Package allow ingress traffic ################################################################################################### {{- if .Values.networkPolicy.ingress.enabled }} @@ -24,21 +24,19 @@ spec: policyTypes: - Ingress ingress: - - ports: + - from: + - podSelector: + matchLabels: + dd.fabrikam.com/egress-package: "true" + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: dronedelivery +{{- if .Values.networkPolicy.ingress.externalSubnet.enabled }} + - ipBlock: + cidr: {{ required "networkPolicy.ingress.externalSubnet.subnetPrefix is required to enable allow traffic from" .Values.networkPolicy.ingress.externalSubnet.subnetPrefix }} +{{- else if .Values.networkPolicy.egress.allowAll }} + - {} +{{- end }} + ports: - protocol: {{ .Values.service.targetProtocol }} port: {{ .Values.service.targetPort }} -{{- if .Values.networkPolicy.ingress.customSelectors }} - from: -{{- if .Values.networkPolicy.ingress.customSelectors.templateSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.templateSelector | indent 6 }} -{{- end }} -{{- if .Values.networkPolicy.ingress.customSelectors.argSelector }} -{{ toYaml .Values.networkPolicy.ingress.customSelectors.argSelector | indent 6 }} {{- end }} -{{- else if .Values.networkPolicy.ingress.allowAll }} - from: - - podSelector: {} - - namespaceSelector: {} - - ipBlock: {} -{{- end -}} -{{ end }} diff --git a/charts/package/values.yaml b/charts/package/values.yaml index f63629c0..e4d4b1d1 100644 --- a/charts/package/values.yaml +++ b/charts/package/values.yaml @@ -68,33 +68,16 @@ autoscaling: networkPolicy: egress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow egress traffic to kubedns - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + external: + enabled: false + clusterSubnetPrefix: ingress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow ingress traffic from workflow - templateSelector: - - podSelector: - matchLabels: - dd.fabrikam.com/egress-package: "true" - app.kubernetes.io/component: backend - app.kubernetes.io/part-of: dronedelivery + externalSubnet: + enabled: false + subnetPrefix: service: diff --git a/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml b/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml index 6e4493b7..f119404d 100644 --- a/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml +++ b/charts/workflow/templates/workflow-networkpolicy-allow-egress-traffic.yaml @@ -23,10 +23,27 @@ spec: app.kubernetes.io/version: {{ .Chart.AppVersion }} policyTypes: - Egress -{{- if .Values.networkPolicy.egress.customSelectors }} egress: -{{ toYaml .Values.networkPolicy.egress.customSelectors | indent 2 }} + # allow egress traffic to kubedns + - to: + - podSelector: + matchLabels: + k8s-app: kube-dns + namespaceSelector: {} + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- if .Values.networkPolicy.egress.external.enabled }} + # allow egress traffic to all external resources except pods within the + # cluster subnet + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - {{ required "networkPolicy.egress.external.clusterSubnetPrefix is required to enable external traffic" .Values.networkPolicy.egress.external.clusterSubnetPrefix }} {{- else if .Values.networkPolicy.egress.allowAll }} - egress: [] + - to: [] {{- end -}} {{ end }} diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml index 46870104..37950879 100644 --- a/charts/workflow/values.yaml +++ b/charts/workflow/values.yaml @@ -94,20 +94,10 @@ autoscaling: networkPolicy: egress: enabled: true - # allowAll will be ingnored if customSelectons are provided. Comment out - # customSelectors if allowAll is required allowAll: false - customSelectors: - # allow egress traffic to kubedns - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + external: + enabled: false + clusterSubnetPrefix: # indicate pods this app will attemp to establish a connection with workflow: diff --git a/deployment.md b/deployment.md index 25836850..39528d10 100644 --- a/deployment.md +++ b/deployment.md @@ -291,8 +291,10 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --set ingress.tls.secrets[0].name=$DELIVERY_INGRESS_TLS_SECRET_NAME \ --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ - --set networkPolicy.ingress.customSelectors.argSelector={ipBlock} \ - --set networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=$GATEWAY_SUBNET_PREFIX \ + --set networkPolicy.egress.external.enabled=true \ + --set networkPolicy.egress.external.clusterSubnetPrefix=$CLUSTER_SUBNET_PREFIX \ + --set networkPolicy.ingress.externalSubnet.enabled=true \ + --set networkPolicy.ingress.externalSubnet.subnetPrefix=$GATEWAY_SUBNET_PREFIX \ --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \ --set cosmosdb.id=$DATABASE_NAME \ @@ -343,6 +345,8 @@ helm install package-v0.1.0-dev package-v0.1.0.tgz \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ --set ingress.hosts[0].serviceName=package \ --set ingress.hosts[0].tls=false \ + --set networkPolicy.egress.external.enabled=true \ + --set networkPolicy.egress.external.clusterSubnetPrefix=$CLUSTER_SUBNET_PREFIX \ --set secrets.appinsights.ikey=$AI_IKEY \ --set secrets.mongo.pwd=$COSMOSDB_CONNECTION \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME \ @@ -395,6 +399,8 @@ helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ --set dockerregistry=$ACR_SERVER \ --set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \ + --set networkPolicy.egress.external.enabled=true \ + --set networkPolicy.egress.external.clusterSubnetPrefix=$CLUSTER_SUBNET_PREFIX \ --set keyvault.name=$WORKFLOW_KEYVAULT_NAME \ --set keyvault.resourcegroup=$RESOURCE_GROUP \ --set keyvault.subscriptionid=$SUBSCRIPTION_ID \ @@ -450,8 +456,10 @@ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \ --set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \ --set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \ - --set networkPolicy.ingress.customSelectors.argSelector={ipBlock} \ - --set networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=$GATEWAY_SUBNET_PREFIX \ + --set networkPolicy.egress.external.enabled=true \ + --set networkPolicy.egress.external.clusterSubnetPrefix=$CLUSTER_SUBNET_PREFIX \ + --set networkPolicy.ingress.externalSubnet.enabled=true \ + --set networkPolicy.ingress.externalSubnet.subnetPrefix=$GATEWAY_SUBNET_PREFIX \ --set secrets.appinsights.ikey=${AI_IKEY} \ --set secrets.queue.keyname=IngestionServiceAccessKey \ --set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \ @@ -516,6 +524,8 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --set ingress.hosts[0].tls=false \ --set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \ --set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \ + --set networkPolicy.egress.external.enabled=true \ + --set networkPolicy.egress.external.clusterSubnetPrefix=$CLUSTER_SUBNET_PREFIX \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ diff --git a/deploymentCICD.md b/deploymentCICD.md index bbe57a76..d5d9701e 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -343,6 +343,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL#$DEV_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL#$DEV_CLUSTER_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ @@ -356,6 +357,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#QA_GATEWAY_SUBNET_PREFIX_VAR_VAL#$QA_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#QA_CLUSTER_SUBNET_PREFIX_VAR_VAL#$QA_CLUSTER_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -369,6 +371,7 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL#$STAGING_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL#$STAGING_CLUSTER_SUBNET_PREFIX#g" | \ # production resources sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -383,7 +386,8 @@ cat $DELIVERY_PATH/azure-pipelines-cd.json | \ sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" \ + sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL#$PROD_CLUSTER_SUBNET_PREFIX#g" \ > $DELIVERY_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -447,18 +451,21 @@ cat $PACKAGE_PATH/azure-pipelines-cd.json | \ sed "s#DEV_ACR_NAME_VAR_VAL#$DEV_ACR_NAME#g" | \ sed "s#DEV_COSMOSDB_COL_NAME_VAR_VAL#$DEV_COSMOSDB_COL_NAME#g" | \ sed "s#DEV_COSMOSDB_CONNECTION_VAR_VAL#${DEV_COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL#$DEV_CLUSTER_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ sed "s#QA_COSMOSDB_COL_NAME_VAR_VAL#$QA_COSMOSDB_COL_NAME#g" | \ sed "s#QA_COSMOSDB_CONNECTION_VAR_VAL#${QA_COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#QA_CLUSTER_SUBNET_PREFIX_VAR_VAL#$QA_CLUSTER_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ sed "s#STAGING_COSMOSDB_COL_NAME_VAR_VAL#$STAGING_COSMOSDB_COL_NAME#g" | \ sed "s#STAGING_COSMOSDB_CONNECTION_VAR_VAL#${STAGING_COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL#$STAGING_CLUSTER_SUBNET_PREFIX#g" | \ # production resources sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ @@ -466,7 +473,8 @@ cat $PACKAGE_PATH/azure-pipelines-cd.json | \ sed "s#PROD_ACR_SERVER_VAR_VAL#$PROD_ACR_SERVER#g" | \ sed "s#PROD_ACR_NAME_VAR_VAL#$PROD_ACR_NAME#g" | \ sed "s#PROD_COSMOSDB_COL_NAME_VAR_VAL#$PROD_COSMOSDB_COL_NAME#g" | \ - sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${PROD_COSMOSDB_CONNECTION//&/\\&}#g" \ + sed "s#PROD_COSMOSDB_CONNECTION_VAR_VAL#${PROD_COSMOSDB_CONNECTION//&/\\&}#g" | \ + sed "s#PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL#$PROD_CLUSTER_SUBNET_PREFIX#g" \ > $PACKAGE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -534,6 +542,7 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#DEV_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$DEV_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#DEV_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#DEV_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + sed "s#DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL#$DEV_CLUSTER_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ @@ -543,6 +552,7 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#QA_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$QA_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#QA_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#QA_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + sed "s#QA_CLUSTER_SUBNET_PREFIX_VAR_VAL#$QA_CLUSTER_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -552,6 +562,7 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$STAGING_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#STAGING_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ sed "s#STAGING_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + sed "s#STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL#$STAGING_CLUSTER_SUBNET_PREFIX#g" | \ # production resources sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -562,7 +573,8 @@ cat $WORKFLOW_PATH/azure-pipelines-cd.json | \ sed "s#PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_WORKFLOW_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#PROD_WORKFLOW_KEYVAULT_NAME_VAR_VAL#$PROD_WORKFLOW_KEYVAULT_NAME#g" | \ sed "s#PROD_SUBSCRIPTION_ID_VAR_VAL#$SUBSCRIPTION_ID#g" | \ - sed "s#PROD_TENANT_ID_VAR_VAL#$TENANT_ID#g" \ + sed "s#PROD_TENANT_ID_VAR_VAL#$TENANT_ID#g" | \ + sed "s#PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL#$PROD_CLUSTER_SUBNET_PREFIX#g" \ > $WORKFLOW_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -636,6 +648,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#DEV_INGRESS_TLS_SECRET_KEY_VAR_VAL#$DEV_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#DEV_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL#$DEV_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL#$DEV_CLUSTER_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_AI_IKEY_VAR_VAL#$QA_AI_IKEY#g" | \ sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ @@ -648,6 +661,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#QA_INGRESS_TLS_SECRET_KEY_VAR_VAL#$QA_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#QA_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#QA_GATEWAY_SUBNET_PREFIX_VAR_VAL#$QA_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#QA_CLUSTER_SUBNET_PREFIX_VAR_VAL#$QA_CLUSTER_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_AI_IKEY_VAR_VAL#$STAGING_AI_IKEY#g" | \ sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ @@ -660,6 +674,7 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_INGRESS_TLS_SECRET_KEY_VAR_VAL#$STAGING_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#STAGING_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ sed "s#STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL#$STAGING_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL#$STAGING_CLUSTER_SUBNET_PREFIX#g" | \ # production resources sed "s#PROD_AI_IKEY_VAR_VAL#$PROD_AI_IKEY#g" | \ sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ @@ -673,7 +688,8 @@ cat $INGESTION_PATH/azure-pipelines-cd.json | \ sed "s#PROD_INGRESS_TLS_SECRET_CERT_VAR_VAL#$PROD_INGRESS_TLS_SECRET_CERT#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_KEY_VAR_VAL#$PROD_INGRESS_TLS_SECRET_KEY#g" | \ sed "s#PROD_INGRESS_TLS_SECRET_NAME_VAR_VAL#$INGRESS_TLS_SECRET_NAME#g" | \ - sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" \ + sed "s#PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL#$PROD_GATEWAY_SUBNET_PREFIX#g" | \ + sed "s#PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL#$PROD_CLUSTER_SUBNET_PREFIX#g" \ > $INGESTION_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ @@ -743,6 +759,7 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#DEV_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$DEV_DRONESCHEDULER_KEYVAULT_URI#g" | \ sed "s#DEV_COSMOSDB_DATABASEID_VAR_VAL#$DEV_COSMOSDB_DATABASEID#g" | \ sed "s#DEV_COSMOSDB_COLLECTIONID_VAR_VAL#$DEV_COSMOSDB_COLLECTIONID#g" | \ + sed "s#DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL#$DEV_CLUSTER_SUBNET_PREFIX#g" | \ # qa resources sed "s#QA_ACR_SERVER_VAR_VAL#$QA_ACR_SERVER#g" | \ sed "s#QA_ACR_NAME_VAR_VAL#$QA_ACR_NAME#g" | \ @@ -751,6 +768,7 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#QA_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$QA_DRONESCHEDULER_KEYVAULT_URI#g" | \ sed "s#QA_COSMOSDB_DATABASEID_VAR_VAL#$QA_COSMOSDB_DATABASEID#g" | \ sed "s#QA_COSMOSDB_COLLECTIONID_VAR_VAL#$QA_COSMOSDB_COLLECTIONID#g" | \ + sed "s#QA_CLUSTER_SUBNET_PREFIX_VAR_VAL#$QA_CLUSTER_SUBNET_PREFIX#g" | \ # staging resources sed "s#STAGING_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#STAGING_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -759,6 +777,7 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#STAGING_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$STAGING_DRONESCHEDULER_KEYVAULT_URI#g" | \ sed "s#STAGING_COSMOSDB_DATABASEID_VAR_VAL#$STAGING_COSMOSDB_DATABASEID#g" | \ sed "s#STAGING_COSMOSDB_COLLECTIONID_VAR_VAL#$STAGING_COSMOSDB_COLLECTIONID#g" | \ + sed "s#STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL#$STAGING_CLUSTER_SUBNET_PREFIX#g" | \ # production resources sed "s#SOURCE_ACR_SERVER_VAR_VAL#$STAGING_ACR_SERVER#g" | \ sed "s#SOURCE_ACR_NAME_VAR_VAL#$STAGING_ACR_NAME#g" | \ @@ -768,7 +787,8 @@ cat $DRONE_PATH/azure-pipelines-cd.json | \ sed "s#PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID_VAR_VAL#$PROD_DRONESCHEDULER_PRINCIPAL_RESOURCE_ID#g" | \ sed "s#PROD_DRONESCHEDULER_KEYVAULT_URI_VAR_VAL#$PROD_DRONESCHEDULER_KEYVAULT_URI#g" | \ sed "s#PROD_COSMOSDB_DATABASEID_VAR_VAL#$PROD_COSMOSDB_DATABASEID#g" | \ - sed "s#PROD_COSMOSDB_COLLECTIONID_VAR_VAL#$PROD_COSMOSDB_COLLECTIONID#g" \ + sed "s#PROD_COSMOSDB_COLLECTIONID_VAR_VAL#$PROD_COSMOSDB_COLLECTIONID#g" | \ + sed "s#PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL#$PROD_CLUSTER_SUBNET_PREFIX#g" \ > $DRONE_PATH/azure-pipelines-cd-0.json curl -sL -w "%{http_code}" -X POST ${AZURE_DEVOPS_VSRM_ORG}/${AZURE_DEVOPS_PROJECT_NAME}/_apis/release/definitions?api-version=5.1-preview.3 \ diff --git a/src/shipping/delivery/azure-pipelines-cd.json b/src/shipping/delivery/azure-pipelines-cd.json index c0eaff6e..50e66be5 100644 --- a/src/shipping/delivery/azure-pipelines-cd.json +++ b/src/shipping/delivery/azure-pipelines-cd.json @@ -61,6 +61,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -205,7 +208,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -416,6 +419,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "QA_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "QA_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -560,7 +566,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -766,6 +772,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -910,7 +919,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1121,6 +1130,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1293,7 +1305,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,delivery-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,delivery-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1549,7 +1561,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",identity.clientid=$(DELIVERY_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DELIVERY_PRINCIPAL_RESOURCE_ID),cosmosdb.id=$(DATABASE_NAME),cosmosdb.collectionid=$(COLLECTION_NAME),keyvault.uri=$(DELIVERY_KEYVAULT_URI),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/dronescheduler/azure-pipelines-cd.json b/src/shipping/dronescheduler/azure-pipelines-cd.json index 6e9376d4..2f438a29 100644 --- a/src/shipping/dronescheduler/azure-pipelines-cd.json +++ b/src/shipping/dronescheduler/azure-pipelines-cd.json @@ -42,6 +42,9 @@ }, "COSMOSDB_COLLECTIONID": { "value": "DEV_COSMOSDB_COLLECTIONID_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -186,7 +189,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$(COSMOSDB_DATABASEID),cosmosdb.collectionid=$(COSMOSDB_COLLECTIONID),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -375,6 +378,9 @@ }, "COSMOSDB_COLLECTIONID": { "value": "QA_COSMOSDB_COLLECTIONID_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "QA_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -519,7 +525,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$(COSMOSDB_DATABASEID),cosmosdb.collectionid=$(COSMOSDB_COLLECTIONID),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -708,6 +714,9 @@ }, "COSMOSDB_COLLECTIONID": { "value": "STAGING_COSMOSDB_COLLECTIONID_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -852,7 +861,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$(COSMOSDB_DATABASEID),cosmosdb.collectionid=$(COSMOSDB_COLLECTIONID),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1047,6 +1056,9 @@ }, "COSMOSDB_COLLECTIONID": { "value": "PROD_COSMOSDB_COLLECTIONID_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1219,7 +1231,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.prod=true,dronescheduler-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$(COSMOSDB_DATABASEID),cosmosdb.collectionid=$(COSMOSDB_COLLECTIONID),reason=\"$(REASON)\",envs.prod=true,dronescheduler-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1475,7 +1487,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$COSMOSDB_DATABASEID,cosmosdb.collectionid=$COSMOSDB_COLLECTIONID,reason=\"$(REASON)\",envs.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(DRONESCHEDULER_PRINCIPAL_CLIENT_ID),identity.resourceid=$(DRONESCHEDULER_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.uri=$(DRONESCHEDULER_KEYVAULT_URI),cosmosdb.id=$(COSMOSDB_DATABASEID),cosmosdb.collectionid=$(COSMOSDB_COLLECTIONID),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/ingestion/azure-pipelines-cd.json b/src/shipping/ingestion/azure-pipelines-cd.json index f1074f1e..4c7bb36c 100644 --- a/src/shipping/ingestion/azure-pipelines-cd.json +++ b/src/shipping/ingestion/azure-pipelines-cd.json @@ -60,6 +60,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "DEV_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -197,7 +200,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -401,6 +404,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "QA_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "QA_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -538,7 +544,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -742,6 +748,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "STAGING_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -879,7 +888,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1089,6 +1098,9 @@ }, "GATEWAY_SUBNET_PREFIX": { "value": "PROD_GATEWAY_SUBNET_PREFIX_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1254,7 +1266,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,ingestion-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,ingestion-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1503,7 +1515,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.ingress.customSelectors.argSelector={ipBlock},networkPolicy.ingress.customSelectors.argSelector[0].ipBlock.cidr=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),ingress.hosts[0].name=$(EXTERNAL_INGEST_FQDN),ingress.hosts[0].serviceName=$(SERVICE_NAME),ingress.hosts[0].tls=true,ingress.hosts[0].tlsSecretName=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].name=$(INGRESS_TLS_SECRET_NAME),ingress.tls.secrets[0].key=\"$(INGRESS_TLS_SECRET_KEY)\",ingress.tls.secrets[0].certificate=\"$(INGRESS_TLS_SECRET_CERT)\",networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",networkPolicy.ingress.externalSubnet.enabled=true,networkPolicy.ingress.externalSubnet.subnetPrefix=\"$(GATEWAY_SUBNET_PREFIX)\",secrets.appinsights.ikey=$(AI_IKEY),secrets.queue.keyname=IngestionServiceAccessKey,secrets.queue.keyvalue=$(INGESTION_ACCESS_KEY_VALUE),secrets.queue.name=$(INGESTION_QUEUE_NAME),secrets.queue.namespace=$(INGESTION_QUEUE_NAMESPACE),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/package/azure-pipelines-cd.json b/src/shipping/package/azure-pipelines-cd.json index 9160822d..78c31950 100644 --- a/src/shipping/package/azure-pipelines-cd.json +++ b/src/shipping/package/azure-pipelines-cd.json @@ -38,6 +38,9 @@ }, "ACR_NAME": { "value": "DEV_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -182,7 +185,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -372,6 +375,9 @@ }, "ACR_NAME": { "value": "QA_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "QA_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -516,7 +522,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -701,6 +707,9 @@ }, "ACR_NAME": { "value": "STAGING_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -845,7 +854,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1036,6 +1045,9 @@ }, "ACR_NAME": { "value": "PROD_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1209,7 +1221,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,package-prod.experimental=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,package-prod.experimental=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1465,7 +1477,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,current=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),secrets.appinsights.ikey=$(AI_IKEY),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",secrets.mongo.pwd=$(COSMOSDB_CONNECTION),cosmosDb.collectionName=$(COSMOSDB_COL_NAME),reason=\"$(REASON)\",envs.prod=true,current=true", "valueFile": "", "destination": "", "canaryimage": "false", diff --git a/src/shipping/workflow/azure-pipelines-cd.json b/src/shipping/workflow/azure-pipelines-cd.json index c2eb4bed..d2c664c5 100644 --- a/src/shipping/workflow/azure-pipelines-cd.json +++ b/src/shipping/workflow/azure-pipelines-cd.json @@ -45,6 +45,9 @@ }, "ACR_NAME": { "value": "DEV_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "DEV_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -182,7 +185,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-dev", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.dev=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.dev=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -374,6 +377,9 @@ }, "ACR_NAME": { "value": "QA_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "QA_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -511,7 +517,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-qa", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.qa=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.qa=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -703,6 +709,9 @@ }, "ACR_NAME": { "value": "STAGING_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "STAGING_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -840,7 +849,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)-staging", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.staging=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.staging=true", "valueFile": "", "destination": "", "canaryimage": "false", @@ -1038,6 +1047,9 @@ }, "ACR_NAME": { "value": "PROD_ACR_NAME_VAR_VAL" + }, + "CLUSTER_SUBNET_PREFIX": { + "value": "PROD_CLUSTER_SUBNET_PREFIX_VAR_VAL" } }, "variableGroups": [], @@ -1211,7 +1223,7 @@ "chartPath": "", "version": "", "releaseName": "$(REPO_NAME)-$(Build.SourceBranchName)", - "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.prod=true", + "overrideValues": "image.tag=$(Build.SourceBranchName),image.repository=$(REPO_NAME),dockerregistry=$(ACR_SERVER),identity.clientid=$(WORKFLOW_PRINCIPAL_CLIENT_ID),identity.resourceid=$(WORKFLOW_PRINCIPAL_RESOURCE_ID),networkPolicy.egress.external.enabled=true,networkPolicy.egress.external.clusterSubnetPrefix=\"$(CLUSTER_SUBNET_PREFIX)\",keyvault.name=$(WORKFLOW_KEYVAULT_NAME),keyvault.resourcegroup=$(WORKFLOW_KEYVAULT_RESOURCE_GROUP),keyvault.subscriptionid=$(SUBSCRIPTION_ID),keyvault.tenantid=$(TENANT_ID),reason=\"$(REASON)\",envs.prod=true", "valueFile": "", "destination": "", "canaryimage": "false", From ca36e03884adb6921ea9efe7646c68a2da338a1a Mon Sep 17 00:00:00 2001 From: Fer Antivero Date: Wed, 1 Apr 2020 16:30:53 +0000 Subject: [PATCH 242/246] Merged PR 22559: start using koa-router - remove unused libs - remove fleek router lib - replace fleek-router with koa-router - rename package service instructions - add unit tests solved: #160885 #148180 --- src/shipping/package/app/api.json | 338 +++++++++--------- src/shipping/package/app/app.ts | 65 ++++ .../app/controllers/healthz-controllers.ts | 18 + src/shipping/package/app/controllers/index.ts | 2 + .../app/controllers/package-controllers.ts | 227 ++++++++---- .../app/controllers/swagger-controllers.ts | 19 + src/shipping/package/app/models/package.ts | 6 +- src/shipping/package/app/models/repository.ts | 25 +- src/shipping/package/app/routes.ts | 46 +++ src/shipping/package/app/server.ts | 89 +---- .../package/app/spec/package-swagger.ts | 29 ++ src/shipping/package/app/util/logging.ts | 10 +- src/shipping/package/app/util/settings.ts | 2 +- src/shipping/package/babel.config.js | 13 + src/shipping/package/gulpfile.js | 9 +- src/shipping/package/jest.config.js | 32 ++ src/shipping/package/package-service.md | 11 - src/shipping/package/package.json | 35 +- src/shipping/package/readme.md | 11 + .../models/test-data-access.ts | 0 .../tests/unit/healthz-controllers.test.ts | 39 ++ .../tests/unit/package-controllers.test.ts | 173 +++++++++ .../tests/unit/swagger-controllers.test.ts | 45 +++ .../package/tests/unit/util-settings.test.ts | 56 +++ src/shipping/package/tsconfig.json | 10 +- 25 files changed, 954 insertions(+), 356 deletions(-) create mode 100644 src/shipping/package/app/app.ts create mode 100644 src/shipping/package/app/controllers/healthz-controllers.ts create mode 100644 src/shipping/package/app/controllers/swagger-controllers.ts create mode 100644 src/shipping/package/app/routes.ts create mode 100644 src/shipping/package/app/spec/package-swagger.ts create mode 100644 src/shipping/package/babel.config.js create mode 100644 src/shipping/package/jest.config.js delete mode 100644 src/shipping/package/package-service.md rename src/shipping/package/{test => tests}/models/test-data-access.ts (100%) create mode 100644 src/shipping/package/tests/unit/healthz-controllers.test.ts create mode 100644 src/shipping/package/tests/unit/package-controllers.test.ts create mode 100644 src/shipping/package/tests/unit/swagger-controllers.test.ts create mode 100644 src/shipping/package/tests/unit/util-settings.test.ts diff --git a/src/shipping/package/app/api.json b/src/shipping/package/app/api.json index 8b5d1cbb..e22c044a 100644 --- a/src/shipping/package/app/api.json +++ b/src/shipping/package/app/api.json @@ -1,188 +1,192 @@ { - "swagger": "2.0", + "openapi": "3.0.0", "info": { - "description": "Drone Delivery - Package Service", - "version": "1.0.0", - "title": "Package Service" + "title": "fabrikam-drone-delivery-package-service", + "description": "Fabrikam Drone Delivery Package Service", + "version": "0.1.0", + "contact": "Microsoft Patterns and Practices", + "termsOfService": "" }, + "basePath": "/api", "schemes": [ - "http", - "https" + "http", + "https" ], "consumes": [ - "application/json" + "application/json" ], "produces": [ - "application/json" + "application/json" ], - "basePath": "/api", "paths": { - "/packages/{packageId}": { - "get": { - "summary": "Get information about a specific package from the service", - "description": "", - "operationId": "getById", - "parameters": [ - { - "name": "packageId", - "in": "path", - "description": "ID of package to return", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/Package" - } + "/packages/{packageId}": { + "get": { + "summary": "Get information about a specific package from the service", + "description": "Returns package by id", + "operationid": "getById", + "parameters": [ + { + "name": "packageId", + "description": "ID of package to return", + "in": "path", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "successful operation", + "schema": { + "$ref": "#/definitions/Package" + } + }, + "400": { + "description": "Invalid ID supplied" + }, + "404": { + "description": "Package not found" + } + } }, - "400": { - "description": "Invalid ID supplied" + "patch": { + "summary": "Update an existing package", + "description": "Update Package by ID", + "operationid": "updateById", + "parameters": [ + { + "name": "packageId", + "description": "ID of package to patch", + "in": "path", + "required": true, + "type": "string" + } + ], + "responses": { + "204": { + "description": "successful operation" + }, + "400": { + "description": "invalid id supplied" + }, + "404": { + "description": "package not found" + }, + "405": { + "description": "Validation exception" + } + } }, - "404": { - "description": "Package not found" + "put": { + "summary": "Create or update a package", + "description": "Creates or updates a package using the data provided in the API", + "operationid": "createOrUpdate", + "parameters": [ + { + "name": "packageId", + "description": "ID of package to patch", + "in": "path", + "required": true, + "type": "string" + } + ], + "responses": { + "201": { + "description": "Created new package", + "schema": { + "$ref": "#/definitions/Package" + } + }, + "204": { + "description": "Updated existing package" + } + } } - } }, - "patch": { - "summary": "Update an existing package", - "description": "", - "operationId": "updateById", - "parameters": [ - { - "name": "packageId", - "in": "path", - "description": "ID of package to patch", - "required": true, - "type": "string" + "/packages/summary/{ownerId}": { + "get": { + "summary": "Get summary information about packages from a user", + "description": "Get summary information about packages from a user", + "operationid": "getSummary", + "parameters": [ + { + "name": "ownerId", + "description": "ID of the package's owner", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "year", + "description": "Year of the summary requested", + "in": "query", + "required": true, + "type": "integer" + }, + { + "name": "month", + "description": "Month of the summary requested", + "in": "path", + "required": true, + "type": "integer" + } + ], + "responses": { + "201": { + "description": "successful operation", + "schema": { + "$ref": "#/definitions/PackageUtilization" + } + }, + "400": { + "description": "Invalid ID, year, or month supplied" + } + } } - ], - "responses": { - "204": { - "description": "successful operation" - }, - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Package not found" - }, - "405": { - "description": "Validation exception" + } + }, + "definitions": { + "Package": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "size": { + "type": "string", + "enum": [ + "small", + "medium", + "large" + ] + }, + "weight": { + "type": "number" + }, + "tag": { + "type": "string" + } } - } }, - "put": { - "summary": "Create or update a package", - "description": "", - "operationId": "createOrUpdate", - "parameters": [ - { - "name": "packageId", - "in": "path", - "description": "ID of package", - "required": true, - "type": "string" - } - ], - "responses": { - "201": { - "description": "Created new package", - "schema": { - "$ref": "#/definitions/Package" - } - }, - "204": { - "description": "Updated existing package" + "PackageUtilization": { + "type": "object", + "properties": { + "totalWeight": { + "type": "number" + } } - } - } - }, - "/packages/summary": { - "get": { - "summary": "Get summary information about packages from a user", - "description": "", - "operationId": "getSummary", - "parameters": [ - { - "name": "ownerId", - "in": "query", - "description": "ID of the owner of the packages", - "required": true, - "type": "string" - }, - { - "name": "year", - "in": "query", - "description": "Year of the summary requested", - "required": true, - "type": "integer" - }, - { - "name": "month", - "in": "query", - "description": "Month of the summary requested", - "required": true, - "type": "integer" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/PackageUtilization" - } - }, - "400": { - "description": "Invalid ID, year, or month supplied" + }, + "Error": { + "type": "object", + "properties": { + "code": { + "type": "number" + }, + "message": { + "type": "string" + } } - } } - } }, - "definitions": { - "Package": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "size": { - "type": "string", - "enum": [ - "small", - "medium", - "large" - ] - }, - "weight": { - "type": "number" - }, - "tag": { - "type": "string" - } - } - }, - "PackageUtilization": { - "type": "object", - "properties": { - "totalWeight": { - "type": "number" - } - } - }, - "Error": { - "type": "object", - "properties": { - "code": { - "type": "number" - }, - "message": { - "type": "string" - } - } - } - } -} \ No newline at end of file + "components": {}, + "tags": [] +} diff --git a/src/shipping/package/app/app.ts b/src/shipping/package/app/app.ts new file mode 100644 index 00000000..5a5cc1ad --- /dev/null +++ b/src/shipping/package/app/app.ts @@ -0,0 +1,65 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +import Koa from 'koa'; +import bodyParser from "koa-bodyparser"; + +var compress = require('koa-compress'); + +import { apiRouter, healthzRouter, swaggerRouter } from './routes'; +import { logger, ILogger } from './util/logging' + +export class KoaApp { + + static create(logLevel: string) : Koa { + + const app = new Koa(); + + // Configure logging + app.use(logger(logLevel)); + + // Configure global exception handling + // Use: ctx.throw('Error Message', 500); + // in the controller methods to set the status code and exception message + app.use(async (ctx : any, next : any) => { + try { + await next(); + } catch (ex) { + + var logger : ILogger = ctx.state.logger; + if (logger) { + logger.error(ex.message); + } + + ctx.status = ex.status || 500; + // consider api specific codes and localized messages as opposed to internal codes + ctx.body = { + level: "error", + code: ex.code, + message: ex.message + } + ctx.app.emit('error', ex, ctx); + } + }); + + // add compression and body parser to the pipeline + app.use(compress()); + app.use(bodyParser()); + + const packageServiceRouter = apiRouter(); + app.use(packageServiceRouter.routes()); + app.use(packageServiceRouter.allowedMethods()); + + const healthChecksRouter = healthzRouter(); + app.use(healthChecksRouter.routes()); + app.use(healthChecksRouter.allowedMethods()); + + const swaggerSpecRouter = swaggerRouter(); + app.use(swaggerSpecRouter.routes()); + app.use(swaggerSpecRouter.allowedMethods()); + + return app; + } +} diff --git a/src/shipping/package/app/controllers/healthz-controllers.ts b/src/shipping/package/app/controllers/healthz-controllers.ts new file mode 100644 index 00000000..07b7b3ab --- /dev/null +++ b/src/shipping/package/app/controllers/healthz-controllers.ts @@ -0,0 +1,18 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +import { ILogger } from '../util/logging' + +export class HealthzControllers { + + static async getReadinessLiveness(ctx: any) { + var logger : ILogger = ctx.state.logger; + logger.info('Readiness/Liveness Probe Status: %s', "OK"); + + ctx.status = 200; + ctx.body = {status: 'OK'}; + } + +} diff --git a/src/shipping/package/app/controllers/index.ts b/src/shipping/package/app/controllers/index.ts index 7c380067..fd101adb 100644 --- a/src/shipping/package/app/controllers/index.ts +++ b/src/shipping/package/app/controllers/index.ts @@ -4,3 +4,5 @@ // ------------------------------------------------------------ export * from './package-controllers'; +export * from './healthz-controllers'; +export * from './swagger-controllers'; diff --git a/src/shipping/package/app/controllers/package-controllers.ts b/src/shipping/package/app/controllers/package-controllers.ts index 21212dd1..9ab2949e 100644 --- a/src/shipping/package/app/controllers/package-controllers.ts +++ b/src/shipping/package/app/controllers/package-controllers.ts @@ -3,48 +3,76 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -import { Repository, UpsertStatus } from '../models/repository' +import { UpsertStatus } from '../models/repository' import * as apiModels from '../models/api-models' import { Package, PackageSize } from '../models/package' import { ILogger } from '../util/logging' import { MongoErrors } from '../util/mongo-err' export class PackageControllers { - - constructor(private repository: Repository) { - - } - - mapPackageDbToApi(pkg: Package): apiModels.Package { - // consider moving this mapping code to data access - // might also want to consider allowing repository to deal with API types or.. automapping, mapping through convention or config - // we could simpley add/remove/change the model in the api vs database - let apiPkg = new apiModels.Package(); - apiPkg.id = pkg._id; - apiPkg.size = pkg.size ? pkg.size.toString() : null; - apiPkg.tag = pkg.tag; - apiPkg.weight = pkg.weight; - return apiPkg; - } - - - mapPackageApiToDb(apiPkg: apiModels.Package, id?: string): Package { - let pkg = new Package(id); - pkg.size = apiPkg.size; - pkg.weight = apiPkg.weight; - pkg.tag = apiPkg.tag; - return pkg; - } - - async getById(ctx: any, next: any) { + /** + * @swagger + * + * definitions: + * Package: + * type: object + * properties: + * id: + * type: string + * size: + * type: string + * enum: + * - small + * - medium + * - large + * weight: + * type: number + * tag: + * type: string + * PackageUtilization: + * type: object + * properties: + * totalWeight: + * type: number + * Error: + * type: object + * properties: + * code: + * type: number + * message: + * type: string + */ + + /** + * @swagger + * /packages/{packageId}: + * get: + * summary: Get information about a specific package from the service + * description: Returns package by id + * operationid: getById + * parameters: + * - name: packageId + * description: ID of package to return + * in: path + * required: true + * type: string + * responses: + * 200: + * description: successful operation + * schema: + * $ref: '#/definitions/Package' + * 400: + * description: Invalid ID supplied + * 404: + * description: Package not found + */ + static async getById(ctx: any) { var logger : ILogger = ctx.state.logger; var packageId = ctx.params.packageId; logger.info('Entering getById, packageId = %s', packageId); - await next(); - - let pkg = await this.repository.findPackage(ctx.params.packageId) + let pkg = await ctx.packageRepository.findPackage(ctx.params.packageId) if (pkg == null) { logger.info(`getById: %s not found`, packageId); @@ -53,52 +81,91 @@ export class PackageControllers { } ctx.response.status = 200; - ctx.response.body = this.mapPackageDbToApi(pkg); + ctx.response.body = ctx.packageRepository.mapPackageDbToApi(pkg); } - async updateById(ctx: any, next: any) { + /** + * @swagger + * /packages/{packageId}: + * patch: + * summary: Update an existing package + * description: Update Package by ID + * operationid: updateById + * parameters: + * - name: packageId + * description: ID of package to patch + * in: path + * required: true + * type: string + * responses: + * 204: + * description: successful operation + * 400: + * description: invalid id supplied + * 404: + * description: package not found + * 405: + * description: Validation exception + */ + static async updateById(ctx: any) { var logger : ILogger = ctx.state.logger; var packageId = ctx.params.packageId; logger.info('updateById', packageId); try { - await next(); - let apiPkg = ctx.request.body; - let pkg = this.mapPackageApiToDb(apiPkg, packageId) + let pkg = ctx.packageRepository.mapPackageApiToDb(apiPkg, packageId) // the update package should take a dictionary of fields instead of package model - await this.repository.updatePackage(pkg); + await ctx.packageRepository.updatePackage(pkg); ctx.response.status = 204; } catch (ex) { - // Need to handle the case were it's 404 vs invalid data - ctx.throw(ex.message, 400); + // Need to handle the case were it's 404 vs invalid data + ctx.throw(400, ex.message); } return; } - // Creates or updates a package using the data provided in the API - async createOrUpdate(ctx: any, next: any) { + /** + * @swagger + * /packages/{packageId}: + * put: + * summary: Create or update a package + * description: Creates or updates a package using the data provided in the API + * operationid: createOrUpdate + * parameters: + * - name: packageId + * description: ID of package to patch + * in: path + * required: true + * type: string + * responses: + * 201: + * description: Created new package + * schema: + * $ref: '#/definitions/Package' + * 204: + * description: Updated existing package + */ + static async createOrUpdate(ctx: any) { var logger : ILogger = ctx.state.logger; var packageId = ctx.params.packageId; logger.info('create', ctx.request.body) try { - await next(); - let apiPkg = ctx.request.body; - let pkg = this.mapPackageApiToDb(apiPkg, packageId); + let pkg = ctx.packageRepository.mapPackageApiToDb(apiPkg, packageId); - var result = await this.repository.addPackage(pkg); + var result = await ctx.packageRepository.addPackage(pkg); switch (result) { case UpsertStatus.Created: - ctx.body = this.mapPackageDbToApi(pkg); + ctx.body = ctx.packageRepository.mapPackageDbToApi(pkg); ctx.response.status = 201; break; case UpsertStatus.Updated: @@ -109,26 +176,58 @@ export class PackageControllers { return; } catch (ex) { - switch (ex.code) { - case MongoErrors.ShardKeyNotFound: - logger.error('Missing shard key', ctx.request.body) - ctx.response.status = 400; - ctx.response.message = "Missing shard key"; - break; - - case MongoErrors.TooManyRequests: - logger.error('Too many requests', ctx.request.body) - ctx.response.status = 429; - break; - - default: - ctx.throw(ex.message, 500); - } + switch (ex.code) { + case MongoErrors.ShardKeyNotFound: + logger.error('Missing shard key', ctx.request.body) + ctx.response.status = 400; + ctx.response.message = "Missing shard key"; + break; + + case MongoErrors.TooManyRequests: + logger.error('Too many requests', ctx.request.body) + ctx.response.status = 429; + break; + + default: + { + ctx.throw(500, ex.message); + } + } } } - // Get summary information about packages from a user - async getSummary(ctx: any, next: any) { + /** + * @swagger + * /packages/summary/{ownerId}: + * get: + * summary: Get summary information about packages from a user + * description: Get summary information about packages from a user + * operationid: getSummary + * parameters: + * - name: ownerId + * description: ID of the package's owner + * in: path + * required: true + * type: string + * - name: year + * description: Year of the summary requested + * in: query + * required: true + * type: integer + * - name: month + * description: Month of the summary requested + * in: path + * required: true + * type: integer + * responses: + * 201: + * description: successful operation + * schema: + * $ref: '#/definitions/PackageUtilization' + * 400: + * description: Invalid ID, year, or month supplied + */ + static async getSummary(ctx: any) { var logger : ILogger = ctx.state.logger; var ownerId = ctx.params.ownerId; @@ -136,8 +235,6 @@ export class PackageControllers { var month = ctx.params.month; logger.info('retrieve summary %s %d/%d', ownerId) - await next(); - let utilization = new apiModels.PackageUtilization(); utilization.totalWeight = 400; diff --git a/src/shipping/package/app/controllers/swagger-controllers.ts b/src/shipping/package/app/controllers/swagger-controllers.ts new file mode 100644 index 00000000..37c95c19 --- /dev/null +++ b/src/shipping/package/app/controllers/swagger-controllers.ts @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +import * as Spec from '../spec/package-swagger'; +import { ILogger } from '../util/logging' + +export class SwaggerControllers { + + static async getSpec(ctx: any) { + var logger : ILogger = ctx.state.logger; + logger.info('Swagger Spec Request %s', "OK"); + + ctx.status = 200; + ctx.body = Spec.PackageServiceSwaggerApi; + } + +} diff --git a/src/shipping/package/app/models/package.ts b/src/shipping/package/app/models/package.ts index 2051aaa0..00195e11 100644 --- a/src/shipping/package/app/models/package.ts +++ b/src/shipping/package/app/models/package.ts @@ -8,12 +8,12 @@ import { ObjectID } from 'mongodb'; export type PackageSize = "small" | "medium" | "large"; export class Package { - readonly _id: ObjectID; + readonly _id: string; tag: string; weight: number; size: PackageSize; constructor(id? : string) { - this._id = id || (new ObjectID(id)).toHexString(); + this._id = id || (new ObjectID()).toHexString(); } -} \ No newline at end of file +} diff --git a/src/shipping/package/app/models/repository.ts b/src/shipping/package/app/models/repository.ts index 9379f9b4..47150334 100644 --- a/src/shipping/package/app/models/repository.ts +++ b/src/shipping/package/app/models/repository.ts @@ -3,7 +3,8 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -import { Package } from "./package" +import { Package, PackageSize } from './package' +import * as apiModels from './api-models' import { Settings } from '../util/settings'; import * as Logger from '../util/logging'; import { MongoErrors } from '../util/mongo-err'; @@ -18,7 +19,7 @@ export enum UpsertStatus { export class Repository { static readonly collectionName = Settings.collectionName(); - private static db; + private static db:any; static async initialize(connection: string) { @@ -53,5 +54,25 @@ export class Repository var collection = this.collection(); return await collection.findOne({_id: id }); // Returns null if not found } + + mapPackageDbToApi(pkg: Package): apiModels.Package { + // consider allowing repository to deal with API types or.. automapping, mapping through convention or config + // we could simpley add/remove/change the model in the api vs database + let apiPkg = new apiModels.Package(); + apiPkg.id = pkg._id; + apiPkg.size = pkg.size ? pkg.size.toString() : null; + apiPkg.tag = pkg.tag; + apiPkg.weight = pkg.weight; + return apiPkg; + } + + mapPackageApiToDb(apiPkg: apiModels.Package, id?: string): Package { + let pkg = new Package(id); + pkg.size = apiPkg.size; + pkg.weight = apiPkg.weight; + pkg.tag = apiPkg.tag; + return pkg; + } + } diff --git a/src/shipping/package/app/routes.ts b/src/shipping/package/app/routes.ts new file mode 100644 index 00000000..0e22ad68 --- /dev/null +++ b/src/shipping/package/app/routes.ts @@ -0,0 +1,46 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +const Router = require('@koa/router'); + +import { PackageControllers, HealthzControllers, SwaggerControllers } from './controllers'; + +export function apiRouter() { + + const router = new Router({ + prefix: '/api' + }); + + router.get('/packages/:packageId', PackageControllers.getById); + router.put('/packages', PackageControllers.createOrUpdate); + router.put('/packages/:packageId', PackageControllers.createOrUpdate); + router.patch('/packages/:packageId', PackageControllers.updateById); + + router.get('/packages/summary/:ownerId', PackageControllers.getSummary); + + return router; +} + +export function healthzRouter() { + + const router = new Router({ + prefix: '/healthz' + }); + + router.get('/', HealthzControllers.getReadinessLiveness); + + return router; +} + +export function swaggerRouter() { + + const router = new Router({ + prefix: '/swagger' + }); + + router.get('/swagger.json', SwaggerControllers.getSpec); + + return router; +} diff --git a/src/shipping/package/app/server.ts b/src/shipping/package/app/server.ts index b25065f2..885c4829 100644 --- a/src/shipping/package/app/server.ts +++ b/src/shipping/package/app/server.ts @@ -3,27 +3,16 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -const _ = require('koa-route'); -import * as Koa from 'koa'; -import * as bodyParser from "koa-bodyparser"; - -var compress = require('koa-compress'); - -const fleekCtx = require('fleek-context'); -const fleekRouter = require('fleek-router'); -const fleekValidator = require('fleek-validator'); - -const SWAGGER = require('./api.json'); - -import { PackageControllers } from './controllers'; +import { KoaApp } from './app'; import { Repository } from './models/repository'; import { Settings } from './util/settings'; -import { logger, ILogger } from './util/logging' export class PackageService { static start() { + const port = process.env.PORT || 80; + console.log('Package service starting...') // Initialize repository with connection string @@ -34,74 +23,12 @@ export class PackageService { process.exit(1); // Crash the container }); - var packageControllers = new PackageControllers(new Repository()); - - let app = new Koa(); - - // Configure logging - app.use(logger(Settings.logLevel())); - - // Add simple health check endpoint - app.use(_.get('/healthz', (ctx) => { - var logger : ILogger = ctx.state.logger; - logger.info('Readiness/Liveness Probe Status: %s', "OK"); - - ctx.status = 200; - ctx.body = {status: 'OK'}; - })); - - // Configure global exception handling - // Use: ctx.throw('Error Message', 500); - // in the controller methods to set the status code and exception message - app.use(async (ctx, next) => { - try { - await next(); - } catch (ex) { - - var logger : ILogger = ctx.state.logger; - if (logger) { - logger.error(ex.message); - } - - ctx.status = ex.status || 500; - // consider api specific codes and localized messages as opposed to internal codes - ctx.body = { - level: "error", - code: ex.code, - message: ex.message - } - ctx.app.emit('error', ex, ctx); - } - }); - - // add compression and body parser to the pipeline - app.use(compress()); - app.use(bodyParser()); - - // configure fleek to handle validation and routing based on api document - app.use(fleekCtx(SWAGGER)); - app.use(fleekValidator().catch((ctx, next) => { - if (ctx.fleek.validation.failed) { - - var logger : ILogger = ctx.state.logger; - if (logger) { - logger.error(ctx.fleek.validation); - } - - ctx.body = ctx.fleek.validation; - ctx.status = 400; - return; - } - return next(); - })); - - let router = new fleekRouter.Router({ controllers: `${__dirname}/controllers` }); + const app = KoaApp.create(Settings.logLevel()); - app.use(router.controllers({ - operation: packageControllers - })); + // Add package repo to the context + app.context.packageRepository = new Repository(); - console.log('listening on port 80'); - app.listen(80); + app.listen(port); + console.log('listening on port %s', port); } } diff --git a/src/shipping/package/app/spec/package-swagger.ts b/src/shipping/package/app/spec/package-swagger.ts new file mode 100644 index 00000000..bb09aaff --- /dev/null +++ b/src/shipping/package/app/spec/package-swagger.ts @@ -0,0 +1,29 @@ +const path = require('path'); +const swaggerJSDoc = require('swagger-jsdoc'); + +const options = { + definition: { + openapi: '3.0.0', + info: { + title: "fabrikam-drone-delivery-package-service", + description: "Fabrikam Drone Delivery Package Service", + version: "0.1.0", + contact: "Microsoft Patterns and Practices", + termsOfService: '' + }, + basePath: '/api', + "schemes": [ + "http", + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ] + }, + apis: [path.join(__dirname, '../controllers/package-controllers.{ts,js}')], +}; + +export const PackageServiceSwaggerApi = swaggerJSDoc(options); diff --git a/src/shipping/package/app/util/logging.ts b/src/shipping/package/app/util/logging.ts index cf25b63b..cb766552 100644 --- a/src/shipping/package/app/util/logging.ts +++ b/src/shipping/package/app/util/logging.ts @@ -26,11 +26,11 @@ const defaultFormat = combine( ); export interface ILogger { - log(level: string, msg: string, meta?: any) - debug(msg: string, meta?: any) - info(msg: string, meta?: any) - warn(msg: string, meta?: any) - error(msg: string, meta?: any) + log(level: string, msg: string, meta?: any) : any + debug(msg: string, meta?: any) : any + info(msg: string, meta?: any) : any + warn(msg: string, meta?: any) : any + error(msg: string, meta?: any) : any } export function logger(level: string) { diff --git a/src/shipping/package/app/util/settings.ts b/src/shipping/package/app/util/settings.ts index 786d9e24..0dbaaedd 100644 --- a/src/shipping/package/app/util/settings.ts +++ b/src/shipping/package/app/util/settings.ts @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -import process = require("process") +const process = require("process") export class Settings { static collectionName() : string { diff --git a/src/shipping/package/babel.config.js b/src/shipping/package/babel.config.js new file mode 100644 index 00000000..694b4a38 --- /dev/null +++ b/src/shipping/package/babel.config.js @@ -0,0 +1,13 @@ +module.exports = { + presets: [ + [ + '@babel/preset-env', + { + targets: { + node: 'current', + }, + }, + ], + '@babel/preset-typescript', + ], +}; diff --git a/src/shipping/package/gulpfile.js b/src/shipping/package/gulpfile.js index d0d53755..22659348 100644 --- a/src/shipping/package/gulpfile.js +++ b/src/shipping/package/gulpfile.js @@ -13,7 +13,7 @@ gulp.task('set-env', gulp.series(function(done) { })); // Generate types that can be used when working with api definitions -gulp.task ('create-api-classes', gulp.series(function() { +gulp.task ('create-api-classes', function(done) { let generated = "// GENERATED FILE - DO NOT EDIT\n\n"; @@ -47,13 +47,16 @@ gulp.task ('create-api-classes', gulp.series(function() { return console.log(err); } }); -})); + + done(); + +}); // delete all build output files gulp.task('clean', gulp.series(function (done) { return del(['.bin/'], done); })); - + gulp.task('build', gulp.series('clean', function () { gulp.src('**/*.json', {'cwd':'app'}) .pipe(gulp.dest('.bin/app')); diff --git a/src/shipping/package/jest.config.js b/src/shipping/package/jest.config.js new file mode 100644 index 00000000..2c379d78 --- /dev/null +++ b/src/shipping/package/jest.config.js @@ -0,0 +1,32 @@ +module.exports = { + testEnvironment: 'node', + bail: true, + verbose: true, + setupFilesAfterEnv: [ + 'jest-extended' + ], + testPathIgnorePatterns: [ + '/node_modules/' + ], + "moduleFileExtensions": ["js", "jsx", "json", "ts", "tsx"], + "collectCoverage": true, + "collectCoverageFrom": [ + "**/*.{ts,tsx,js,jsx}", + "!**/tests/models/*.{ts,tsx,js,jsx}", + "!**/node_modules/**", + "!**/build/**", + "!**/coverage/**" + ], + "transform": { + "\\.ts$": "ts-jest" + }, + "coverageThreshold": { + "global": { + "lines": 60, + } + }, + "coverageReporters": [ + "text", + "text-summary" + ] +}; diff --git a/src/shipping/package/package-service.md b/src/shipping/package/package-service.md deleted file mode 100644 index 067aace5..00000000 --- a/src/shipping/package/package-service.md +++ /dev/null @@ -1,11 +0,0 @@ -# Package service - -## Create local development environment - -1. Install Docker and docker-compose -2. From a bash CLI navigate to the project root and type `./up.sh` -3. try ```curl -X PUT --header 'Accept: application/json' 'http://localhost:7080/api/packages/42'``` - -> Known issue: -> ```'failed to connect to server [packagedb:27017] on first connect [MongoError: connect ECONNREFUSED 172.24.0.2:27017]'``` -> First time the package service starts, it might fail connecting to mongo. Package would automatically restart if needed. diff --git a/src/shipping/package/package.json b/src/shipping/package/package.json index 21f9060c..d78938af 100644 --- a/src/shipping/package/package.json +++ b/src/shipping/package/package.json @@ -1,11 +1,12 @@ { - "name": "drone-delivery-package-service", - "version": "0.0.1", - "description": "Drone Delivery Package Service", + "name": "fabrikam-drone-delivery-package-service", + "version": "0.1.0", + "description": "Fabrikam Drone Delivery Package Service", "scripts": { "start": "gulp serve", "build": "gulp build", "clean": "gulp clean", + "test": "jest", "api": "gulp create-api-classes", "int-test": "gulp int-test", "ncu": "ncu", @@ -23,25 +24,26 @@ "author": "Microsoft Patterns and Practices", "license": "MIT", "dependencies": { + "@koa/router": "^8.0.8", "applicationinsights": "^1.2.0", - "fleek-context": "^1.0.6", - "fleek-router": "^2.1.0", - "fleek-validator": "^2.0.1", "koa": "^2.6.1", - "koa-body": "^4.0.4", "koa-bodyparser": "^4.2.1", "koa-compress": "^3.0.0", - "koa-convert": "^1.2.0", - "koa-cors": "0.0.16", - "koa-route": "^3.2.0", "mongodb": "3.2.1", + "swagger-jsdoc": "^4.0.0", "winston": "^3.2.1" }, "devDependencies": { - "@types/koa": "^2.0.48", + "@babel/core": "^7.9.0", + "@babel/preset-env": "^7.9.0", + "@babel/preset-typescript": "^7.9.0", + "@types/jest": "^25.1.4", + "@types/koa": "^2.11.3", "@types/koa-bodyparser": "^5.0.1", - "@types/mocha": "^5.2.6", + "@types/koa-router": "^7.4.0", + "@types/mongodb": "^3.5.4", "@types/node": "^11.10.5", + "babel-jest": "^25.2.3", "del": "^4.0.0", "gulp": "^4.0.0", "gulp-cli": "^2.0.1", @@ -49,9 +51,14 @@ "gulp-nodemon": "^2.4.2", "gulp-typescript": "^5.0.0", "gulp-util": "^3.0.8", + "jest": "^25.2.3", + "jest-extended": "^0.11.5", "mocha": "^6.0.2", - "nodemon": "^1.18.10", + "nodemon": "^1.19.4", "npm-check-updates": "^3.0.2", - "typescript": "^3.3.3333" + "supertest": "^4.0.2", + "ts-jest": "^25.2.1", + "ts-node": "^8.8.1", + "typescript": "^3.8.3" } } diff --git a/src/shipping/package/readme.md b/src/shipping/package/readme.md index e69de29b..067aace5 100644 --- a/src/shipping/package/readme.md +++ b/src/shipping/package/readme.md @@ -0,0 +1,11 @@ +# Package service + +## Create local development environment + +1. Install Docker and docker-compose +2. From a bash CLI navigate to the project root and type `./up.sh` +3. try ```curl -X PUT --header 'Accept: application/json' 'http://localhost:7080/api/packages/42'``` + +> Known issue: +> ```'failed to connect to server [packagedb:27017] on first connect [MongoError: connect ECONNREFUSED 172.24.0.2:27017]'``` +> First time the package service starts, it might fail connecting to mongo. Package would automatically restart if needed. diff --git a/src/shipping/package/test/models/test-data-access.ts b/src/shipping/package/tests/models/test-data-access.ts similarity index 100% rename from src/shipping/package/test/models/test-data-access.ts rename to src/shipping/package/tests/models/test-data-access.ts diff --git a/src/shipping/package/tests/unit/healthz-controllers.test.ts b/src/shipping/package/tests/unit/healthz-controllers.test.ts new file mode 100644 index 00000000..caea169e --- /dev/null +++ b/src/shipping/package/tests/unit/healthz-controllers.test.ts @@ -0,0 +1,39 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +const supertest = require('supertest'); + +import { KoaApp } from '../../app/app'; + +const app = KoaApp.create('debug'); + +const server = app.listen(); + +afterAll((done) => { + server.close(done) +}); + +describe('HealthzControllers', () => { + const request = supertest(server); + + describe('GET /', () => { + it('<200> should always return with the package utilization information', async () => { + //Arrange + const expected = ['status']; + + // Act + const res = await request + .get('/healthz') + .expect('Content-Type', /json/) + .expect(200); + const pkg = res.body; + + // Assert + expect(Object.keys(pkg)).toEqual(expect.arrayContaining(expected)); + expect(pkg.status).toBe("OK"); + }); + }); + +}); diff --git a/src/shipping/package/tests/unit/package-controllers.test.ts b/src/shipping/package/tests/unit/package-controllers.test.ts new file mode 100644 index 00000000..f24cef19 --- /dev/null +++ b/src/shipping/package/tests/unit/package-controllers.test.ts @@ -0,0 +1,173 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +const supertest = require('supertest'); + +import * as apiModels from '../../app/models/api-models' +import { Package } from '../../app/models/package' + +import { KoaApp } from '../../app/app'; + +const app = KoaApp.create('debug'); + +// Add mocks to the context +const mockFindPackage = jest.fn(id => (id === "507f1f77bcf86cd799439011") ? new Package(id) : null); +const mockMapPackageDbToApi = jest.fn(pkg => { + if (pkg == null) return null; + + var pkgApi = new apiModels.Package(); + pkgApi.id = "507f1f77bcf86cd799439011"; + pkgApi.size = "small"; + pkgApi.weight = 1; + pkgApi.tag = "test"; + + return pkgApi; +}); + +const mockMapPackageApiToDb = jest.fn((pkg, id) => new Package(id)); +const mockAddPackage = jest.fn(pkg => { + switch (pkg._id) { + case "42": + return 2; + case "43": + throw new Error("mock error"); + default: + return 1; + } +}); + +const mockUpdatePackage = jest.fn(pkg => {return;}); +app.context.packageRepository = { + findPackage: mockFindPackage, + mapPackageDbToApi: mockMapPackageDbToApi, + mapPackageApiToDb: mockMapPackageApiToDb, + addPackage: mockAddPackage, + updatePackage: mockUpdatePackage +}; + +const server = app.listen(); + +afterAll((done) => { + server.close(done) +}); + +describe('PackageControllers', () => { + const request = supertest(server); + + describe('GET /', () => { + it('<404> should always return when the package id does not exist', async () => { + //Arrange + const ramdonId = "507f1f77bcf86cd799439012"; + + // Act + const res = await request + .get('/api/packages/' + ramdonId); + // Assert + expect(res.status).toBe(404); + }); + }); + + describe('GET /', () => { + it('<200> should always return with the package information', async () => { + //Arrange + const ramdonId = "507f1f77bcf86cd799439011"; + const expected = ['id', 'size', 'weight', 'tag']; + + // Act + const res = await request + .get('/api/packages/' + ramdonId) + .expect('Content-Type', /json/) + .expect(200); + const pkg = res.body; + + // Assert + expect(Object.keys(pkg)).toEqual(expect.arrayContaining(expected)); + expect(pkg.id).toBe("507f1f77bcf86cd799439011"); + expect(pkg.size).toBe("small"); + expect(pkg.weight).toBe(1); + expect(pkg.tag).toBe("test"); + }); + }); + + describe('PUT /', () => { + it('<204> should always return if exists', async () => { + //Arrange + const id = "42"; + + // Act + const res = await request + .put('/api/packages/' + id); + + // Assert + expect(res.status).toBe(204); + + }); + }); + + describe('PUT /', () => { + it('<201> should always return if not exist and autogenerate id', async () => { + //Arrange + const id = ""; + + // Act + const res = await request + .put('/api/packages/' + id); + + // Assert + expect(res.status).toBe(201); + + }); + }); + + describe('PATCH /', () => { + it('<204> should always return if updated', async () => { + //Arrange + const id = "42"; + + // Act + const res = await request + .patch('/api/packages/' + id); + + // Assert + expect(res.status).toBe(204); + + }); + }); + + describe('PUT /', () => { + it('<500> should return 500 if something went really wrong', async () => { + //Arrange + const id = "43"; + + // Act + const res = await request + .put('/api/packages/' + id); + + // Assert + expect(res.status).toBe(500); + + }); + }); + + describe('GET /', () => { + it('<200> should always return with the package utilization information', async () => { + //Arrange + const ownerId = "42"; + const expected = ['totalWeight']; + + // Act + const res = await request + .get('/api/packages/summary/' + ownerId) + .expect('Content-Type', /json/) + .expect(200); + const pkg = res.body; + + // Assert + expect(Object.keys(pkg)).toEqual(expect.arrayContaining(expected)); + expect(pkg.totalWeight).toBe(400); + }); + }); + +}); diff --git a/src/shipping/package/tests/unit/swagger-controllers.test.ts b/src/shipping/package/tests/unit/swagger-controllers.test.ts new file mode 100644 index 00000000..4b857eaa --- /dev/null +++ b/src/shipping/package/tests/unit/swagger-controllers.test.ts @@ -0,0 +1,45 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +const pkg = require('../../package.json'); +const supertest = require('supertest'); + +import { KoaApp } from '../../app/app'; + +const app = KoaApp.create('debug'); + +const server = app.listen(); + +afterAll((done) => { + server.close(done) +}); + +describe('SwaggerControllers', () => { + const request = supertest(server); + + describe('GET /', () => { + it('<200> should always return with the openAPI specinformation', async () => { + // Arrange + // N/A + + // Act + const res = await request + .get('/swagger/swagger.json') + .expect('Content-Type', /json/) + .expect(200); + + const spec = res.body; + + // Assert + const expected = ["openapi", "info", "basePath", "schemes", "consumes", "produces", "paths", "definitions", "components", "tags"]; + expect(Object.keys(spec)).toEqual(expect.arrayContaining(expected)); + expect(spec.info.title).toBe(pkg.name); + expect(spec.info.version).toBe(pkg.version); + expect(spec.info.description).toBe(pkg.description); + expect(spec.info.contact).toBe(pkg.author); + }); + }); + +}); diff --git a/src/shipping/package/tests/unit/util-settings.test.ts b/src/shipping/package/tests/unit/util-settings.test.ts new file mode 100644 index 00000000..84936e35 --- /dev/null +++ b/src/shipping/package/tests/unit/util-settings.test.ts @@ -0,0 +1,56 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. +// ------------------------------------------------------------ + +const supertest = require('supertest'); + +import { Settings } from '../../app/util/settings'; + +describe('Util Settings', () => { + + describe('Settings default values', () => { + it('settings should return fake values', async () => { + // Arrange + // N/A + + // Act + const colNameReceived = Settings.collectionName(); + const connStrReceived = Settings.connectionString(); + const containerNameReceived = Settings.containerName(); + const logLevelReceived = Settings.logLevel(); + + // Assert + expect(colNameReceived).not.toBeNull(); + expect(connStrReceived).not.toBeNull(); + expect(containerNameReceived).not.toBeNull(); + expect(logLevelReceived).not.toBeNull(); + }); + + }); describe('Settings values', () => { + it('settings should return fake values', async () => { + // Arrange + const colNameExpected = "test-col"; + process.env["COLLECTION_NAME"] = colNameExpected; + const connStrExpected = "test-connstr"; + process.env["CONNECTION_STRING"] = connStrExpected; + const containerNameExpected = "test-container"; + process.env["CONTAINER_NAME"] = containerNameExpected; + const logLevelExpected = "test-loglvl"; + process.env["LOG_LEVEL"] = logLevelExpected; + + // Act + const colNameReceived = Settings.collectionName(); + const connStrReceived = Settings.connectionString(); + const containerNameReceived = Settings.containerName(); + const logLevelReceived = Settings.logLevel(); + + // Assert + expect(colNameReceived).toBe(colNameExpected); + expect(connStrReceived).toBe(connStrExpected); + expect(containerNameReceived).toBe(containerNameExpected); + expect(logLevelReceived).toBe(logLevelExpected); + }); + }); + +}); diff --git a/src/shipping/package/tsconfig.json b/src/shipping/package/tsconfig.json index 405d81c7..5ea83103 100644 --- a/src/shipping/package/tsconfig.json +++ b/src/shipping/package/tsconfig.json @@ -2,13 +2,15 @@ "compilerOptions": { "outDir": ".bin", "module": "commonjs", - "target": "ES2015", - "noImplicitAny": false, - "sourceMap": true + "target": "es2017", + "noImplicitAny": true, + "sourceMap": true, + "resolveJsonModule": true, + "esModuleInterop": true }, "exclude": [ "node_modules" ], "typeRoots": ["node_modules/@types"], "types": [".*"] -} \ No newline at end of file +} From 49b08daa85f961a69b110fb7c594b6a7c36080b7 Mon Sep 17 00:00:00 2001 From: Fer Antivero Date: Wed, 1 Apr 2020 19:36:00 +0000 Subject: [PATCH 243/246] Merged PR 22994: move add pod identity and agic to kube-system move add pod identity and agic to kube-system --- deployment.md | 8 ++++---- deploymentCICD.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deployment.md b/deployment.md index 39528d10..00bd585b 100644 --- a/deployment.md +++ b/deployment.md @@ -155,8 +155,7 @@ sudo az aks install-cli az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME # Create namespaces -kubectl create namespace backend-dev && \ -kubectl create namespace ingress-controllers +kubectl create namespace backend-dev ``` Setup Helm @@ -190,7 +189,8 @@ Note: the tested nmi version was 1.4. It enables namespaced pod identity. ```bash # setup AAD pod identity -kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml +helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/chart && \ +helm install aad-pod-identity/aad-pod-identity -n kube-system kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` @@ -208,7 +208,7 @@ export APP_GATEWAY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azurede export APP_GATEWAY_PUBLIC_IP_FQDN=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appGatewayPublicIpFqdn.value -o tsv) helm install ingress-azure-dev application-gateway-kubernetes-ingress/ingress-azure \ - --namespace ingress-controllers \ + --namespace kube-system \ --set appgw.name=$APP_GATEWAY_NAME \ --set appgw.resourceGroup=$RESOURCE_GROUP \ --set appgw.subscriptionId=$SUBSCRIPTION_ID \ diff --git a/deploymentCICD.md b/deploymentCICD.md index d5d9701e..57f272f6 100644 --- a/deploymentCICD.md +++ b/deploymentCICD.md @@ -100,8 +100,7 @@ az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME kubectl create namespace backend-dev && \ kubectl create namespace backend-qa && \ kubectl create namespace backend-staging && \ -kubectl create namespace backend && \ -kubectl create namespace ingress-controllers +kubectl create namespace backend ``` Setup Helm @@ -126,7 +125,8 @@ Note: the tested nmi version was 1.4. It enables namespaced pod identity. ```bash # setup AAD pod identity -kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml +helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/chart && \ +helm install aad-pod-identity/aad-pod-identity -n kube-system kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml ``` @@ -242,7 +242,7 @@ export {${ENV}_APP_GATEWAY_PUBLIC_IP_FQDN,APP_GATEWAY_PUBLIC_IP_FQDN}=$(az group export ENV_NAMESPACE=$([ $env == 'prod' ] && echo 'backend' || echo "backend-$env") helm install ingress-azure-${env} application-gateway-kubernetes-ingress/ingress-azure \ - --namespace ingress-controllers \ + --namespace kube-system \ --set appgw.name=$APP_GATEWAY_NAME \ --set appgw.resourceGroup=$RESOURCE_GROUP \ --set appgw.subscriptionId=$SUBSCRIPTION_ID \ From 0d3f12049f6e0a189a52b8d3213bb0bbad78867f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Sun, 5 Apr 2020 16:08:54 -0300 Subject: [PATCH 244/246] required permissions note (#155) add a note to the deployment steps indicating required permissions --- deployment.md | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment.md b/deployment.md index 134b1b09..52f8de94 100644 --- a/deployment.md +++ b/deployment.md @@ -3,6 +3,7 @@ ## Prerequisites - Azure subscription + > Important: The user initiating the deployment process must have access to the `Microsoft.Authorization/roleAssignments/write` permission. For more information, please refer to [the Container Insights doc](https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-troubleshoot#authorization-error-during-onboarding-or-update-operation) - [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - [Docker](https://docs.docker.com/) - [JQ](https://stedolan.github.io/jq/download/) From 864f71ccb884f446a6b3bd66bd75df1711868710 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 28 Apr 2020 10:53:07 -0300 Subject: [PATCH 245/246] add missing s the the add-pod-identity chart repo (#157) --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 61ca40b7..e00dd725 100644 --- a/deployment.md +++ b/deployment.md @@ -190,7 +190,7 @@ Note: the tested nmi version was 1.4. It enables namespaced pod identity. ```bash # setup AAD pod identity -helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/chart && \ +helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts && \ helm install aad-pod-identity/aad-pod-identity -n kube-system kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml From 4ce4f853527ced7a4f50652edb3676d53e0bdb62 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 28 Apr 2020 10:58:04 -0300 Subject: [PATCH 246/246] add more links to our docs (#159) * add more links to our docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2bdcc2c1..93c82d7a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ This reference implementation shows a set of best practices for building and run This project has a companion set of articles that describe challenges, design patterns, and best practices for building microservices architecture. You can find these articles on the Azure Architecture Center: - [Designing, building, and operating microservices on Azure with Kubernetes](https://docs.microsoft.com/azure/architecture/microservices) +- [Microservices architecture on Azure Kubernetes Service (AKS)](https://docs.microsoft.com/azure/architecture/reference-architectures/microservices/aks) +- [Building a CI/CD pipeline for microservices on Kubernetes](https://docs.microsoft.com/azure/architecture/microservices/ci-cd-kubernetes) ## Scenario