e2e/local-testing: prepare for enabling etcd RBAC

This commit adds new etcd client certificates to be generated in testing
environments, as with etcd RBAC enabled, each certificate's CN will
represent a separate user. The intention is to have following users:
- root - fully-privileged user for administrative actions
- kube-apiserver - user dedicated for kube-apiserver, also fully
  privileged for the time being because of
  etcd-io/etcd#8458.
- prometheus - user for Prometheus to scrape etcd metrics

For local-testing, we also add rendering of few scripts, which can be
used for testing with etcdctl and to manually enable RBAC on etcd
cluster.

Signed-off-by: Mateusz Gozdek <[email protected]>

e2e/main.tf

@@ -21,7 +21,11 @@ module "etcd_pki" {
   server_ips   = local.controller_ips
   server_names = local.controller_names
 
-  client_cns = ["kube-apiserver-etcd-client"]
+  client_cns = [
+    "root",
+    "kube-apiserver",
+    "prometheus",
+  ]
 
   organization = "example"
 }
@@ -58,8 +62,8 @@ locals {
     kubelet_client_certificate     = module.kubernetes_pki.kubernetes_api_server_kubelet_client_cert
     kubelet_client_key             = module.kubernetes_pki.kubernetes_api_server_kubelet_client_key
     etcd_ca_certificate            = module.etcd_pki.etcd_ca_cert
-    etcd_client_certificate        = module.etcd_pki.client_certs[0]
-    etcd_client_key                = module.etcd_pki.client_keys[0]
+    etcd_client_certificate        = module.etcd_pki.client_certs[1]
+    etcd_client_key                = module.etcd_pki.client_keys[1]
     etcd_servers                   = formatlist("https://%s:2379", module.etcd_pki.etcd_peer_ips)
     replicas                       = var.controllers_count
   })
@@ -241,8 +245,8 @@ resource "flexkube_controlplane" "bootstrap" {
     kubelet_client_key         = module.kubernetes_pki.kubernetes_api_server_kubelet_client_key
     service_account_public_key = module.kubernetes_pki.service_account_public_key
     etcd_ca_certificate        = module.etcd_pki.etcd_ca_cert
-    etcd_client_certificate    = module.etcd_pki.client_certs[0]
-    etcd_client_key            = module.etcd_pki.client_keys[0]
+    etcd_client_certificate    = module.etcd_pki.client_certs[1]
+    etcd_client_key            = module.etcd_pki.client_keys[1]
     service_cidr               = "11.0.0.0/24"
     etcd_servers               = formatlist("https://%s:2379", module.etcd_pki.etcd_peer_ips)
     bind_address               = local.bootstrap_api_bind

local-testing/files.tf

@@ -57,3 +57,78 @@ resource "local_file" "kubelet_pool_state" {
   sensitive_content = flexkube_kubelet_pool.controller.state_yaml
   filename          = "./resources/kubelet-pool/state.yaml"
 }
+
+resource "local_file" "etcd_ca_certificate" {
+  content  = module.etcd_pki.etcd_ca_cert
+  filename = "./resources/etcd-cluster/ca.pem"
+}
+
+resource "local_file" "etcd_root_user_certificate" {
+  content  = module.etcd_pki.client_certs[0]
+  filename = "./resources/etcd-cluster/client.pem"
+}
+
+resource "local_file" "etcd_root_user_private_key" {
+  sensitive_content = module.etcd_pki.client_keys[0]
+  filename          = "./resources/etcd-cluster/client.key"
+}
+
+resource "local_file" "etcd_prometheus_user_certificate" {
+  content  = module.etcd_pki.client_certs[2]
+  filename = "./resources/etcd-cluster/prometheus_client.pem"
+}
+
+resource "local_file" "etcd_prometheus_user_private_key" {
+  sensitive_content = module.etcd_pki.client_keys[2]
+  filename          = "./resources/etcd-cluster/prometheus_client.key"
+}
+
+resource "local_file" "etcd_environment" {
+  filename = "./resources/etcd-cluster/environment.sh"
+  content  = <<EOF
+#!/bin/bash
+export ETCDCTL_API=3
+export ETCDCTL_CACERT=${abspath(local_file.etcd_ca_certificate.filename)}
+export ETCDCTL_CERT=${abspath(local_file.etcd_root_user_certificate.filename)}
+export ETCDCTL_KEY=${abspath(local_file.etcd_root_user_private_key.filename)}
+export ETCDCTL_ENDPOINTS=${join(",", formatlist("https://%s:2379", module.etcd_pki.etcd_peer_ips))}
+EOF
+
+  depends_on = [
+    flexkube_etcd_cluster.etcd,
+  ]
+}
+
+resource "local_file" "etcd_prometheus_environment" {
+  filename = "./resources/etcd-cluster/prometheus-environment.sh"
+  content  = <<EOF
+#!/bin/bash
+export ETCDCTL_API=3
+export ETCDCTL_CACERT=${abspath(local_file.etcd_ca_certificate.filename)}
+export ETCDCTL_CERT=${abspath(local_file.etcd_prometheus_user_certificate.filename)}
+export ETCDCTL_KEY=${abspath(local_file.etcd_prometheus_user_private_key.filename)}
+export ETCDCTL_ENDPOINTS=${join(",", formatlist("https://%s:2379", module.etcd_pki.etcd_peer_ips))}
+EOF
+
+  depends_on = [
+    flexkube_etcd_cluster.etcd,
+  ]
+}
+
+resource "local_file" "etcd_enable_rbac" {
+  filename = "./resources/etcd-cluster/enable-rbac.sh"
+  content  = <<EOF
+#!/bin/bash
+etcdctl user add --no-password=true root
+etcdctl role add root
+etcdctl user grant-role root root
+etcdctl auth enable
+etcdctl user add --no-password=true kube-apiserver
+etcdctl role add kube-apiserver
+etcdctl role grant-permission kube-apiserver readwrite --prefix=true /
+etcdctl user grant-role kube-apiserver kube-apiserver
+# Until https://github.com/etcd-io/etcd/issues/8458 is resolved.
+etcdctl user grant-role kube-apiserver root
+etcdctl user add --no-password=true prometheus
+EOF
+}