Merge pull request kubeflow#405 from lightbend/promfix

Various Prometheus-related fixes and improvements
puneetloya · Feb 14, 2019 · ba9258d · ba9258d
2 parents 822726d + 11f80c4
commit ba9258d
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 32 deletions.
diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml
@@ -26,6 +26,8 @@ spec:
   imagePullPolicy: Always
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar"
+  arguments:
+    - "100000"
   sparkVersion: "2.4.0"
   restartPolicy:
     type: Never
@@ -47,4 +49,4 @@ spec:
     exposeExecutorMetrics: true
     prometheus:
       jmxExporterJar: "/prometheus/jmx_prometheus_javaagent-0.3.1.jar"
-      port: 8090 
+      port: 8090
diff --git a/pkg/config/constants.go b/pkg/config/constants.go
@@ -177,84 +177,84 @@ const DefaultPrometheusConfiguration = `
 lowercaseOutputName: true
 attrNameSnakeCase: true
 rules:
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.(BlockManager|DAGScheduler|jvm)\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.(BlockManager|DAGScheduler|jvm)\.(\S+)><>Value
     name: spark_driver_$3_$4
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.(\S+)\.StreamingMetrics\.streaming\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.(\S+)\.StreamingMetrics\.streaming\.(\S+)><>Value
     name: spark_streaming_driver_$4
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.spark\.streaming\.(\S+)\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.spark\.streaming\.(\S+)\.(\S+)><>Value
     name: spark_structured_streaming_driver_$4
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
       query_name: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.(\S+)\.executor\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.(\S+)\.executor\.(\S+)><>Value
     name: spark_executor_$4
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
       executor_id: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.DAGScheduler\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.DAGScheduler\.(.*)><>Count
     name: spark_driver_DAGScheduler_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.HiveExternalCatalog\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.HiveExternalCatalog\.(.*)><>Count
     name: spark_driver_HiveExternalCatalog_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.CodeGenerator\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.CodeGenerator\.(.*)><>Count
     name: spark_driver_CodeGenerator_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.LiveListenerBus\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.LiveListenerBus\.(.*)><>Count
     name: spark_driver_LiveListenerBus_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.LiveListenerBus\.(.*)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.LiveListenerBus\.(.*)><>Value
     name: spark_driver_LiveListenerBus_$3
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
-  - pattern: metrics<name=(\S+)/(\S+)\.(.*)\.executor\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.(.*)\.executor\.(.*)><>Count
     name: spark_executor_$4_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
       executor_id: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.(jvm|NettyBlockTransfer)\.(.*)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.(jvm|NettyBlockTransfer)\.(.*)><>Value
     name: spark_executor_$4_$5
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
       executor_id: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.HiveExternalCatalog\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.HiveExternalCatalog\.(.*)><>Count
     name: spark_executor_HiveExternalCatalog_$4_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
       executor_id: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.CodeGenerator\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.CodeGenerator\.(.*)><>Count
     name: spark_executor_CodeGenerator_$4_count
     type: COUNTER
     labels:

diff --git a/pkg/controller/sparkapplication/monitoring_config.go b/pkg/controller/sparkapplication/monitoring_config.go
@@ -18,7 +18,6 @@ package sparkapplication
 
 import (
 	"fmt"
-
 	corev1 "k8s.io/api/core/v1"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,7 +31,7 @@ import (
 const (
 	metricsPropertiesKey          = "metrics.properties"
 	prometheusConfigKey           = "prometheus.yaml"
-	prometheusConfigMapNameSuffix = "prometheus-config"
+	prometheusConfigMapNameSuffix = "prom-conf"
 	prometheusConfigMapMountPath  = "/etc/metrics/conf"
 	prometheusScrapeAnnotation    = "prometheus.io/scrape"
 	prometheusPortAnnotation      = "prometheus.io/port"

diff --git a/pkg/webhook/patch.go b/pkg/webhook/patch.go
@@ -205,7 +205,7 @@ func addGeneralConfigMaps(pod *corev1.Pod) []*patchOperation {
 	var patchOps []*patchOperation
 	namesToMountPaths := config.FindGeneralConfigMaps(pod.Annotations)
 	for name, mountPath := range namesToMountPaths {
-		volumeName := name + "-volume"
+		volumeName := name + "-vol"
 		patchOps = append(patchOps, addConfigMapVolume(pod, name, volumeName))
 		patchOps = append(patchOps, addConfigMapVolumeMount(pod, volumeName, mountPath))
 	}

diff --git a/pkg/webhook/patch_test.go b/pkg/webhook/patch_test.go
@@ -223,7 +223,7 @@ func TestPatchSparkPod_ConfigMaps(t *testing.T) {
 	}
 
 	assert.Equal(t, 1, len(modifiedPod.Spec.Volumes))
-	assert.Equal(t, "foo-volume", modifiedPod.Spec.Volumes[0].Name)
+	assert.Equal(t, "foo-vol", modifiedPod.Spec.Volumes[0].Name)
 	assert.True(t, modifiedPod.Spec.Volumes[0].ConfigMap != nil)
 	assert.Equal(t, 1, len(modifiedPod.Spec.Containers[0].VolumeMounts))
 	assert.Equal(t, "/path/to/foo", modifiedPod.Spec.Containers[0].VolumeMounts[0].MountPath)

diff --git a/spark-docker/conf/prometheus.yaml b/spark-docker/conf/prometheus.yaml
@@ -20,22 +20,22 @@ attrNameSnakeCase: true
 rules:
   # These come from the application driver if it's a streaming application
   # Example: default/streaming.driver.com.example.ClassName.StreamingMetrics.streaming.lastCompletedBatch_schedulingDelay
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.(\S+)\.StreamingMetrics\.streaming\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.(\S+)\.StreamingMetrics\.streaming\.(\S+)><>Value
     name: spark_streaming_driver_$4
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # These come from the application driver if it's a structured streaming application
   # Example: default/sstreaming.driver.spark.streaming.QueryName.inputRate-total
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.spark\.streaming\.(\S+)\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.spark\.streaming\.(\S+)\.(\S+)><>Value
     name: spark_structured_streaming_driver_$4
     labels:
       app_namespace: "$1"
       app_name: "$2"
       query_name: "$3"
   # These come from the application executors
   # Example: default/spark-pi.0.executor.threadpool.activeTasks
-  - pattern: metrics<name=(\S+)/(\S+)\.(\S+)\.executor\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.(\S+)\.executor\.(\S+)><>Value
     name: spark_executor_$4
     type: GAUGE
     labels:
@@ -44,52 +44,52 @@ rules:
       executor_id: "$3"
   # These come from the application driver
   # Example: default/spark-pi.driver.DAGScheduler.stage.failedStages
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.(BlockManager|DAGScheduler|jvm)\.(\S+)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.(BlockManager|DAGScheduler|jvm)\.(\S+)><>Value
     name: spark_driver_$3_$4
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # These come from the application driver
   # Emulate timers for DAGScheduler like messagePRocessingTime
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.DAGScheduler\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.DAGScheduler\.(.*)><>Count
     name: spark_driver_DAGScheduler_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # HiveExternalCatalog is of type counter
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.HiveExternalCatalog\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.HiveExternalCatalog\.(.*)><>Count
     name: spark_driver_HiveExternalCatalog_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # These come from the application driver
   # Emulate histograms for CodeGenerator
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.CodeGenerator\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.CodeGenerator\.(.*)><>Count
     name: spark_driver_CodeGenerator_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # These come from the application driver
   # Emulate timer (keep only count attribute) plus counters for LiveListenerBus
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.LiveListenerBus\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.LiveListenerBus\.(.*)><>Count
     name: spark_driver_LiveListenerBus_$3_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # Get Gauge type metrics for LiveListenerBus
-  - pattern: metrics<name=(\S+)/(\S+)\.driver\.LiveListenerBus\.(.*)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.driver\.LiveListenerBus\.(.*)><>Value
     name: spark_driver_LiveListenerBus_$3
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
   # Executors counters
-  - pattern: metrics<name=(\S+)/(\S+)\.(.*)\.executor\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.(.*)\.executor\.(.*)><>Count
     name: spark_executor_$4_count
     type: COUNTER
     labels:
@@ -98,14 +98,14 @@ rules:
       executor_id: "$3"
   # These come from the application executors
   # Example: app-20160809000059-0000.0.jvm.threadpool.activeTasks
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.(jvm|NettyBlockTransfer)\.(.*)><>Value
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.(jvm|NettyBlockTransfer)\.(.*)><>Value
     name: spark_executor_$4_$5
     type: GAUGE
     labels:
       app_namespace: "$1"
       app_name: "$2"
       executor_id: "$3"
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.HiveExternalCatalog\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.HiveExternalCatalog\.(.*)><>Count
     name: spark_executor_HiveExternalCatalog_$4_count
     type: COUNTER
     labels:
@@ -114,10 +114,10 @@ rules:
       executor_id: "$3"
   # These come from the application driver
   # Emulate histograms for CodeGenerator
-  - pattern: metrics<name=(\S+)/(\S+)\.([0-9]+)\.CodeGenerator\.(.*)><>Count
+  - pattern: metrics<name=(\S+)-(\S+)\.([0-9]+)\.CodeGenerator\.(.*)><>Count
     name: spark_executor_CodeGenerator_$4_count
     type: COUNTER
     labels:
       app_namespace: "$1"
       app_name: "$2"
-      executor_id: "$3"
+      executor_id: "$3"