Added support for some config options new in Spark 3.0.0 (kubeflow#954)

Dockerfile

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.5
+ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v3.0.0
 
 FROM golang:1.14.1-alpine as builder
 
@@ -35,6 +35,7 @@ COPY pkg/ pkg/
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /usr/bin/spark-operator main.go
 
 FROM ${SPARK_IMAGE}
+USER root
 COPY --from=builder /usr/bin/spark-operator /usr/bin/
 RUN apt-get update \
     && apt-get install -y openssl curl tini \

docs/api-docs.md

@@ -936,6 +936,32 @@ Kubernetes core/v1.Lifecycle
 <p>Lifecycle for running preStop or postStart commands</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>kubernetesMaster</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>KubernetesMaster is the URL of the Kubernetes master used by the driver to manage executor pods and
+other Kubernetes resources. Default to <a href="https://kubernetes.default.svc">https://kubernetes.default.svc</a>.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>serviceAnnotations</code></br>
+<em>
+map[string]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ServiceAnnotations defines the annoations to be added to the Kubernetes headless service used by
+executors to connect to the driver.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="sparkoperator.k8s.io/v1beta2.DriverState">DriverState
@@ -1138,8 +1164,8 @@ string
 </td>
 <td>
 <em>(Optional)</em>
-<p>MetricsPropertiesFile is the container local path of file metrics.properties for configuring the Spark metric system.
-If not specified, value /etc/metrics/conf/metrics.properties will be used.</p>
+<p>MetricsPropertiesFile is the container local path of file metrics.properties for configuring
+the Spark metric system. If not specified, value /etc/metrics/conf/metrics.properties will be used.</p>
 </td>
 </tr>
 <tr>
@@ -2610,5 +2636,5 @@ map[string]string
 <hr/>
 <p><em>
 Generated with <code>gen-crd-api-reference-docs</code>
-on git commit <code>43fb5e7</code>.
+on git commit <code>f313873</code>.
 </em></p>

docs/developer-guide.md

@@ -35,7 +35,7 @@ $ cd spark-on-k8s-operator
 To update the auto-generated code, run the following command. (This step is only required if the CRD types have been changed):
 
 ```bash
-$ hack/update-gencode.sh
+$ hack/update-codegen.sh
 ```
 
 To update the auto-generated CRD definitions, run the following command:

manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -1030,6 +1030,8 @@ spec:
                       type: array
                     javaOptions:
                       type: string
+                    kubernetesMaster:
+                      type: string
                     labels:
                       additionalProperties:
                         type: string
@@ -1218,6 +1220,10 @@ spec:
                       type: object
                     serviceAccount:
                       type: string
+                    serviceAnnotations:
+                      additionalProperties:
+                        type: string
+                      type: object
                     sidecars:
                       items:
                         properties:

manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml

@@ -1016,6 +1016,8 @@ spec:
                   type: array
                 javaOptions:
                   type: string
+                kubernetesMaster:
+                  type: string
                 labels:
                   additionalProperties:
                     type: string
@@ -1204,6 +1206,10 @@ spec:
                   type: object
                 serviceAccount:
                   type: string
+                serviceAnnotations:
+                  additionalProperties:
+                    type: string
+                  type: object
                 sidecars:
                   items:
                     properties:

pkg/apis/sparkoperator.k8s.io/v1beta2/types.go

@@ -506,6 +506,14 @@ type DriverSpec struct {
 	// Lifecycle for running preStop or postStart commands
 	// +optional
 	Lifecycle *apiv1.Lifecycle `json:"lifecycle,omitempty"`
+	// KubernetesMaster is the URL of the Kubernetes master used by the driver to manage executor pods and
+	// other Kubernetes resources. Default to https://kubernetes.default.svc.
+	// +optional
+	KubernetesMaster *string `json:"kubernetesMaster,omitempty"`
+	// ServiceAnnotations defines the annoations to be added to the Kubernetes headless service used by
+	// executors to connect to the driver.
+	// +optional
+	ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`
 }
 
 // ExecutorSpec is specification of the executor.
@@ -585,9 +593,9 @@ type MonitoringSpec struct {
 	// +optional
 	// If not specified, the content in spark-docker/conf/metrics.properties will be used.
 	MetricsProperties *string `json:"metricsProperties,omitempty"`
-	// MetricsPropertiesFile is the container local path of file metrics.properties for configuring the Spark metric system.
+	// MetricsPropertiesFile is the container local path of file metrics.properties for configuring
+	//the Spark metric system. If not specified, value /etc/metrics/conf/metrics.properties will be used.
 	// +optional
-	// If not specified, value /etc/metrics/conf/metrics.properties will be used.
 	MetricsPropertiesFile *string `json:"metricsPropertiesFile,omitempty"`
 	// Prometheus is for configuring the Prometheus JMX exporter.
 	// +optional

pkg/config/constants.go

@@ -143,6 +143,11 @@ const (
 	SparkExecutorJavaOptions = "spark.executor.extraJavaOptions"
 	// SparkExecutorDeleteOnTermination is the Spark configuration for specifying whether executor pods should be deleted in case of failure or normal termination
 	SparkExecutorDeleteOnTermination = "spark.kubernetes.executor.deleteOnTermination"
+	// SparkDriverKubernetesMaster is the Spark configuration key for specifying the Kubernetes master the driver use
+	// to manage executor pods and other Kubernetes resources.
+	SparkDriverKubernetesMaster = "spark.kubernetes.driver.master"
+	// SparkDriverServiceAnnotationKeyPrefix is the key prefix of annotations to be added to the driver service.
+	SparkDriverServiceAnnotationKeyPrefix = "spark.kubernetes.driver.service.annotation."
 )
 
 const (

pkg/controller/sparkapplication/submission.go

@@ -267,6 +267,16 @@ func addDriverConfOptions(app *v1beta2.SparkApplication, submissionID string) ([
 			fmt.Sprintf("%s=%s", config.SparkDriverServiceAccountName, *app.Spec.Driver.ServiceAccount))
 	}
 
+	if app.Spec.Driver.JavaOptions != nil {
+		driverConfOptions = append(driverConfOptions,
+			fmt.Sprintf("%s=%s", config.SparkDriverJavaOptions, *app.Spec.Driver.JavaOptions))
+	}
+
+	if app.Spec.Driver.KubernetesMaster != nil {
+		driverConfOptions = append(driverConfOptions,
+			fmt.Sprintf("%s=%s", config.SparkDriverKubernetesMaster, *app.Spec.Driver.KubernetesMaster))
+	}
+
 	//Populate SparkApplication Labels to Driver
 	driverLabels := make(map[string]string)
 	for key, value := range app.Labels {
@@ -291,9 +301,9 @@ func addDriverConfOptions(app *v1beta2.SparkApplication, submissionID string) ([
 			fmt.Sprintf("%s%s=%s:%s", config.SparkDriverSecretKeyRefKeyPrefix, key, value.Name, value.Key))
 	}
 
-	if app.Spec.Driver.JavaOptions != nil {
+	for key, value := range app.Spec.Driver.ServiceAnnotations {
 		driverConfOptions = append(driverConfOptions,
-			fmt.Sprintf("%s=%s", config.SparkDriverJavaOptions, *app.Spec.Driver.JavaOptions))
+			fmt.Sprintf("%s%s=%s", config.SparkDriverServiceAnnotationKeyPrefix, key, value))
 	}
 
 	driverConfOptions = append(driverConfOptions, config.GetDriverSecretConfOptions(app)...)