Updated the use the v1beta1 version of the APIs

satheshs · Jan 17, 2019 · c41576b · c41576b
1 parent 97d4b14
commit c41576b
Show file tree

Hide file tree

Showing 53 changed files with 741 additions and 664 deletions.
diff --git a/README.md b/README.md
@@ -10,9 +10,11 @@
 
 ## Project Status
 
-**Project status:** *alpha* 
+**Project status:** *beta* 
 
-The Kubernetes Operator for Apache Spark is still under active development. Backward compatibility of the APIs is not guaranteed for alpha releases.
+The Kubernetes Operator for Apache Spark is under active development, but backward compatibility of the APIs is guaranteed for beta releases. 
+
+**If you are currently using the `v1alpha1` version of the APIs in your manifests, please update them to use the `v1beta1` version by changing `apiVersion: "sparkoperator.k8s.io/v1alpha1"` to `apiVersion: "sparkoperator.k8s.io/v1beta1"`. You will also need to delete the `v1alpha1` version of the CustomResourceDefinitions named `sparkapplications.sparkoperator.k8s.io` and `scheduledsparkapplications.sparkoperator.k8s.io`, and replace them with the `v1beta1` version either by installing the latest version of the operator or by running `kubectl create -f manifest/spark-operator-crds.yaml`.**
 
 Customization of Spark pods, e.g., mounting arbitrary volumes and setting pod affinity, is currently experimental and implemented using a Kubernetes
 [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), which became beta in Kubernetes 1.9. 

diff --git a/docs/api.md b/docs/api.md
@@ -4,8 +4,8 @@ The Kubernetes Operator for Apache Spark uses  [CustomResourceDefinitions](https
 named `SparkApplication` and `ScheduledSparkApplication` for specifying one-time Spark applications and Spark applications
 that are supposed to run on a standard [cron](https://en.wikipedia.org/wiki/Cron) schedule. Similarly to other kinds of 
 Kubernetes resources, they consist of a specification in a `Spec` field and a `Status` field. The definitions are organized 
-in the following structure. The v1alpha1 version of the API definition is implemented 
-[here](../pkg/apis/sparkoperator.k8s.io/v1alpha1/types.go).
+in the following structure. The v1beta1 version of the API definition is implemented 
+[here](../pkg/apis/sparkoperator.k8s.io/v1beta1/types.go).
 
 ```
 ScheduledSparkApplication

diff --git a/docs/gcp.md b/docs/gcp.md
@@ -43,7 +43,7 @@ The ones set in `core-site.xml` apply to all applications using the image. Also
 variable `GCS_PROJECT_ID` must be set when using the image at `gcr.io/ynli-k8s/spark:v2.3.0-gcs`.
 
 ```yaml
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: SparkApplication
 metadata:
   name: foo-gcs-bg

diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md
@@ -135,7 +135,7 @@ $ kubectl get sparkapplications spark-pi -o=yaml
 This will show something similar to the following:
 
 ```yaml
-apiVersion: sparkoperator.k8s.io/v1alpha1
+apiVersion: sparkoperator.k8s.io/v1beta1
 kind: SparkApplication
 metadata:
   ...

diff --git a/docs/user-guide.md b/docs/user-guide.md
@@ -45,7 +45,7 @@ It also has fields for specifying the unified container image (to use for both t
 Below is an example showing part of a `SparkApplication` specification:
 
 ```yaml
-apiVersion: sparkoperator.k8s.io/v1alpha1
+apiVersion: sparkoperator.k8s.io/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-pi
@@ -387,7 +387,7 @@ client so effectively the driver gets restarted.
 The operator supports running a Spark application on a standard [cron](https://en.wikipedia.org/wiki/Cron) schedule using objects of the `ScheduledSparkApplication` custom resource type. A `ScheduledSparkApplication` object specifies a cron schedule on which the application should run and a `SparkApplication` template from which a `SparkApplication` object for each run of the application is created. The following is an example `ScheduledSparkApplication`:
 
 ```yaml
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: ScheduledSparkApplication
 metadata:
   name: spark-pi-scheduled

diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: SparkApplication
 metadata:
   name: spark-pi

diff --git a/examples/spark-pi-schedule.yaml b/examples/spark-pi-schedule.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: ScheduledSparkApplication
 metadata:
   name: spark-pi-scheduled

diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: SparkApplication
 metadata:
   name: spark-pi

diff --git a/examples/spark-py-pi.yaml b/examples/spark-py-pi.yaml
@@ -16,7 +16,7 @@
 # Support for Python is experimental, and requires building SNAPSHOT image of Apache Spark,
 # with `imagePullPolicy` set to Always
 
-apiVersion: "sparkoperator.k8s.io/v1alpha1"
+apiVersion: "sparkoperator.k8s.io/v1beta1"
 kind: SparkApplication
 metadata:
   name: pyspark-pi

diff --git a/manifest/spark-operator-crds.yaml b/manifest/spark-operator-crds.yaml
@@ -100,7 +100,7 @@ spec:
               - Scala
               - Python
               - R
-  version: v1alpha1
+  version: v1beta1
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
@@ -202,4 +202,4 @@ spec:
                   - Scala
                   - Python
                   - R
-  version: v1alpha1
+  version: v1beta1
diff --git a/manifest/spark-operator-with-metrics.yaml b/manifest/spark-operator-with-metrics.yaml
@@ -21,13 +21,13 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1beta1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.4.0-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1beta1
   strategy:
     type: Recreate
   template:
@@ -38,14 +38,14 @@ spec:
         prometheus.io/path: "/metrics"
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.4.0-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1beta1
       initializers:
         pending: []
     spec:
       serviceAccountName: sparkoperator
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-latest
         imagePullPolicy: Always
         ports:
           - containerPort: 10254

diff --git a/manifest/spark-operator-with-webhook.yaml b/manifest/spark-operator-with-webhook.yaml
@@ -21,20 +21,20 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1beta1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.4.0-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1beta1
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.4.0-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1beta1
       initializers:
         pending: []
     spec:
@@ -45,7 +45,7 @@ spec:
           secretName: spark-webhook-certs
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-latest
         imagePullPolicy: Always
         volumeMounts:
         - name: webhook-certs
@@ -63,20 +63,20 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1beta1
 spec:
   backoffLimit: 3
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.4.0-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1beta1
     spec:
       serviceAccountName: sparkoperator
       restartPolicy: Never
       containers:
       - name: main
-        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-latest
         imagePullPolicy: IfNotPresent
         command: ["/usr/bin/gencerts.sh", "-p"]
 ---
@@ -92,4 +92,4 @@ spec:
       name: webhook
   selector:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1beta1
diff --git a/manifest/spark-operator.yaml b/manifest/spark-operator.yaml
@@ -21,27 +21,27 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1beta1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.4.0-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1beta1
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.4.0-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1beta1
       initializers:
         pending: []
     spec:
       serviceAccountName: sparkoperator
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-latest
         imagePullPolicy: Always
         args:
         - -logtostderr
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta1/defaults.go b/pkg/apis/sparkoperator.k8s.io/v1beta1/defaults.go
@@ -0,0 +1,74 @@
+/*
+Copyright 2017 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+// SetSparkApplicationDefaults sets default values for certain fields of a SparkApplication.
+func SetSparkApplicationDefaults(app *SparkApplication) {
+	if app == nil {
+		return
+	}
+
+	if app.Spec.Mode == "" {
+		app.Spec.Mode = ClusterMode
+	}
+
+	if app.Spec.RestartPolicy.Type == "" {
+		app.Spec.RestartPolicy.Type = Never
+	}
+
+	if app.Spec.RestartPolicy.Type != Never {
+		// Default to 5 sec if the RestartPolicy is OnFailure or Always and these values aren't specified.
+		if app.Spec.RestartPolicy.OnFailureRetryInterval == nil {
+			app.Spec.RestartPolicy.OnFailureRetryInterval = new(int64)
+			*app.Spec.RestartPolicy.OnFailureRetryInterval = 5
+		}
+
+		if app.Spec.RestartPolicy.OnSubmissionFailureRetryInterval == nil {
+			app.Spec.RestartPolicy.OnSubmissionFailureRetryInterval = new(int64)
+			*app.Spec.RestartPolicy.OnSubmissionFailureRetryInterval = 5
+		}
+	}
+
+	setDriverSpecDefaults(app.Spec.Driver)
+	setExecutorSpecDefaults(app.Spec.Executor)
+}
+
+func setDriverSpecDefaults(spec DriverSpec) {
+	if spec.Cores == nil {
+		spec.Cores = new(float32)
+		*spec.Cores = 1
+	}
+	if spec.Memory == nil {
+		spec.Memory = new(string)
+		*spec.Memory = "1g"
+	}
+}
+
+func setExecutorSpecDefaults(spec ExecutorSpec) {
+	if spec.Cores == nil {
+		spec.Cores = new(float32)
+		*spec.Cores = 1
+	}
+	if spec.Memory == nil {
+		spec.Memory = new(string)
+		*spec.Memory = "1g"
+	}
+	if spec.Instances == nil {
+		spec.Instances = new(int32)
+		*spec.Instances = 1
+	}
+}
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta1/types.go b/pkg/apis/sparkoperator.k8s.io/v1beta1/types.go
@@ -256,6 +256,7 @@ const (
 	InvalidatingState     ApplicationStateType = "INVALIDATING"
 	SucceedingState       ApplicationStateType = "SUCCEEDING"
 	FailingState          ApplicationStateType = "FAILING"
+	UnknownState          ApplicationStateType = "UNKNOWN"
 )
 
 // ApplicationState tells the current state of the application and an error message in case of failures.

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -19,7 +19,7 @@ package config
 import (
 	"fmt"
 
-	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
+	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1beta1"
 )
 
 // GetDriverAnnotationOption returns a spark-submit option for a driver annotation of the given key and value.
@@ -33,7 +33,7 @@ func GetExecutorAnnotationOption(key string, value string) string {
 }
 
 // GetDriverEnvVarConfOptions returns a list of spark-submit options for setting driver environment variables.
-func GetDriverEnvVarConfOptions(app *v1alpha1.SparkApplication) []string {
+func GetDriverEnvVarConfOptions(app *v1beta1.SparkApplication) []string {
 	var envVarConfOptions []string
 	for key, value := range app.Spec.Driver.EnvVars {
 		envVar := fmt.Sprintf("%s%s=%s", SparkDriverEnvVarConfigKeyPrefix, key, value)
@@ -43,7 +43,7 @@ func GetDriverEnvVarConfOptions(app *v1alpha1.SparkApplication) []string {
 }
 
 // GetExecutorEnvVarConfOptions returns a list of spark-submit options for setting executor environment variables.
-func GetExecutorEnvVarConfOptions(app *v1alpha1.SparkApplication) []string {
+func GetExecutorEnvVarConfOptions(app *v1beta1.SparkApplication) []string {
 	var envVarConfOptions []string
 	for key, value := range app.Spec.Executor.EnvVars {
 		envVar := fmt.Sprintf("%s%s=%s", SparkExecutorEnvVarConfigKeyPrefix, key, value)

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -22,14 +22,14 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
+	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1beta1"
 )
 
 func TestGetDriverEnvVarConfOptions(t *testing.T) {
-	app := &v1alpha1.SparkApplication{
-		Spec: v1alpha1.SparkApplicationSpec{
-			Driver: v1alpha1.DriverSpec{
-				SparkPodSpec: v1alpha1.SparkPodSpec{
+	app := &v1beta1.SparkApplication{
+		Spec: v1beta1.SparkApplicationSpec{
+			Driver: v1beta1.DriverSpec{
+				SparkPodSpec: v1beta1.SparkPodSpec{
 					EnvVars: map[string]string{
 						"ENV1": "VALUE1",
 						"ENV2": "VALUE2",
@@ -50,10 +50,10 @@ func TestGetDriverEnvVarConfOptions(t *testing.T) {
 }
 
 func TestGetExecutorEnvVarConfOptions(t *testing.T) {
-	app := &v1alpha1.SparkApplication{
-		Spec: v1alpha1.SparkApplicationSpec{
-			Executor: v1alpha1.ExecutorSpec{
-				SparkPodSpec: v1alpha1.SparkPodSpec{
+	app := &v1beta1.SparkApplication{
+		Spec: v1beta1.SparkApplicationSpec{
+			Executor: v1beta1.ExecutorSpec{
+				SparkPodSpec: v1beta1.SparkPodSpec{
 					EnvVars: map[string]string{
 						"ENV1": "VALUE1",
 						"ENV2": "VALUE2",