Merge pull request replicatedhq#2342 from replicatedhq/emosbaugh/sc-3…

…9107/testgrid-post-install-and-upgrade-scripts

Testgrid post install and upgrade scripts

testgrid/migrations/tables/testinstance.yaml

@@ -43,6 +43,10 @@ spec:
           type: text
         - name: supportbundle_yaml
           type: text
+        - name: post_install_script
+          type: text
+        - name: post_upgrade_script
+          type: text
         - name: os_name
           type: text
         - name: os_version

testgrid/tgapi/pkg/handlers/dequeue_instance.go

@@ -21,6 +21,8 @@ type DequeueInstanceResponse struct {
 	KurlURL           string `json:"kurlUrl"`
 	UpgradeURL        string `json:"upgradeUrl"`
 	SupportbundleYAML string `json:"supportbundleYaml"`
+	PostInstallScript string `json:"postInstallScript"`
+	PostUpgradeScript string `json:"postUpgradeScript"`
 	KurlRef           string `json:"kurlRef"`
 }
 
@@ -57,6 +59,8 @@ func DequeueInstance(w http.ResponseWriter, r *http.Request) {
 		KurlURL:           testInstance.KurlURL,
 		UpgradeURL:        testInstance.UpgradeURL,
 		SupportbundleYAML: testInstance.SupportbundleYAML,
+		PostInstallScript: testInstance.PostInstallScript,
+		PostUpgradeScript: testInstance.PostUpgradeScript,
 		KurlRef:           testInstance.RefID,
 	}
 

testgrid/tgapi/pkg/handlers/ref_start.go

@@ -31,6 +31,9 @@ type PlannedInstance struct {
 
 	SupportbundleYAML string
 
+	PostInstallScript string
+	PostUpgradeScript string
+
 	OperatingSystemName    string
 	OperatingSystemVersion string
 	OperatingSystemImage   string
@@ -87,6 +90,8 @@ func StartRef(w http.ResponseWriter, r *http.Request) {
 			plannedInstance.UpgradeYAML,
 			plannedInstance.UpgradeURL,
 			plannedInstance.SupportbundleYAML,
+			plannedInstance.PostInstallScript,
+			plannedInstance.PostUpgradeScript,
 			plannedInstance.OperatingSystemName,
 			plannedInstance.OperatingSystemVersion,
 			plannedInstance.OperatingSystemImage,

testgrid/tgapi/pkg/testinstance/testinstance.go

@@ -22,12 +22,12 @@ type KurlInstallerMetadata struct {
 	Name string `json:"name" yaml:"name"`
 }
 
-func Create(id, refID, kurlYAML, kurlURL, upgradeYAML, upgradeURL, supportbundleYAML, osName, osVersion, osImage, osPreInit string) error {
+func Create(id, refID, kurlYAML, kurlURL, upgradeYAML, upgradeURL, supportbundleYAML, postInstallScript, postUpgradeScript, osName, osVersion, osImage, osPreInit string) error {
 	pg := persistence.MustGetPGSession()
 
-	query := `insert into testinstance (id, enqueued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, os_name, os_version, os_image, os_preinit)
-values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`
-	if _, err := pg.Exec(query, id, time.Now(), refID, kurlYAML, kurlURL, upgradeYAML, upgradeURL, supportbundleYAML, osName, osVersion, osImage, osPreInit); err != nil {
+	query := `insert into testinstance (id, enqueued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, post_install_script, post_upgrade_script, os_name, os_version, os_image, os_preinit)
+values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
+	if _, err := pg.Exec(query, id, time.Now(), refID, kurlYAML, kurlURL, upgradeYAML, upgradeURL, supportbundleYAML, postInstallScript, postUpgradeScript, osName, osVersion, osImage, osPreInit); err != nil {
 		return errors.Wrap(err, "failed to insert")
 	}
 
@@ -43,20 +43,22 @@ set dequeued_at = now() where id in (
 select id from testinstance
 where dequeued_at is null
 order by enqueued_at asc
-limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, os_name, os_version, os_image, os_preinit
-) select id, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, os_name, os_version, os_image, os_preinit from updated`
+limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, post_install_script, post_upgrade_script, os_name, os_version, os_image, os_preinit
+) select id, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, post_install_script, post_upgrade_script, os_name, os_version, os_image, os_preinit from updated`
 
 	row := db.QueryRow(query)
 
 	testInstance := types.TestInstance{}
-	var upgradeYAML, upgradeURL, supportbundleYAML, osPreInit sql.NullString
+	var upgradeYAML, upgradeURL, supportbundleYAML, postInstallScript, postUpgradeScript, osPreInit sql.NullString
 	if err := row.Scan(&testInstance.ID,
 		&testInstance.RefID,
 		&testInstance.KurlYAML,
 		&testInstance.KurlURL,
 		&upgradeYAML,
 		&upgradeURL,
 		&supportbundleYAML,
+		&postInstallScript,
+		&postUpgradeScript,
 		&testInstance.OSName,
 		&testInstance.OSVersion,
 		&testInstance.OSImage,
@@ -68,6 +70,8 @@ limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_ya
 	testInstance.UpgradeYAML = upgradeYAML.String
 	testInstance.UpgradeURL = upgradeURL.String
 	testInstance.SupportbundleYAML = supportbundleYAML.String
+	testInstance.PostInstallScript = postInstallScript.String
+	testInstance.PostUpgradeScript = postUpgradeScript.String
 	testInstance.OSPreInit = osPreInit.String
 
 	return &testInstance, nil
@@ -85,20 +89,22 @@ AND dequeued_at <  now() - INTERVAL '12 hours'
 AND dequeued_at >  now() - INTERVAL '24 hours'
 AND enqueued_at >  now() - INTERVAL '24 hours'
 order by enqueued_at asc
-limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, os_name, os_version, os_image, os_preinit
-) select id, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, os_name, os_version, os_image, os_preinit from updated`
+limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, post_install_script, post_upgrade_script, os_name, os_version, os_image, os_preinit
+) select id, testrun_ref, kurl_yaml, kurl_url, upgrade_yaml, upgrade_url, supportbundle_yaml, post_install_script, post_upgrade_script, os_name, os_version, os_image, os_preinit from updated`
 
 	row := db.QueryRow(query)
 
 	testInstance := types.TestInstance{}
-	var upgradeYAML, upgradeURL, supportbundleYAML, osPreInit sql.NullString
+	var upgradeYAML, upgradeURL, supportbundleYAML, postInstallScript, postUpgradeScript, osPreInit sql.NullString
 	if err := row.Scan(&testInstance.ID,
 		&testInstance.RefID,
 		&testInstance.KurlYAML,
 		&testInstance.KurlURL,
 		&upgradeYAML,
 		&upgradeURL,
 		&supportbundleYAML,
+		&postInstallScript,
+		&postUpgradeScript,
 		&testInstance.OSName,
 		&testInstance.OSVersion,
 		&testInstance.OSImage,
@@ -110,6 +116,8 @@ limit 1) returning id, dequeued_at, testrun_ref, kurl_yaml, kurl_url, upgrade_ya
 	testInstance.UpgradeYAML = upgradeYAML.String
 	testInstance.UpgradeURL = upgradeURL.String
 	testInstance.SupportbundleYAML = supportbundleYAML.String
+	testInstance.PostInstallScript = postInstallScript.String
+	testInstance.PostUpgradeScript = postUpgradeScript.String
 	testInstance.OSPreInit = osPreInit.String
 
 	return &testInstance, nil
@@ -225,7 +233,7 @@ where id = $1`
 func List(refID string, limit int, offset int, addons map[string]string) ([]types.TestInstance, error) {
 	db := persistence.MustGetPGSession()
 
-	query := `SELECT ti.id, ti.kurl_yaml, ti.kurl_url, ti.upgrade_yaml, ti.upgrade_url, ti.supportbundle_yaml, ti.os_name, ti.os_version, ti.os_image, ti.enqueued_at, ti.dequeued_at, ti.started_at, ti.finished_at, ti.is_success, ti.failure_reason, ti.is_unsupported
+	query := `SELECT ti.id, ti.kurl_yaml, ti.kurl_url, ti.upgrade_yaml, ti.upgrade_url, ti.supportbundle_yaml, ti.post_install_script, ti.post_upgrade_script, ti.os_name, ti.os_version, ti.os_image, ti.enqueued_at, ti.dequeued_at, ti.started_at, ti.finished_at, ti.is_success, ti.failure_reason, ti.is_unsupported
 FROM testinstance ti
 WHERE ti.testrun_ref = $1`
 
@@ -262,7 +270,7 @@ WHERE ti.testrun_ref = $1`
 		var startedAt sql.NullTime
 		var finishedAt sql.NullTime
 		var isSuccess, isUnsupported sql.NullBool
-		var upgradeYAML, upgradeURL, supportbundleYAML, failureReason sql.NullString
+		var upgradeYAML, upgradeURL, supportbundleYAML, postInstallScript, postUpgradeScript, failureReason sql.NullString
 
 		if err := rows.Scan(
 			&testInstance.ID,
@@ -271,6 +279,8 @@ WHERE ti.testrun_ref = $1`
 			&upgradeYAML,
 			&upgradeURL,
 			&supportbundleYAML,
+			&postInstallScript,
+			&postUpgradeScript,
 			&testInstance.OSName,
 			&testInstance.OSVersion,
 			&testInstance.OSImage,
@@ -297,24 +307,15 @@ WHERE ti.testrun_ref = $1`
 		if finishedAt.Valid {
 			testInstance.FinishedAt = &finishedAt.Time
 		}
-		if isSuccess.Valid {
-			testInstance.IsSuccess = isSuccess.Bool
-		}
-		if failureReason.Valid {
-			testInstance.FailureReason = failureReason.String
-		}
-		if isUnsupported.Valid {
-			testInstance.IsUnsupported = isUnsupported.Bool
-		}
-		if upgradeYAML.Valid {
-			testInstance.UpgradeYAML = upgradeYAML.String
-		}
-		if upgradeURL.Valid {
-			testInstance.UpgradeURL = upgradeURL.String
-		}
-		if supportbundleYAML.Valid {
-			testInstance.SupportbundleYAML = supportbundleYAML.String
-		}
+
+		testInstance.IsSuccess = isSuccess.Bool
+		testInstance.FailureReason = failureReason.String
+		testInstance.IsUnsupported = isUnsupported.Bool
+		testInstance.UpgradeYAML = upgradeYAML.String
+		testInstance.UpgradeURL = upgradeURL.String
+		testInstance.SupportbundleYAML = supportbundleYAML.String
+		testInstance.PostInstallScript = postInstallScript.String
+		testInstance.PostUpgradeScript = postUpgradeScript.String
 
 		testInstances = append(testInstances, testInstance)
 	}

testgrid/tgapi/pkg/testinstance/types/types.go

@@ -20,6 +20,8 @@ type TestInstance struct {
 	UpgradeURL  string `json:"upgradeUrl"`
 
 	SupportbundleYAML string `json:"supportbundleYaml"`
+	PostInstallScript string `json:"postInstallScript"`
+	PostUpgradeScript string `json:"postUpgradeScript"`
 
 	Output string `json:"-"`
 

testgrid/tgrun/pkg/runner/embed/runcmd.sh

@@ -140,6 +140,40 @@ function run_upgrade() {
     collect_debug_info_after_kurl || true
 }
 
+function run_post_install_script() {
+    if [ ! -f /opt/kurl-testgrid/postinstall.sh ] ; then
+        return # file does not exist
+    fi
+
+    bash -ex /opt/kurl-testgrid/postinstall.sh
+    local exit_status="$?"
+
+    send_logs
+
+    if [ "$exit_status" -ne 0 ]; then
+        report_failure "post_install_script"
+        collect_support_bundle
+        exit 1
+    fi
+}
+
+function run_post_upgrade_script() {
+    if [ ! -f /opt/kurl-testgrid/postupgrade.sh ] ; then
+        return # file does not exist
+    fi
+
+    bash -ex /opt/kurl-testgrid/postupgrade.sh
+    local exit_status="$?"
+
+    send_logs
+
+    if [ "$exit_status" -ne 0 ]; then
+        report_failure "post_upgrade_script"
+        collect_support_bundle
+        exit 1
+    fi
+}
+
 function collect_debug_info_after_kurl() {
     if [ "$KURL_EXIT_STATUS" -ne 0 ]; then
         echo "kubelet status"
@@ -375,6 +409,8 @@ function main() {
         exit 1
     fi
 
+    run_post_install_script
+
     run_tasks_join_token
 
     if [ "$KURL_UPGRADE_URL" != "" ]; then
@@ -389,6 +425,10 @@ function main() {
         exit 1
     fi
 
+    if [ "$KURL_UPGRADE_URL" != "" ]; then
+        run_post_upgrade_script
+    fi
+
     failureReason=""
     if ! check_airgap ; then
         failureReason="airgap_violation"

testgrid/tgrun/pkg/runner/loop.go

@@ -97,6 +97,8 @@ func MainRunLoop(runnerOptions types.RunnerOptions) error {
 				KurlURL:           dequeuedInstance.KurlURL,
 				UpgradeURL:        dequeuedInstance.UpgradeURL,
 				SupportbundleYAML: dequeuedInstance.SupportbundleYAML,
+				PostInstallScript: dequeuedInstance.PostInstallScript,
+				PostUpgradeScript: dequeuedInstance.PostUpgradeScript,
 				KurlRef:           dequeuedInstance.KurlRef,
 
 				TestGridAPIEndpoint: runnerOptions.APIEndpoint,

testgrid/tgrun/pkg/runner/runner.go

@@ -260,10 +260,18 @@ export KURL_URL='%s'
 export KURL_UPGRADE_URL='%s'
 export SUPPORTBUNDLE_SPEC='%s'
 `,
-		singleTest.TestGridAPIEndpoint, singleTest.ID, singleTest.KurlURL, singleTest.UpgradeURL, singleTest.SupportbundleYAML)
+		singleTest.TestGridAPIEndpoint,
+		singleTest.ID,
+		singleTest.KurlURL,
+		singleTest.UpgradeURL,
+		singleTest.SupportbundleYAML,
+	)
 
 	varsB64 := base64.StdEncoding.EncodeToString([]byte(varsSh))
 
+	postInstallB64 := base64.StdEncoding.EncodeToString([]byte(singleTest.PostInstallScript))
+	postUpgradeB64 := base64.StdEncoding.EncodeToString([]byte(singleTest.PostUpgradeScript))
+
 	script := fmt.Sprintf(`#cloud-config
 
 password: kurl
@@ -276,6 +284,8 @@ runcmd:
   - [ bash, -c, 'echo %s | base64 -d > /opt/kurl-testgrid/preinit.sh' ]
   - [ bash, -c, 'echo %s | base64 -d > /opt/kurl-testgrid/vars.sh' ]
   - [ bash, -c, 'echo %s | base64 -d > /opt/kurl-testgrid/runcmd.sh' ]
+  - [ bash, -c, '[ %d -eq 0 ] || echo %s | base64 -d > /opt/kurl-testgrid/postinstall.sh' ]
+  - [ bash, -c, '[ %d -eq 0 ] || echo %s | base64 -d > /opt/kurl-testgrid/postupgrade.sh' ]
   - [ bash, -c, 'sudo bash /opt/kurl-testgrid/preinit.sh' ]
   - [ bash, -c, 'sudo bash -c ". /opt/kurl-testgrid/vars.sh && bash /opt/kurl-testgrid/runcmd.sh"' ]
   - [ bash, -c, 'sleep 10 && sudo poweroff' ]
@@ -287,6 +297,10 @@ power_state:
 		base64.StdEncoding.EncodeToString([]byte(singleTest.OperatingSystemPreInit)),
 		varsB64,
 		runcmdB64,
+		len(singleTest.PostInstallScript),
+		postInstallB64,
+		len(singleTest.PostUpgradeScript),
+		postUpgradeB64,
 	)
 
 	file := filepath.Join(tempDir, "startup-script.sh")

testgrid/tgrun/pkg/runner/types/types.go

@@ -19,6 +19,8 @@ type SingleRun struct {
 	KurlURL           string
 	UpgradeURL        string
 	SupportbundleYAML string
+	PostInstallScript string
+	PostUpgradeScript string
 	KurlRef           string
 
 	TestGridAPIEndpoint string

testgrid/tgrun/pkg/scheduler/scheduler.go

@@ -181,6 +181,9 @@ func Run(schedulerOptions types.SchedulerOptions) error {
 
 				SupportbundleYAML: string(supportbundleYAML),
 
+				PostInstallScript: instance.PostInstallScript,
+				PostUpgradeScript: instance.PostUpgradeScript,
+
 				OperatingSystemName:    operatingSystem.Name,
 				OperatingSystemVersion: operatingSystem.Version,
 				OperatingSystemImage:   operatingSystem.VMImageURI,

testgrid/tgrun/pkg/scheduler/types/types.go

@@ -42,6 +42,8 @@ type Instance struct {
 	InstallerSpec     kurlv1beta1.InstallerSpec          `json:"installerSpec" yaml:"installerSpec"`
 	UpgradeSpec       *kurlv1beta1.InstallerSpec         `json:"upgradeSpec,omitempty" yaml:"upgradeSpec,omitempty"`
 	SupportbundleSpec *troubleshootv1beta2.SupportBundle `json:"supportbundleSpec,omitempty" yaml:"supportbundleSpec,omitempty"`
+	PostInstallScript string                             `json:"postInstallScript,omitempty" yaml:"postInstallScript,omitempty"`
+	PostUpgradeScript string                             `json:"postUpgradeScript,omitempty" yaml:"postUpgradeScript,omitempty"`
 	Airgap            bool                               `json:"airgap,omitempty" yaml:"airgap,omitempty"`
 	UnsupportedOSIDs  []string                           `json:"unsupportedOSIDs,omitempty" yaml:"unsupportedOSIDs,omitempty"`
 }