feat: optionally ignore failed snapshots

ryaneorth · Apr 6, 2024 · 2436766 · 2436766
1 parent e3a98be
commit 2436766
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 6 deletions.
diff --git a/helm/charts/scheduled-volume-snapshotter/README.md b/helm/charts/scheduled-volume-snapshotter/README.md
@@ -19,6 +19,7 @@ helm upgrade --install scheduled-volume-snapshotter scheduled-volume-snapshotter
 | `failedJobsHistoryLimit`      | The number of failed jobs to retain                                                                    | `1`                                      |
 | `rbac.enabled`                | Flag indicating whether the rbac resources should be installed                                         | `true`                                   |
 | `logLevel`                    | The Python log level for the jobs                                                                      | `INFO`                                   |
+| `ignoreUnsuccessfulSnapshots` | Flag indicating if previously generated snapshots in a non-successful state should be considered when determining if a new snapshot needs to be generated                                                                      | `false`                                   |
 | `podLabels`                   | Additional labels to add to the CronJob pods                                                           | `{}`                                     |
 | `podAnnotations`              | Annotations to be added to pods                                                                        | `{}`                                     |
 | `snapshotClasses`             | Optional list of VolumeSnapshotClass resources                                                         | `[]`                                     |

diff --git a/helm/charts/scheduled-volume-snapshotter/templates/cronjob.yaml b/helm/charts/scheduled-volume-snapshotter/templates/cronjob.yaml
@@ -50,4 +50,6 @@ spec:
               env:
                 - name: LOG_LEVEL
                   value: "{{ .Values.logLevel }}"
+                - name: IGNORE_UNSUCCESSFUL_SNAPSHOTS
+                  value: "{{ .Values.ignoreUnsuccessfulSnapshots }}"
           restartPolicy: Never
diff --git a/helm/charts/scheduled-volume-snapshotter/values.yaml b/helm/charts/scheduled-volume-snapshotter/values.yaml
@@ -22,6 +22,8 @@ rbac:
 
 logLevel: INFO
 
+ignoreUnsuccessfulSnapshots: false
+
 resources:
   limits:
     cpu: 100m

diff --git a/snapshotter.py b/snapshotter.py
@@ -13,6 +13,7 @@
 # strip off any trailing non-numeric characters
 K8S_MAJOR_VERSION = int(re.split('[^0-9]', K8S_VERSION_INFO.major)[0])
 K8S_MINOR_VERSION = int(re.split('[^0-9]', K8S_VERSION_INFO.minor)[0])
+IGNORE_UNSUCCESSFUL_SNAPSHOTS = os.getenv('IGNORE_UNSUCCESSFUL_SNAPSHOTS', 'true').lower() == 'true'
 
 SVS_CRD_GROUP = 'k8s.ryanorth.io'
 SVS_CRD_VERSION = 'v1beta1'
@@ -57,12 +58,14 @@ def get_associated_snapshots(scheduled_snapshot, volume_snapshots):
 
 def new_snapshot_needed(scheduled_snapshot, existing_snapshots):
     try:
-        successful_snapshots = list(filter(
-            lambda s: (
-                s.get('status', {}).get('readyToUse', False)
-                or not s.get('status', {}).get('error', {})
-            ), existing_snapshots
-        ))
+        successful_snapshots = existing_snapshots
+        if not IGNORE_UNSUCCESSFUL_SNAPSHOTS:
+            successful_snapshots = list(filter(
+                lambda s: (
+                    s.get('status', {}).get('readyToUse', False)
+                    or not s.get('status', {}).get('error', {})
+                ), existing_snapshots
+            ))
         raw_snapshot_frequency = scheduled_snapshot.get('spec', {}).get('snapshotFrequency')
         if isinstance(raw_snapshot_frequency, int):
             raw_snapshot_frequency = f'{str(raw_snapshot_frequency)}h'