upload-queue: tweak timeouts and give up after 5 failures

Previously we would repeat the same request five times over
a total of 45 seconds, with a max of 10 seconds between the
last two requests.  This might be too little in case a rate limit
is hit, so make this increase exponentially so that the final
attempt is over five minutes after the previous one, but
first few requests happen within a minute.

With that change, we no longer need to sleep 30 seconds after
a failed request.

If we get a server response after five successive tries,
go ahead and move the request into a separate directory
so that we won't get stuck trying the same request over and
over again.

Signed-off-by: Bob Copeland <[email protected]>

http.h

@@ -3,13 +3,16 @@
 
 #include <stddef.h>
 #include <stdarg.h>
+#include <curl/curl.h>
 
 struct http_param_set
 {
 	char **argv;
 	size_t n_alloced;
 };
 
+#define HTTP_ERROR_CODE	CURLE_HTTP_RETURNED_ERROR
+
 void http_post_add_params(struct http_param_set *params, ...);
 char *http_post_lastpass(const char *page, const char *session, size_t *len, ...);
 char *http_post_lastpass_v(const char *page, const char *session, size_t *len, char **argv);

upload-queue.c

@@ -55,15 +55,13 @@
 #include <fcntl.h>
 #include <signal.h>
 
-static void upload_queue_write_entry(const char *entry, unsigned const char key[KDF_HASH_LEN])
+static void make_upload_dir(const char *path)
 {
 	_cleanup_free_ char *base_path = NULL;
-	_cleanup_free_ char *name = NULL;
 	struct stat sbuf;
 	int ret;
-	unsigned long serial;
 
-	base_path = config_path("upload-queue");
+	base_path = config_path(path);
 
 	ret = stat(base_path, &sbuf);
 	if ((ret == -1 && errno == ENOENT) || !S_ISDIR(sbuf.st_mode)) {
@@ -73,6 +71,15 @@ static void upload_queue_write_entry(const char *entry, unsigned const char key[
 	} else if (ret == -1)
 		die_errno("stat(%s)", base_path);
 
+}
+
+static void upload_queue_write_entry(const char *entry, unsigned const char key[KDF_HASH_LEN])
+{
+	_cleanup_free_ char *name = NULL;
+	unsigned long serial;
+
+	make_upload_dir("upload-queue");
+
 	for (serial = 0; serial < ULONG_MAX; ++serial) {
 		free(name);
 		xasprintf(&name, "upload-queue/%lu%lu", time(NULL), serial);
@@ -85,6 +92,28 @@ static void upload_queue_write_entry(const char *entry, unsigned const char key[
 	config_write_encrypted_string(name, entry, key);
 }
 
+static void upload_queue_drop(const char *name)
+{
+	_cleanup_free_ char *newname = NULL;
+	_cleanup_free_ char *old_full = NULL;
+	_cleanup_free_ char *new_full = NULL;
+	char *basename;
+
+	make_upload_dir("upload-fail");
+
+	basename = strrchr(name, '/');
+	if (!basename) {
+		unlink(name);
+		return;
+	}
+	basename += 1;
+	xasprintf(&newname, "upload-fail/%s", basename);
+
+	old_full = config_path(name);
+	new_full = config_path(newname);
+	rename(old_full, new_full);
+}
+
 static char *upload_queue_next_entry(unsigned const char key[KDF_HASH_LEN], char **name, char **lock)
 {
 	unsigned long long smallest = ULLONG_MAX, current;
@@ -169,6 +198,8 @@ static void upload_queue_upload_all(const struct session *session, unsigned cons
 	bool should_fetch_new_blob_after = false;
 	int curl_ret;
 	long http_code;
+	bool http_failed_all;
+	int backoff;
 
 	while ((entry = upload_queue_next_entry(key, &name, &lock))) {
 		size = 0;
@@ -196,20 +227,32 @@ static void upload_queue_upload_all(const struct session *session, unsigned cons
 			}
 		}
 		argv[size] = NULL;
+
+		http_failed_all = true;
+		backoff = 1;
 		for (int i = 0; i < 5; ++i) {
-			sleep(i * 2);
+			if (i) {
+				sleep(backoff);
+				backoff *= 8;
+			}
+
 			result = http_post_lastpass_v_noexit(argv[0],
 				session->sessionid, NULL, &argv[1],
 				&curl_ret, &http_code);
 
+			http_failed_all &= curl_ret == HTTP_ERROR_CODE;
+
 			if (result && strlen(result))
 				should_fetch_new_blob_after = true;
 			free(result);
 			if (result)
 				break;
 		}
 		if (!result) {
-			sleep(30);
+			/* server failed response 5 times, remove it */
+			if (http_failed_all)
+				upload_queue_drop(name);
+
 			config_unlink(lock);
 		} else {
 			config_unlink(name);