enable automatic update of indexes controlled by --cache-max-age

This modifies apk cache for indexes to be automatically refreshed
periodically without explicit 'update' or '--update-cache' usage.

The default is to do if-modified-since request if the local copy
is older than 4 hours. This age can be changed with --cache-max-age.
Using --update-cache will change this age to 60 seconds to make
sure the cached copy is relatively new. The small age is in order
to try to avoid downloading indexes second time when apk-tools is
upgraded and apk re-execs after self-upgrade.

Accordingly using explicitly 'apk update' will now enforce
--force-refresh and request the very latest index by requesting
any potential http proxy to do refresh too.

9 files changed, 53 insertions, 36 deletions

diff --git a/src/apk.c b/src/apk.c
index 22acf41..ecd89a2 100644
--- a/src/apk.c
+++ b/src/apk.c
@@ -120,9 +120,6 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
 	case 'i':
 		apk_flags |= APK_INTERACTIVE;
 		break;
-	case 'U':
-		apk_flags |= APK_UPDATE_CACHE;
-		break;
 	case 0x101:
 		apk_flags |= APK_PROGRESS;
 		break;
@@ -153,6 +150,14 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
 	case 0x116:
 		dbopts->cache_dir = optarg;
 		break;
+	case 'U':
+		/* Make it one minute, to avoid updating indexes twice
+		 * when doing self-upgrade's re-exec */
+		dbopts->cache_max_age = 60;
+		break;
+	case 0x119:
+		dbopts->cache_max_age = atoi(optarg) * 60;
+		break;
 	case 0x112:
 		dbopts->arch = optarg;
 		break;
@@ -193,7 +198,7 @@ static const struct apk_option options_global[] = {
 	{ 0x121, "force-old-apk", "Continue even if packages use unsupported features" },
 	{ 0x120, "force-overwrite", "Overwrite files in other packages" },
 	{ 0x123, "force-refresh", "Do not use cached files (local or from proxy)" },
-	{ 'U', "update-cache",	"Update the repository cache" },
+	{ 'U', "update-cache",	"Alias for --cache-max-age 60" },
 	{ 0x101, "progress",	"Show a progress bar" },
 	{ 0x10f, "progress-fd",	"Write progress to fd", required_argument, "FD" },
 	{ 0x110, "no-progress",	"Disable progress bar even for TTYs" },
@@ -211,6 +216,8 @@ static const struct apk_option options_global[] = {
 	{ 0x115, "no-cache",	"Do not use any local cache path" },
 	{ 0x116, "cache-dir",	"Override cache directory",
 				required_argument, "CACHEDIR" },
+	{ 0x119, "cache-max-age", "Maximum AGE (in minutes) for index in cache before refresh",
+				required_argument, "AGE" },
 	{ 0x112, "arch",	"Use architecture with --root",
 				required_argument, "ARCH" },
 	{ 0x114, "print-arch",	"Print default arch and exit" },
@@ -528,6 +535,7 @@ int main(int argc, char **argv)
 			ctx = calloc(1, applet->context_size);
 		dbopts.open_flags = applet->open_flags;
 		apk_flags |= applet->forced_flags;
+		apk_force |= applet->forced_force;
 	}
 	for (opt = all_options, sopt = short_options; opt->name != NULL; opt++) {
 		if (opt->flag == NULL &&
diff --git a/src/apk_applet.h b/src/apk_applet.h
index e459435..0eea731 100644
--- a/src/apk_applet.h
+++ b/src/apk_applet.h
@@ -42,7 +42,7 @@ struct apk_applet {
 	const char *help;
 	const struct apk_option_group *optgroups[4];
 
-	unsigned int open_flags, forced_flags;
+	unsigned int open_flags, forced_flags, forced_force;
 	int context_size;
 
 	int (*main)(void *ctx, struct apk_database *db, struct apk_string_array *args);
diff --git a/src/apk_database.h b/src/apk_database.h
index 20454d9..daf4ddc 100644
--- a/src/apk_database.h
+++ b/src/apk_database.h
@@ -122,6 +122,7 @@ struct apk_repository_list {
 
 struct apk_db_options {
 	int lock_wait;
+	unsigned int cache_max_age;
 	unsigned long open_flags;
 	const char *root;
 	const char *arch;
@@ -150,8 +151,8 @@ struct apk_database {
 	char *cache_remount_dir, *root_proc_dir;
 	unsigned long cache_remount_flags;
 	apk_blob_t *arch;
-	unsigned int local_repos, available_repos;
-	int repo_update_errors;
+	unsigned int local_repos, available_repos, cache_max_age;
+	unsigned int repo_update_errors, repo_update_counter;
 	unsigned int pending_triggers;
 	int performing_self_upgrade : 1;
 	int permanent : 1;
@@ -246,7 +247,7 @@ unsigned int apk_db_get_pinning_mask_repos(struct apk_database *db, unsigned sho
 
 int apk_db_cache_active(struct apk_database *db);
 int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
-		       struct apk_package *pkg, int verify,
+		       struct apk_package *pkg, int verify, int autoupdate,
 		       apk_progress_cb cb, void *cb_ctx);
 
 typedef void (*apk_cache_item_cb)(struct apk_database *db,
diff --git a/src/apk_defines.h b/src/apk_defines.h
index 7e79135..c7ab7e6 100644
--- a/src/apk_defines.h
+++ b/src/apk_defines.h
@@ -68,7 +68,6 @@ extern char **apk_argv;
 #define APK_CLEAN_PROTECTED	0x0004
 #define APK_PROGRESS		0x0008
 #define APK_RECURSIVE		0x0020
-#define APK_UPDATE_CACHE	0x0080
 #define APK_ALLOW_UNTRUSTED	0x0100
 #define APK_PURGE		0x0200
 #define APK_INTERACTIVE		0x0400
diff --git a/src/apk_io.h b/src/apk_io.h
index 48bae5f..94aa989 100644
--- a/src/apk_io.h
+++ b/src/apk_io.h
@@ -147,7 +147,7 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream);
 struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int));
 struct apk_bstream *apk_bstream_from_file(int atfd, const char *file);
 struct apk_bstream *apk_bstream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
-struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to,
+struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta,
 				    apk_progress_cb cb, void *cb_ctx);
 
 static inline struct apk_bstream *apk_bstream_from_fd(int fd)
diff --git a/src/cache.c b/src/cache.c
index f694085..aa43bdd 100644
--- a/src/cache.c
+++ b/src/cache.c
@@ -66,9 +66,9 @@ static int cache_download(struct apk_database *db)
 		if (repo == NULL)
 			continue;
 
-		r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY,
+		r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY, 0,
 				       progress_cb, &prog);
-		if (r) {
+		if (r && r != -EALREADY) {
 			apk_error(PKG_VER_FMT ": %s", PKG_VER_PRINTF(pkg), apk_error_str(r));
 			ret++;
 		}
diff --git a/src/database.c b/src/database.c
index 4ea4fb4..df06ade 100644
--- a/src/database.c
+++ b/src/database.c
@@ -611,10 +611,10 @@ int apk_repo_format_item(struct apk_database *db, struct apk_repository *repo, s
 }
 
 int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
-		       struct apk_package *pkg, int verify,
+		       struct apk_package *pkg, int verify, int autoupdate,
 		       apk_progress_cb cb, void *cb_ctx)
 {
-	struct stat st;
+	struct stat st = {0};
 	struct apk_istream *is;
 	struct apk_bstream *bs;
 	struct apk_sign_ctx sctx;
@@ -622,6 +622,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	char tmpcacheitem[128], *cacheitem = &tmpcacheitem[tmpprefix.len];
 	apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
 	int r, fd;
+	time_t now = time(NULL);
 
 	apk_blob_push_blob(&b, tmpprefix);
 	if (pkg != NULL)
@@ -633,9 +634,9 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	r = apk_repo_format_real_url(db, repo, pkg, url, sizeof(url));
 	if (r < 0) return r;
 
-	if ((apk_force & APK_FORCE_REFRESH) ||
-	    fstatat(db->cache_fd, cacheitem, &st, 0) != 0)
-		st.st_mtime = 0;
+	if (!(apk_force & APK_FORCE_REFRESH))
+		(void) fstatat(db->cache_fd, cacheitem, &st, 0);
+	if (autoupdate && now - st.st_mtime <= db->cache_max_age) return -EALREADY;
 
 	apk_message("fetch %s", url);
 
@@ -645,7 +646,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	if (verify != APK_SIGN_NONE) {
 		apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL, db->keys_fd);
 		bs = apk_bstream_from_url_if_modified(url, st.st_mtime);
-		bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, cb, cb_ctx);
+		bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, !autoupdate, cb, cb_ctx);
 		is = apk_bstream_gunzip_mpart(bs, apk_sign_ctx_mpart_cb, &sctx);
 		if (!IS_ERR_OR_NULL(is))
 			r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx, FALSE, &db->id_cache);
@@ -662,13 +663,18 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 		if (fd >= 0) {
 			struct apk_file_meta meta;
 			r = apk_istream_splice(is, fd, APK_SPLICE_ALL, cb, cb_ctx);
-			apk_istream_get_meta(is, &meta);
-			apk_file_meta_to_fd(fd, &meta);
+			if (!autoupdate) {
+				apk_istream_get_meta(is, &meta);
+				apk_file_meta_to_fd(fd, &meta);
+			}
 			close(fd);
 		}
 	}
 	if (!IS_ERR_OR_NULL(is)) apk_istream_close(is);
-	if (r == -EALREADY) return 0;
+	if (r == -EALREADY) {
+		if (autoupdate) utimensat(db->cache_fd, cacheitem, NULL, 0);
+		return r;
+	}
 	if (r < 0) {
 		unlinkat(db->cache_fd, tmpcacheitem, 0);
 		return r;
@@ -1515,6 +1521,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 
 	apk_db_setup_repositories(db, dbopts->cache_dir);
 
+	db->cache_max_age = dbopts->cache_max_age ?: 4*60*60; /* 4 hours default */
 	db->root = strdup(dbopts->root ?: "/");
 	db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC);
 	if (db->root_fd < 0 && (dbopts->open_flags & APK_OPENF_CREATE)) {
@@ -1677,7 +1684,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 			add_repos_from_file(db, db->root_fd, dbopts->repositories_file);
 		}
 
-		if (apk_flags & APK_UPDATE_CACHE)
+		if (db->repo_update_counter)
 			apk_db_index_write_nr_cache(db);
 
 		apk_hash_foreach(&db->available.names, apk_db_name_rdepends, db);
@@ -2118,10 +2125,13 @@ static int apk_repository_update(struct apk_database *db, struct apk_repository
 {
 	int r, verify = (apk_flags & APK_ALLOW_UNTRUSTED) ? APK_SIGN_NONE : APK_SIGN_VERIFY;
 
-	r = apk_cache_download(db, repo, NULL, verify, NULL, NULL);
+	r = apk_cache_download(db, repo, NULL, verify, 1, NULL, NULL);
+	if (r == -EALREADY) return 0;
 	if (r != 0) {
 		apk_error("%s: %s", repo->url, apk_error_str(r));
 		db->repo_update_errors++;
+	} else {
+		db->repo_update_counter++;
 	}
 
 	return r;
@@ -2247,16 +2257,13 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
 	apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
 
 	if (apk_url_local_file(repo->url) == NULL) {
-		if (!(apk_flags & APK_NO_NETWORK)) {
+		if (!(apk_flags & APK_NO_NETWORK))
 			db->available_repos |= BIT(repo_num);
-			if (apk_flags & APK_UPDATE_CACHE)
-				apk_repository_update(db, repo);
-		}
 		if (apk_flags & APK_NO_CACHE) {
 			r = apk_repo_format_real_url(db, repo, NULL, buf, sizeof(buf));
-			if (r == 0)
-				apk_message("fetch %s", buf);
+			if (r == 0) apk_message("fetch %s", buf);
 		} else {
+			apk_repository_update(db, repo);
 			r = apk_repo_format_cache_index(APK_BLOB_BUF(buf), repo);
 		}
 	} else {
@@ -2723,7 +2730,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 		apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
 		apk_blob_push_blob(&b, tmpprefix);
 		apk_pkg_format_cache_pkg(b, pkg);
-		cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, NULL, NULL);
+		cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, 1, NULL, NULL);
 		if (!IS_ERR_OR_NULL(cache_bs))
 			bs = cache_bs;
 		else
diff --git a/src/io.c b/src/io.c
index 7e2c89e..ff254fd 100644
--- a/src/io.c
+++ b/src/io.c
@@ -441,7 +441,7 @@ struct apk_bstream *apk_bstream_from_file(int atfd, const char *file)
 struct apk_tee_bstream {
 	struct apk_bstream bs;
 	struct apk_bstream *inner_bs;
-	int fd;
+	int fd, copy_meta;
 	size_t size;
 	apk_progress_cb cb;
 	void *cb_ctx;
@@ -475,9 +475,10 @@ static void tee_close(void *stream, size_t *size)
 	struct apk_tee_bstream *tbs =
 		container_of(stream, struct apk_tee_bstream, bs);
 
-	/* copy info */
-	apk_bstream_get_meta(tbs->inner_bs, &meta);
-	apk_file_meta_to_fd(tbs->fd, &meta);
+	if (tbs->copy_meta) {
+		apk_bstream_get_meta(tbs->inner_bs, &meta);
+		apk_file_meta_to_fd(tbs->fd, &meta);
+	}
 
 	apk_bstream_close(tbs->inner_bs, NULL);
 	if (size != NULL) *size = tbs->size;
@@ -491,7 +492,7 @@ static const struct apk_bstream_ops tee_bstream_ops = {
 	.close = tee_close,
 };
 
-struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, apk_progress_cb cb, void *cb_ctx)
+struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta, apk_progress_cb cb, void *cb_ctx)
 {
 	struct apk_tee_bstream *tbs;
 	int fd, r;
@@ -519,6 +520,7 @@ struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const ch
 	};
 	tbs->inner_bs = from;
 	tbs->fd = fd;
+	tbs->copy_meta = copy_meta;
 	tbs->size = 0;
 	tbs->cb = cb;
 	tbs->cb_ctx = cb_ctx;
diff --git a/src/update.c b/src/update.c
index 72e8673..02dcb5c 100644
--- a/src/update.c
+++ b/src/update.c
@@ -49,7 +49,7 @@ static struct apk_applet apk_update = {
 	.name = "update",
 	.help = "Update repository indexes from all remote repositories",
 	.open_flags = APK_OPENF_WRITE,
-	.forced_flags = APK_UPDATE_CACHE,
+	.forced_force = APK_FORCE_REFRESH,
 	.main = update_main,
 };