various: installation fixes

- extract everything as .apk-new and overwrite only after data
  has been checksummed
- url construction fixes (to work with simple http servers)
- end of gunzip stream fixed
- remove oneshot digesting flag for now as it's usage was broken

4 files changed, 156 insertions, 88 deletions

diff --git a/src/archive.c b/src/archive.c
index 26419d4..2cfb3e6 100644
--- a/src/archive.c
+++ b/src/archive.c
@@ -219,9 +219,10 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
 	/* Read remaining end-of-archive records, to ensure we read all of
 	 * the file. The underlying istream is likely doing checksumming. */
 	if (r == 512) {
-		while ((r = is->read(is, &buf, 512)) == 512)
+		while ((r = is->read(is, &buf, 512)) == 512) {
 			if (buf.name[0] != 0)
 				return -1;
+		}
 	}
 
 	/* Check that there was no partial record */
diff --git a/src/database.c b/src/database.c
index 4f9bb37..db0a4eb 100644
--- a/src/database.c
+++ b/src/database.c
@@ -305,6 +305,28 @@ struct apk_db_file *apk_db_file_query(struct apk_database *db,
 						   APK_BLOB_BUF(&key));
 }
 
+static struct apk_db_file *apk_db_file_new(struct apk_db_dir_instance *diri,
+					   apk_blob_t name,
+					   struct hlist_node ***after)
+{
+	struct apk_db_file *file;
+
+	file = malloc(sizeof(*file) + name.len + 1);
+	if (file == NULL)
+		return NULL;
+
+	memset(file, 0, sizeof(*file));
+	memcpy(file->name, name.ptr, name.len);
+	file->name[name.len] = 0;
+	file->namelen = name.len;
+
+	file->diri = diri;
+	hlist_add_after(&file->diri_files_list, *after);
+	*after = &file->diri_files_list.next;
+
+	return file;
+}
+
 static struct apk_db_file *apk_db_file_get(struct apk_database *db,
 					   struct apk_db_dir_instance *diri,
 					   apk_blob_t name,
@@ -326,33 +348,13 @@ static struct apk_db_file *apk_db_file_get(struct apk_database *db,
 	if (file != NULL)
 		return file;
 
-	file = malloc(sizeof(*file) + name.len + 1);
-	memset(file, 0, sizeof(*file));
-	memcpy(file->name, name.ptr, name.len);
-	file->name[name.len] = 0;
-	file->namelen = name.len;
-
-	file->diri = diri;
-	hlist_add_after(&file->diri_files_list, *after);
-	*after = &file->diri_files_list.next;
-
+	file = apk_db_file_new(diri, name, after);
 	apk_hash_insert_hashed(&db->installed.files, file, hash);
 	db->installed.stats.files++;
 
 	return file;
 }
 
-static void apk_db_file_change_owner(struct apk_database *db,
-				     struct apk_db_file *file,
-				     struct apk_db_dir_instance *diri,
-				     struct hlist_node ***after)
-{
-	hlist_del(&file->diri_files_list, &file->diri->owned_files);
-	file->diri = diri;
-	hlist_add_after(&file->diri_files_list, *after);
-	*after = &file->diri_files_list.next;
-}
-
 static void apk_db_pkg_rdepends(struct apk_database *db, struct apk_package *pkg)
 {
 	int i, j;
@@ -469,8 +471,7 @@ int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
 				apk_error("FDB file entry before directory entry");
 				return -1;
 			}
-			file = apk_db_file_get(db, diri, l,
-					       &file_diri_node);
+			file = apk_db_file_get(db, diri, l, &file_diri_node);
 			break;
 		case 'Z':
 			if (file == NULL) {
@@ -1031,8 +1032,10 @@ static struct apk_bstream *apk_repository_file_open(struct apk_repository *repo,
 						    const char *file)
 {
 	char tmp[256];
+	const char *url = repo->url;
 
-	snprintf(tmp, sizeof(tmp), "%s/%s", repo->url, file);
+	snprintf(tmp, sizeof(tmp), "%s%s%s",
+		 url, url[strlen(url)-1] == '/' ? "" : "/", file);
 
 	return apk_bstream_from_url(tmp);
 }
@@ -1063,9 +1066,9 @@ int apk_cache_download(struct apk_database *db, struct apk_checksum *csum,
 		apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL);
 		is = apk_bstream_gunzip_mpart(apk_bstream_from_file(tmp2),
 			apk_sign_ctx_mpart_cb, &sctx);
-		is->close(is);
 		r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx);
-		ok = (r != 0) && sctx.control_verified && sctx.data_verified;
+		is->close(is);
+		ok = (r == 0) && sctx.control_verified && sctx.data_verified;
 		apk_sign_ctx_free(&sctx);
 		if (!ok) {
 			unlink(tmp2);
@@ -1115,7 +1118,7 @@ int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
 			       APK_SIGN_VERIFY);
 	if (r == 0 || r == -10) {
 		if (r == -10)
-			apk_error("Failed to update %s: bad signature!");
+			apk_error("%s: untrusted or bad signature!", repo->url);
 		apk_cache_delete(db, &repo->csum, apk_index_gz);
 		return r;
 	}
@@ -1279,7 +1282,6 @@ static int apk_db_install_archive_entry(void *_ctx,
 	apk_blob_t name = APK_BLOB_STR(ae->name), bdir, bfile;
 	struct apk_db_dir_instance *diri = ctx->diri;
 	struct apk_db_file *file;
-	struct apk_file_info fi;
 	char alt_name[PATH_MAX];
 	const char *p;
 	int r = 0, type = APK_SCRIPT_INVALID;
@@ -1355,14 +1357,8 @@ static int apk_db_install_archive_entry(void *_ctx,
 			ctx->file_diri_node = hlist_tail_ptr(&diri->owned_files);
 		}
 
-		file = apk_db_file_get(db, diri, bfile, &ctx->file_diri_node);
-		if (file == NULL) {
-			apk_error("%s: Failed to create fdb entry for '%*s'\n",
-				  pkg->name->name, name.len, name.ptr);
-			return -1;
-		}
-
-		if (file->diri != diri) {
+		file = apk_db_file_query(db, bdir, bfile);
+		if (file != NULL) {
 			opkg = file->diri->pkg;
 			if (opkg->name != pkg->name) {
 				if (!(apk_flags & APK_FORCE)) {
@@ -1376,36 +1372,19 @@ static int apk_db_install_archive_entry(void *_ctx,
 					    pkg->name->name, ae->name,
 					    opkg->name->name);
 			}
-
-			apk_db_file_change_owner(db, file, diri,
-						 &ctx->file_diri_node);
 		}
 
+		/* Create the file entry without adding it to hash */
+		file = apk_db_file_new(diri, bfile, &ctx->file_diri_node);
+
 		if (apk_verbosity > 1)
 			printf("%s\n", ae->name);
 
-		if ((diri->dir->flags & APK_DBDIRF_PROTECTED) &&
-		    apk_file_get_info(ae->name, file->csum.type, &fi) == 0 &&
-		    apk_checksum_compare(&file->csum, &fi.csum) != 0) {
-			/* Protected file. Extract to separate place */
-			if (!(apk_flags & APK_CLEAN_PROTECTED)) {
-				snprintf(alt_name, sizeof(alt_name),
-					 "%s/%s.apk-new",
-					 diri->dir->name, file->name);
-				r = apk_archive_entry_extract(ae, is, alt_name,
-							      extract_cb, ctx);
-
-				/* remove identical apk-new */
-				if (ae->csum.type != fi.csum.type)
-					apk_file_get_info(ae->name, ae->csum.type, &fi);
-				if (apk_checksum_compare(&ae->csum, &fi.csum) == 0)
-					unlink(alt_name);
-			}
-		} else {
-			r = apk_archive_entry_extract(ae, is, NULL,
-						      extract_cb, ctx);
-		}
-		memcpy(&file->csum, &ae->csum, sizeof(file->csum));
+		/* Extract the file as name.apk-new */
+		snprintf(alt_name, sizeof(alt_name), "%s/%s.apk-new",
+			 diri->dir->name, file->name);
+		r = apk_archive_entry_extract(ae, is, alt_name,
+					      extract_cb, ctx);
 	} else {
 		if (apk_verbosity > 1)
 			printf("%s\n", ae->name);
@@ -1427,8 +1406,8 @@ static int apk_db_install_archive_entry(void *_ctx,
 	return r;
 }
 
-static void apk_db_purge_pkg(struct apk_database *db,
-			     struct apk_package *pkg)
+static void apk_db_purge_pkg(struct apk_database *db, struct apk_package *pkg,
+			     const char *exten)
 {
 	struct apk_db_dir_instance *diri;
 	struct apk_db_file *file;
@@ -1439,8 +1418,8 @@ static void apk_db_purge_pkg(struct apk_database *db,
 
 	hlist_for_each_entry_safe(diri, dc, dn, &pkg->owned_dirs, pkg_dirs_list) {
 		hlist_for_each_entry_safe(file, fc, fn, &diri->owned_files, diri_files_list) {
-			snprintf(name, sizeof(name), "%s/%s",
-				 diri->dir->name, file->name);
+			snprintf(name, sizeof(name), "%s/%s%s",
+				 diri->dir->name, file->name, exten ?: "");
 
 			key = (struct apk_db_file_hash_key) {
 				.dirname = APK_BLOB_PTR_LEN(diri->dir->name, diri->dir->namelen),
@@ -1451,8 +1430,10 @@ static void apk_db_purge_pkg(struct apk_database *db,
 			if (apk_verbosity > 1)
 				printf("%s\n", name);
 			__hlist_del(fc, &diri->owned_files.first);
-			apk_hash_delete_hashed(&db->installed.files, APK_BLOB_BUF(&key), hash);
-			db->installed.stats.files--;
+			if (exten == NULL) {
+				apk_hash_delete_hashed(&db->installed.files, APK_BLOB_BUF(&key), hash);
+				db->installed.stats.files--;
+			}
 		}
 		apk_db_diri_rmdir(diri);
 		__hlist_del(dc, &pkg->owned_dirs.first);
@@ -1461,6 +1442,94 @@ static void apk_db_purge_pkg(struct apk_database *db,
 	apk_pkg_set_state(db, pkg, APK_PKG_NOT_INSTALLED);
 }
 
+
+static void apk_db_migrate_files(struct apk_database *db,
+				 struct apk_package *pkg)
+{
+	struct apk_db_dir_instance *diri;
+	struct apk_db_dir *dir;
+	struct apk_db_file *file, *ofile;
+	struct apk_db_file_hash_key key;
+	struct apk_file_info fi;
+	struct hlist_node *dc, *dn, *fc, *fn;
+	unsigned long hash;
+	char name[1024], tmpname[1024];
+
+	hlist_for_each_entry_safe(diri, dc, dn, &pkg->owned_dirs, pkg_dirs_list) {
+		dir = diri->dir;
+
+		hlist_for_each_entry_safe(file, fc, fn, &diri->owned_files, diri_files_list) {
+			snprintf(name, sizeof(name), "%s/%s",
+				 diri->dir->name, file->name);
+			snprintf(tmpname, sizeof(tmpname), "%s/%s.apk-new",
+				 diri->dir->name, file->name);
+
+			key = (struct apk_db_file_hash_key) {
+				.dirname = APK_BLOB_PTR_LEN(dir->name, dir->namelen),
+				.filename = APK_BLOB_PTR_LEN(file->name, file->namelen),
+			};
+
+			hash = apk_blob_hash_seed(key.filename, dir->hash);
+
+			/* check for existing file */
+			ofile = (struct apk_db_file *) apk_hash_get_hashed(
+				&db->installed.files, APK_BLOB_BUF(&key), hash);
+
+			if ((diri->dir->flags & APK_DBDIRF_PROTECTED) &&
+			    ofile != NULL &&
+			    apk_file_get_info(name, ofile->csum.type, &fi) == 0 &&
+			    apk_checksum_compare(&ofile->csum, &fi.csum) != 0) {
+				/* Protected dir and existing file has been
+				 * changed */
+				if (ofile->csum.type != file->csum.type)
+					apk_file_get_info(name, file->csum.type, &fi);
+				if (apk_checksum_compare(&file->csum, &fi.csum) == 0)
+					unlink(tmpname);
+			} else {
+				/* Overwrite the old file */
+				rename(tmpname, name);
+			}
+
+			/* Claim ownership of the file in db */
+			if (ofile != NULL) {
+				hlist_del(&ofile->diri_files_list,
+					  &diri->owned_files);
+				apk_hash_delete_hashed(&db->installed.files,
+						       APK_BLOB_BUF(&key), hash);
+			} else
+				db->installed.stats.files++;
+
+			apk_hash_insert_hashed(&db->installed.files, file, hash);
+		}
+	}
+}
+
+#if 0
+		if ((diri->dir->flags & APK_DBDIRF_PROTECTED) &&
+		    apk_file_get_info(ae->name, file->csum.type, &fi) == 0 &&
+		    apk_checksum_compare(&file->csum, &fi.csum) != 0) {
+			/* Protected file. Extract to separate place */
+			if (!(apk_flags & APK_CLEAN_PROTECTED)) {
+				snprintf(alt_name, sizeof(alt_name),
+					 "%s/%s.apk-new",
+					 diri->dir->name, file->name);
+				r = apk_archive_entry_extract(ae, is, alt_name,
+							      extract_cb, ctx);
+
+				/* remove identical apk-new */
+				if (ae->csum.type != fi.csum.type)
+					apk_file_get_info(ae->name, ae->csum.type, &fi);
+				if (apk_checksum_compare(&ae->csum, &fi.csum) == 0)
+					unlink(alt_name);
+			}
+		} else {
+			r = apk_archive_entry_extract(ae, is, NULL,
+						      extract_cb, ctx);
+		}
+		memcpy(&file->csum, &ae->csum, sizeof(file->csum));
+#endif
+
+
 static int apk_db_unpack_pkg(struct apk_database *db,
 			     struct apk_package *newpkg,
 			     int upgrade, apk_progress_cb cb, void *cb_ctx)
@@ -1493,9 +1562,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 			bs = apk_db_cache_open(db, &newpkg->csum, pkgname);
 
 		if (bs == NULL) {
-			snprintf(file, sizeof(file), "%s/%s",
-				 repo->url, pkgname);
-			bs = apk_bstream_from_url(file);
+			bs = apk_repository_file_open(repo, pkgname);
 			if (repo->csum.type != APK_CHECKSUM_NONE)
 				need_copy = TRUE;
 		}
@@ -1533,11 +1600,13 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 	if (r != 0) {
 		apk_error("%s-%s: package integrity check failed",
 			  newpkg->name->name, newpkg->version);
-		return -1;
+		goto err;
 	}
 	r = apk_db_run_pending_script(&ctx);
 	if (r != 0)
-		return r;
+		goto err;
+
+	apk_db_migrate_files(db, newpkg);
 
 	if (need_copy) {
 		char file2[256];
@@ -1547,6 +1616,9 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 	}
 
 	return 0;
+err:
+	apk_db_purge_pkg(db, newpkg, ".apk-new");
+	return r;
 }
 
 int apk_db_install_pkg(struct apk_database *db,
@@ -1566,7 +1638,7 @@ int apk_db_install_pkg(struct apk_database *db,
 		if (r != 0)
 			return r;
 
-		apk_db_purge_pkg(db, oldpkg);
+		apk_db_purge_pkg(db, oldpkg, NULL);
 
 		r = apk_pkg_run_script(oldpkg, db->root_fd,
 				       APK_SCRIPT_POST_DEINSTALL);
@@ -1583,7 +1655,7 @@ int apk_db_install_pkg(struct apk_database *db,
 	apk_pkg_set_state(db, newpkg, APK_PKG_INSTALLED);
 
 	if (oldpkg != NULL)
-		apk_db_purge_pkg(db, oldpkg);
+		apk_db_purge_pkg(db, oldpkg, NULL);
 
 	r = apk_pkg_run_script(newpkg, db->root_fd,
 			       (oldpkg == NULL) ?
diff --git a/src/gunzip.c b/src/gunzip.c
index 8ccfbab..f14b396 100644
--- a/src/gunzip.c
+++ b/src/gunzip.c
@@ -50,7 +50,8 @@ static size_t gzi_read(void *stream, void *ptr, size_t size)
 		if (gis->zs.avail_in == 0) {
 			apk_blob_t blob;
 
-			if (gis->cb != NULL && gis->cbprev != NULL) {
+			if (gis->cb != NULL && gis->cbprev != NULL &&
+			    gis->cbprev != gis->zs.next_in) {
 				gis->cb(gis->cbctx, APK_MPART_DATA,
 					APK_BLOB_PTR_LEN(gis->cbprev,
 					(void *)gis->zs.next_in - gis->cbprev));
@@ -63,16 +64,15 @@ static size_t gzi_read(void *stream, void *ptr, size_t size)
 				gis->err = -1;
 				goto ret;
 			} else if (gis->zs.avail_in == 0) {
+				gis->err = 1;
 				if (gis->cb != NULL) {
 					r = gis->cb(gis->cbctx, APK_MPART_END,
 						    APK_BLOB_NULL);
-					if (r != 0) {
-						if (r > 0)
-							r = -1;
+					if (r > 0)
+						r = -1;
+					if (r != 0)
 						gis->err = r;
-					}
-				} else
-					gis->err = 1;
+				}
 				goto ret;
 			}
 		}
@@ -107,7 +107,7 @@ static size_t gzi_read(void *stream, void *ptr, size_t size)
 
 ret:
 	if (size - gis->zs.avail_out == 0)
-		return gis->err;
+		return gis->err < 0 ? gis->err : 0;
 
 	return size - gis->zs.avail_out;
 }
diff --git a/src/package.c b/src/package.c
index 16d72f9..9fb4024 100644
--- a/src/package.c
+++ b/src/package.c
@@ -293,7 +293,6 @@ void apk_sign_ctx_init(struct apk_sign_ctx *ctx, int action,
 	}
 	EVP_MD_CTX_init(&ctx->mdctx);
 	EVP_DigestInit_ex(&ctx->mdctx, ctx->md, NULL);
-	EVP_MD_CTX_set_flags(&ctx->mdctx, EVP_MD_CTX_FLAG_ONESHOT);
 }
 
 
@@ -406,7 +405,6 @@ int apk_sign_ctx_mpart_cb(void *ctx, int part, apk_blob_t data)
 
 	switch (part) {
 	case APK_MPART_DATA:
-		EVP_MD_CTX_clear_flags(&sctx->mdctx, EVP_MD_CTX_FLAG_ONESHOT);
 		EVP_DigestUpdate(&sctx->mdctx, data.ptr, data.len);
 		break;
 	case APK_MPART_BOUNDARY:
@@ -417,7 +415,6 @@ int apk_sign_ctx_mpart_cb(void *ctx, int part, apk_blob_t data)
 		if (!sctx->control_started) {
 			EVP_DigestFinal_ex(&sctx->mdctx, calculated, NULL);
 			EVP_DigestInit_ex(&sctx->mdctx, sctx->md, NULL);
-			EVP_MD_CTX_set_flags(&sctx->mdctx, EVP_MD_CTX_FLAG_ONESHOT);
 			return 0;
 		}
 
@@ -442,7 +439,6 @@ int apk_sign_ctx_mpart_cb(void *ctx, int part, apk_blob_t data)
 
 			sctx->control_verified = 1;
 			EVP_DigestInit_ex(&sctx->mdctx, sctx->md, NULL);
-			EVP_MD_CTX_set_flags(&sctx->mdctx, EVP_MD_CTX_FLAG_ONESHOT);
 			return 0;
 		} else if (sctx->action == APK_SIGN_GENERATE) {
 			/* Package identity is checksum of control block */
@@ -453,7 +449,6 @@ int apk_sign_ctx_mpart_cb(void *ctx, int part, apk_blob_t data)
 			/* Reset digest for hashing data */
 			EVP_DigestFinal_ex(&sctx->mdctx, calculated, NULL);
 			EVP_DigestInit_ex(&sctx->mdctx, sctx->md, NULL);
-			EVP_MD_CTX_set_flags(&sctx->mdctx, EVP_MD_CTX_FLAG_ONESHOT);
 
 			if (sctx->action == APK_SIGN_VERIFY_IDENTITY) {
 				if (memcmp(calculated, sctx->identity.data,