diff --git a/debian/changelog b/debian/changelog index 6113fd1..c68592d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +debootstrap (1.0.126+nmu1ubuntu0.6) jammy; urgency=medium + + * Support Packages files which are not ordered alphabetically by Package + field, by backporting upstream commit + 86ca8bcc736ceba53ad4a7d9b10b4c2ab65d739d (LP: #1990856) + + -- Daniel Watkins Wed, 12 Jul 2023 12:16:29 -0400 + debootstrap (1.0.126+nmu1ubuntu0.5) jammy; urgency=medium * Restore the old symlinks removed in the previous SRU. While not diff --git a/debian/tests/control b/debian/tests/control index b111df9..7b1e9bc 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -9,3 +9,11 @@ Depends: systemd-container [linux-any], ca-certificates, Restrictions: allow-stderr, needs-root + +Tests: unsorted-packages-files +Depends: + debootstrap, + python3-debian, + python3-flask, + python3-requests, +Restrictions: allow-stderr, needs-root diff --git a/debian/tests/mitm.py b/debian/tests/mitm.py new file mode 100644 index 0000000..cd2cd75 --- /dev/null +++ b/debian/tests/mitm.py @@ -0,0 +1,92 @@ +"""Flask app which MITM's an archive to generate out-of-order apt lists. + +Specifically, it prepends an additional Packages file stanza for a non-existent +lower version of apt: a fixed version of debootstrap will find the second +(correct) apt stanza and succeed; a broken version of debootstrap will find +only the first (non-existent) apt stanza and fail. +""" +import functools +import gzip +import hashlib +import os + +import requests +from debian.deb822 import Packages +from flask import Flask, redirect + +app = Flask(__name__) + +ARCH = os.environ.get("FLASK_ARCH", "amd64") +DIST = os.environ.get("FLASK_DIST", "bookworm") +DISTRO = os.environ.get("FLASK_DISTRO", "debian") +MIRROR = os.environ.get("FLASK_MIRROR", "http://deb.debian.org") + + +if DISTRO == "debian": + hash_funcs = [hashlib.md5, hashlib.sha256] +else: + # Ubuntu includes SHA1 still + hash_funcs = [hashlib.md5, hashlib.sha1, hashlib.sha256] + + +def _munge_release_file(url: str) -> bytes: + """Given a Release file URL, rewrite it for our modified Packages content.""" + original = requests.get(MIRROR + "/" + url).content + packages_content = _packages_content( + f"{DISTRO}/dists/{DIST}/main/binary-{ARCH}/Packages" + ) + size = bytes(str(len(packages_content)), "ascii") + sums = [ + bytes(hash_func(packages_content).hexdigest(), "ascii") + for hash_func in hash_funcs + ] + new_lines = [] + filename = f"main/binary-{ARCH}/Packages".encode("ascii") + for line in original.splitlines(): + if not line.endswith(filename): + new_lines.append(line) + continue + new_lines.append(b" ".join([b"", sums.pop(0), size, filename])) + return b"\n".join(new_lines) + + +@functools.lru_cache +def _packages_content(url: str) -> bytes: + """Given a Packages URL, fetch it and prepend a broken apt stanza.""" + resp = requests.get(MIRROR + "/" + url + ".gz") + upstream_content = gzip.decompress(resp.content) + + # Find the first `apt` stanza + for stanza in Packages.iter_paragraphs(upstream_content): + if stanza["Package"] == "apt": + break + + # Generate the broken stanza + new_version = stanza["Version"] + "~test" + stanza["Filename"] = stanza["Filename"].replace(stanza["Version"], new_version) + stanza["Version"] = new_version + + # Prepend the stanza to the upstream content + return bytes(stanza) + b"\n" + upstream_content + + +@app.route("/", methods=["GET", "POST"]) +def root(url): + """Handler for all requests.""" + if ( + url == f"{DISTRO}/dists/{DIST}/InRelease" + or "by-hash" in url + or "Packages.xz" in url + or "Packages.gz" in url + ): + # 404 these URLs to force clients to fetch by path and without compression, to + # make MITM easier + return "", 404 + if url == f"{DISTRO}/dists/{DIST}/Release": + # If Release is being fetched, return our modified version + return _munge_release_file(url) + if url == f"{DISTRO}/dists/{DIST}/main/binary-{ARCH}/Packages": + # If Packages is being fetched, return our modified version + return _packages_content(url) + # For anything we don't need to modify, redirect clients to upstream mirror + return redirect(f"{MIRROR}/{url}") diff --git a/debian/tests/unsorted-packages-files b/debian/tests/unsorted-packages-files new file mode 100755 index 0000000..7ab3121 --- /dev/null +++ b/debian/tests/unsorted-packages-files @@ -0,0 +1,22 @@ +# This test runs the mitm.py Flask app which debootstrap is then pointed at. +# mitm.py will return a Packages file which has an additional `apt` stanza +# prepended, with the Version and Filename adjusted to point at a lower, +# non-existent version. Versions of debootstrap which process _all_ Packages +# files entries will find the original stanza later in the file (and +# succesfully fetch the corresponding package file): versions that don't will +# find the prepended stanza and fail (with a 404 of the nonexistent package +# file). + +export FLASK_ARCH=amd64 +export FLASK_DIST=bookworm +export FLASK_DISTRO=debian +export FLASK_MIRROR=http://deb.debian.org + +# Launch our MitM "mirror" server, ensure that request logging is sent to stdout +(cd debian/tests; FLASK_APP=mitm.py flask run 2>&1 &) + +# Give Flask time to come up +sleep 2 + +# Run debootstrap against our MitM "mirror", ignoring the inevitable GPG errors +debootstrap --variant minbase --no-check-gpg ${FLASK_DIST} bootstrap http://127.0.0.1:5000/${FLASK_DISTRO}/ diff --git a/functions b/functions index aa8d2f9..476ea07 100644 --- a/functions +++ b/functions @@ -1415,26 +1415,31 @@ if in_path perl; then # uniq field mirror Packages values... perl -le ' $unique = shift @ARGV; $field = lc(shift @ARGV); $mirror = shift @ARGV; -%fields = map { $_, 0 } @ARGV; +%expected = map { $_, 0 } @ARGV; +%outputs; $prevpkg = ""; $chksumfield = lc($ENV{DEBOOTSTRAP_CHECKSUM_FIELD}).":"; + +sub emit_or_store_pkg { + if ($unique && defined $output_val) { + # Store the output for deduplicated emission later + $outputs{$output_val} = $output; + } else { + print $output if defined $output; + } +} + while () { if (/^([^:]*:)\s*(.*)$/) { $f = lc($1); $v = $2; if ($f eq "package:") { - $last = 0; $pkg = $v; if ($pkg ne $prevpkg) { - print $output if defined $output; - if ($unique && defined $output_val) { - delete $fields{$output_val}; - $last = 1 unless keys %fields; - } + emit_or_store_pkg; $prevpkg = $pkg; } undef $output; undef $output_val; - last if $last; } $ver = $v if ($f eq "version:"); $arc = $v if ($f eq "architecture:"); @@ -1443,7 +1448,7 @@ while () { $siz = $v if ($f eq "size:"); $val = $v if ($f eq $field); } elsif (/^$/) { - if (defined $val && defined $fields{$val}) { + if (defined $val && defined $expected{$val}) { $output = sprintf "%s %s %s %s %s %s %s", $pkg, $ver, $arc, $mirror, $fil, $chk, $siz; $output_val = $val; @@ -1451,10 +1456,15 @@ while () { undef $val; } } -print $output if defined $output; -delete $fields{$output_val} if $unique && defined $output_val; -for $v (keys %fields) { - printf ("%s -\n", $v) if ($unique); +emit_or_store_pkg; + +if ($unique) { + # Emit all of our deduplicated values + map { print } sort values %outputs; + # And emit any expected packages that were not found + foreach my $v (keys %expected) { + printf ("%s -\n", $v) if !defined $outputs{$v}; + } } ' "$@" }