From 2ad3d5b0d9a6354421f3c006e93d087ac6b1295f Mon Sep 17 00:00:00 2001 From: Jonas Smedegaard Date: Wed, 26 Dec 2007 02:22:15 +0000 Subject: Major rewrite. Works now --- localdebpool | 299 +++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 180 insertions(+), 119 deletions(-) (limited to 'localdebpool') diff --git a/localdebpool b/localdebpool index 3a38fd1..79fbbe7 100755 --- a/localdebpool +++ b/localdebpool @@ -1,62 +1,90 @@ #!/bin/sh set -e -#set -x ORIGIN="Jones" LABEL="Jones" DISTS="etch lenny sid" ARCHS="i386 powerpc amd64" -SRCBASE=~/public_debian +PRVBASE=~/public_debian PUBBASE=~/public_websites/debian.jones.dk #PUBTOSRC=../../public_debian -archsfrompaths() { - find "$@" -type l -exec find '{}'/ -type f -name '*.deb' ';' \ - | sed 's/^.*_\([a-z0-9-]\+\)\.deb$/\1/' \ - | sort -u +compactlist() { + ( for item in $@; do echo $item; done ) \ + | sort -u \ + | tr '\n' ' ' \ + | sed 's/[[:space:]]\+/ /g; s/^[[:space:]]//; s/[[:space:]]$//' } -distsfrompaths() { - find "$@" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' +regexfromlist() { + echo $@ | sed 's/[[:space:]]\+/\\|/g; s/^/\\(/; s/$/\\)/' } -pathsfromsections() { - case "$@" in - all) - sections_re="" - ;; - "") - echo 1>&2 "ERROR: At least one section (or the magic \"all\") is needed." - exit 1 - ;; - *) - sections_re="\b\($(echo "$@" | sed 's/[[:space:]]\+/\\|/g')\)\b" - ;; - esac - find $SRCBASE -type f -name HINTS -exec grep -q "^Sections:.*$sections_re" '{}' ';' -printf '%h\n' \ - | sed "s,^$SRCBASE/,," +sectionsfromhintstream() { + perl -ne 's/^Sections:\W*// && do { s/\W*,\W*/\n/g; s/\W*$/\n/; print; }' } -sectionsfrompaths() { - case "$@" in - all) - fullpaths="$SRCBASE" - ;; - *) - fullpaths= - for path in "$@"; do - # TODO: rewrite to support spaces in directory names - fullpaths="$fullpaths $SRCBASE/$path" +pkginfofromprvpkgdir() { + package="$(basename "$1")" + pool="$(dirname "$1" | sed 's,.*/pool-\([^/]\+\)/\?.*,\1,; s,-all$,,')" + dist="$(echo "$pool" | sed 's,[^a-z].*$,,')" + buildhost="$(echo $(dirname "$@") | sed 's,^\([^/]\+\)/pool-.*,\1,')" + + echo "$package $pool $dist $buildhost" +} + +archsfromdirs() { + archs="$( \ + find "$@" -type l -exec find '{}'/ -type f -name '*.deb' ';' \ + | sed 's/^.*_\([a-z0-9-]\+\)\.deb$/\1/; s/\ball\b/'"$ARCHS"'/g' \ + )" + archs="$(compactlist $archs)" + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "Archs found: \"$archs\"" + for arch in $archs; do + valid= + for arch_supported in $ARCHS; do + case $arch in + $arch_supported) + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "Arch ok: \"$arch\"" + echo "$arch" + valid=1 + ;; + esac done - ;; - esac - find $fullpaths -type f -name HINTS -exec cat '{}' ';' \ - | grep '^Sections:' \ - | sed 's/^Sections://; s/[[:space:]]\+/ /g; s/^ //; s/ $//' \ - | tr ' ' '\n' \ - | sort -u + if [ -z "$valid" ]; then + # TODO: Optionally just skip + echo 1>&2 "ERROR: Unsupported architecture: \"$arch\"." + exit 1 + fi + done +} + +distsfromprvgrpdirs() { + dists="$( \ + find "$@" -type l -exec find '{}'/ -type f -name '*.deb' ';' \ + | sed 's,^.*/sections/[^/]*/\([^/]\+\)/.*$,\1,' \ + )" + dists="$(compactlist $dists)" + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "Dists found: \"$dists\"" + for dist in $dists; do + valid= + for dist_supported in $DISTS; do + case $dist in + $dist_supported) + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "Dist ok: \"$dist\"" + echo "$dist" + valid=1 + ;; + esac + done + if [ -z "$valid" ]; then + # TODO: Optionally just skip + echo 1>&2 "ERROR: Unsupported distribution: \"$dist\"." + exit 1 + fi + done } genlist() { @@ -72,103 +100,136 @@ genlist() { ;; *) destdir="$basedir/binary-$arch"; - cmd="dpkg-scanpackages"; + cmd="dpkg-scanpackages -a$arch"; file="Packages"; ;; esac mkdir -p "$destdir" - ( cd "$srcdir" && $cmd $dist /dev/null pools/ ) > "$destdir/$file" - gzip -c "$destdir/$file" > "$destdir/$file.gz" - bzip2 -c "$destdir/$file" > "$destdir/$file.bz2" - cat < "$destdir/Release" + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "genlist: \"$cmd . /dev/null pkg/\"" + ( cd "$srcdir" && $cmd . /dev/null pkg/ ) > "$destdir/$file" + if [ -s "$destdir/$file" ]; then + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "Finalizing list at \"$destdir/\"" + gzip -c "$destdir/$file" > "$destdir/$file.gz" + bzip2 -c "$destdir/$file" > "$destdir/$file.bz2" + cat < "$destdir/Release" Archive: $dist Component: $section Origin: $ORIGIN Label: $LABEL Architecture: $arch EOF + else + [ -z "$DEBUG" ] || [ "$DEBUG" -lt 2 ] || echo 1>&2 "purge empty list at \"$destdir/\"" + rm -f "$destdir/$file" "$destdir/$file.gz" "$destdir/$file.bz2" "$destdir/Release" + rmdir -p --ignore-fail-on-non-empty "$destdir" + fi } -genlistsfromsection() { - section="$1" - case "$section" in - all) - srcdir="$SRCBASE/sections" - ;; - "") - echo 1>&2 "ERROR: At least one section (or the magic \"all\") is needed." - exit 1 - ;; - *) - srcdir="$SRCBASE/sections/$section" - ;; +genrelease() { + dist="$1" + shift + sections="$@" + archs="$(archsfromdirs "$PRVBASE/sections"/*/"$dist")" + archs="$(compactlist $archs)" + case "$dist" in + etch) suite="stable";; + lenny) suite="testing";; + sid) suite="unstable";; esac - dists="$(distsfrompaths "$srcdir")" - archs="$(archsfrompaths "$srcdir")" - for dist in $DISTS; do - do_dist= - for dist_found in $dists; do - if [ "$dist_found" = "$dist" ]; then - do_dist=1 - continue - fi - done - [ -n "$do_dist" ] || continue - archlist= - for arch in $ARCHS; do - do_arch= - for arch_found in $archs; do - case $arch_found in - $arch|all) - do_arch=1 - archlist="$archlist $arch" - continue - ;; - esac - done - [ -n "$do_arch" ] || continue - genlist "$dist" "$arch" "$srcdir" "$PUBBASE/dists/$dist/$section" - done - genlist "$dist" "source" "$srcdir" "$PUBBASE/dists/$dist/$section" - case "$dist" in - etch) suite="stable";; - lenny) suite="testing";; - sid) suite="unstable";; - esac -# TODO: Enable the below when running magic section "all" -# (needs all sections fed in here instead of each single section) -# cat < "$PUBBASE/dists/$dist/Release" -#Origin: $ORIGIN -#Label: $LABEL -#Suite: $suite -#Codename: $dist -#Architectures:$archlist -#Components: $sections -#EOF - done + cat < "$PUBBASE/dists/$dist/Release" +Origin: $ORIGIN +Label: $LABEL +Suite: $suite +Codename: $dist +Architectures: $archs +Components: $sections +EOF } -sections="$@" -paths="$(pathsfromsections "$sections")" -#echo $paths; exit 0 -#sectionsfrompaths all; exit 0 - -for path in $paths; do - package="$(basename "$path")" - pool="$(echo $(dirname "$path") | sed 's,.*/pool-\([^/]\+\)/\?.*,\1,; s,-all$,,')" - buildhost="$(echo $(dirname "$path") | sed 's,^\([^/]\+\)/pool-.*,\1,')" - symlink_dir="$pool/$package" - mkdir -p "$PUBBASE/pools/$symlink_dir" -# ln -sf -T "../../../$PUBTOSRC/$path" "$PUBBASE/pools/$symlink_dir/$buildhost" - ln -sf -T "$SRCBASE/$path" "$PUBBASE/pools/$symlink_dir/$buildhost" - for section in $(sectionsfrompaths "$path"); do - mkdir -p "$SRCBASE/sections/$section/$symlink_dir" - ln -sf -T "../../../../$path" "$SRCBASE/sections/$section/$symlink_dir/$buildhost" +packages="$@" +packages="$(compactlist $packages)" +[ -z "$DEBUG" ] || echo 1>&2 "Packages: \"$packages\"" + +case "$packages" in + -a|--all) + pkg_re=".*" + ;; + "") + echo 1>&2 "ERROR: No package(s) provided." + exit 1 + ;; + *) + pkg_re="\b$(regexfromlist $packages)\b" + ;; +esac + +batchsections="$( \ + find "$PRVBASE" -type f -regex '.*/'"$pkg_re"'/HINTS' -exec cat '{}' ';' \ + | sectionsfromhintstream +)" +batchsections="$(compactlist $batchsections)" +[ -z "$DEBUG" ] || echo 1>&2 "Sections in batch: \"$batchsections\"" + +batchsections_re= +if [ -n "$batchsections" ]; then + batchsections_re="\b$(regexfromlist $batchsections)\b" +else + echo 1>&2 "ERROR: Package(s) $@ was not found in any section." + exit 1 +fi +# Find packages in $prvpkgdir belonging to $sections, and +# create/update symlink farms $prvgrpdir and $pubpkgdir. +# +# $PRVBASE/$buildhost/pool-$pool/$package/ +# $PRVBASE/sections/$section/$dist/$package/$pool/$buildhost +# $PUBBASE/pkg/$package/$pool/$buildhost +# +cd "$PRVBASE" && find * -type f -name HINTS -exec grep -q "^Sections:.*$batchsections_re" '{}' ';' -printf '%h\n' \ + | while read prvpkgdir; do + [ -z "$DEBUG" ] || echo 1>&2 "prvpkgdir: \"$prvpkgdir\"" + set -- $(pkginfofromprvpkgdir "$prvpkgdir") + package="$1" pool="$2" dist="$3" buildhost="$4" + shift 4 + pkgsections="$(cat "$PRVBASE/$prvpkgdir/HINTS" | sectionsfromhintstream)" + pkgsections="$(compactlist $pkgsections)" + subdir="$package/$pool" + mkdir -p "$PUBBASE/pkg/$subdir" + ln -sf -T "$PRVBASE/$prvpkgdir" "$PUBBASE/pkg/$subdir/$buildhost" + for section in $pkgsections; do + mkdir -p "$PRVBASE/sections/$section/$dist/$subdir" + ln -sf -T "../../../../../$prvpkgdir" "$PRVBASE/sections/$section/$dist/$subdir/$buildhost" done done -for section in $sections; do - genlistsfromsection $section +# Create/update package lists +# +# $PUBBASE/dists/$dist/$section/{binary-$arch,source} +# +for section in $batchsections; do + dists="$(distsfromprvgrpdirs "$PRVBASE/sections/$section")" + for dist in $dists; do + [ -z "$DEBUG" ] || echo 1>&2 "genlists \"\$PRVBASE/sections/$section/$dist\" -> \"\$PUBBASE/dists/$dist/$section\"" + prvgrpdir="$PRVBASE/sections/$section/$dist" + pubpkgdir="$PUBBASE/dists/$dist/$section" + archs="$(archsfromdirs "$prvgrpdir")" + for arch in $archs; do + genlist "$dist" "$arch" "$prvgrpdir" "$pubpkgdir" + done + [ -z "$archs" ] || genlist "$dist" "source" "$prvgrpdir" "$pubpkgdir" + done +done + +allsections="$( \ + find "$PRVBASE" -type f -name HINTS -exec cat '{}' ';' \ + | sectionsfromhintstream +)" +allsections="$(compactlist $allsections)" +[ -z "$DEBUG" ] || echo 1>&2 "All sections: \"$allsections\"" + +dists="$(distsfromprvgrpdirs "$PRVBASE/sections")" +dists="$(compactlist $dists)" +for dist in $dists; do + genrelease "$dist" ${allsections} done exit 0 -- cgit v1.2.3