Tested on Fuel node.

config/common.cfg

@@ -6,16 +6,21 @@ LOG_ROOT="/var/log/mirror-sync"
 
 # Attempt to autodetect current Fuel version
 if hash fuel 2>/dev/null; then
-    # working on Fuel master node
+    # working on Fuel node
     FUEL_VERSION=`fuel --fuel-version 2>&1 | awk -v sq="'" '/^release:/ { gsub(sq,""); print $2 }'`
     if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_VERSION}/config.mk -O /dev/null; then
         FUEL_BRANCH=master
     else
         FUEL_BRANCH=$FUEL_VERSION
     fi
+    # On Fuel node we will be using dockerized Ubuntu for dependencies resolving.
+    # Assuming that Docker is up and running.
+    DOCKER_MODE=true
 else
     # working outside Fuel master node
-    echo "Cannot detect current Fuel version, using defaults. Please configure settings in config/common.cfg"
+    echo "Cannot detect current Fuel version, using defaults. Please configure FUEL_* settings in config/common.cfg"
+    sleep 3
     FUEL_VERSION=6.1
     FUEL_BRANCH=master
+    DOCKER_MODE=false
 fi

config/mos-ubuntu-updatesonly.cfg

@@ -2,8 +2,8 @@
 source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg"
 
 # Sync source
-UPSTREAM="osci-mirror-kha.kha.mirantis.net"
+UPSTREAM="mirror.fuel-infra.org"
-UPSTREAM_DIR="mirror-sync/mos/ubuntu"
+UPSTREAM_DIR="mirror/mos/ubuntu"
 UPSTREAM_DIR_HTTP="mos/ubuntu"
 
 # NOTE! Partial repository sync feature requires apt support on your OS!
@@ -35,7 +35,7 @@ FETCH_DIFF="no"        # Fetch diffs of 'Packages' (speed's up client update)
 FETCH_INDICES="no"     # Fetch indices
 
 # Misc
-DEBUG="no"
+DEBUG="yes"
 QUIET="no"
 DISABLE_CLEANUP="no"
 LOG_FILE="${LOG_ROOT:-"/var/log/mirror-sync"}/ubuntu-update.log"

config/mos-ubuntu.cfg

@@ -2,8 +2,8 @@
 source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg"
 
 # Sync source
-UPSTREAM="osci-mirror-kha.kha.mirantis.net"
+UPSTREAM="mirror.fuel-infra.org"
-UPSTREAM_DIR="mirror-sync/mos/ubuntu"
+UPSTREAM_DIR="mirror/mos/ubuntu"
 UPSTREAM_DIR_HTTP="mos/ubuntu"
 
 # NOTE! Partial repository sync feature requires apt support on your OS!

config/ubuntu.cfg

@@ -38,7 +38,7 @@ FETCH_DIFF="no"        # Fetch diffs of 'Packages' (speed's up client update)
 FETCH_INDICES="yes"     # Fetch indices
 
 # Misc
-DEBUG="no"
+DEBUG="yes"
 QUIET="no"
 DISABLE_CLEANUP="no"
 LOG_FILE="${LOG_ROOT:-"/var/log/mirror-sync"}/ubuntu-update.log"

deb-mirror

@@ -451,8 +451,6 @@ if [[ "$PARTIAL_UPSTREAM" = "1" ]]; then
 
 info "Resolving dependencies for partial mirror"
 
-[ -f /usr/bin/apt-get ] || fatal "APT not found on this system, creation of partial repository will fail"
-
 # Detect kernel version of debian-installer
 fuel_config_temp=`mktemp --suffix="-fuel-config-temp"`
 if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/config.mk -O $fuel_config_temp; then
@@ -473,7 +471,14 @@ rm -f "$fuel_config_temp"
 export apt_altstate=`mktemp -d --suffix="-apt-altstate"`
 export FUEL_BRANCH
 
-$BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
+if [[ "$DOCKER_MODE" = "true" ]]; then 
+	# docker pull ubuntu:latest
+	docker -D run --net=bridge --rm -a stdout -a stderr -i -t --privileged -e FUEL_BRANCH=$FUEL_BRANCH -e BINROOT=$BINROOT -e apt_altstate=$apt_altstate -v $BINROOT:$BINROOT:rw -v $apt_altstate:$apt_altstate:rw ubuntu:latest $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
+        # cleanup ubuntu container
+        docker images | awk '/ubuntu/ { print $3; }' | xargs --no-run-if-empty docker rmi -f 
+else
+	$BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
+fi
 
 # Create download lists for deb and udeb
 awk 'FNR==NR {arr[$0];next} $3 in arr' $apt_altstate/deb "$deb_md5" > $apt_altstate/deb_md5
@@ -605,8 +610,10 @@ EOF
 
 # Build partial mirror
 info "Generating metadata for partial mirror"
+info "Apply fix for upstream dpkg-scanpackages"
+patch -N /usr/bin/dpkg-scanpackages < $BINROOT/util/dpkg.patch
 export BINROOT
-$BINROOT/util/regenerate_ubuntu_repo ${PARTIAL_UPSTREAM_PATH} ${UBUNTU_RELEASE} || failure "Failed to generate partial mirror"
+$BINROOT/util/regenerate_ubuntu_repo ${PARTIAL_UPSTREAM_PATH} ${UBUNTU_RELEASE} || fatal "Failed to generate partial mirror"
 rm -rf $LOCAL_DIR
 fi # "$PARTIAL_UPSTREAM" = "1"
 info "Done"

util/dpkg-scanpackages

@@ -1,289 +0,0 @@
-#!/usr/bin/perl
-#
-# dpkg-scanpackages
-#
-# Copyright © 2006-2012 Guillem Jover <guillem@debian.org>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-use warnings;
-use strict;
-
-use IO::Handle;
-use IO::File;
-use Getopt::Long qw(:config posix_default bundling no_ignorecase);
-
-use Dpkg ();
-use Dpkg::Gettext;
-use Dpkg::ErrorHandling;
-#use Dpkg::Util qw(:list);
-use Dpkg::Control;
-use Dpkg::Version;
-use Dpkg::Checksums;
-use Dpkg::Compression::FileHandle;
-use Dpkg::IPC;
-
-textdomain('dpkg-dev');
-
-# Do not pollute STDOUT with info messages
-report_options(info_fh => \*STDERR);
-
-my (@samemaint, @changedmaint);
-my @spuriousover;
-my %packages;
-my %overridden;
-
-my %options = (help            => sub { usage(); exit 0; },
-	       version         => \&version,
-	       type            => undef,
-	       arch            => undef,
-	       multiversion    => 0,
-	       'extra-override'=> undef,
-               medium          => undef,
-	      );
-
-my @options_spec = (
-    'help|?',
-    'version',
-    'type|t=s',
-    'arch|a=s',
-    'multiversion|m!',
-    'extra-override|e=s',
-    'medium|M=s',
-);
-
-sub version {
-    printf _g("Debian %s version.\n"), $Dpkg::PROGNAME;
-    exit;
-}
-
-sub usage {
-    printf _g(
-"Usage: %s [<option>...] <binary-path> [<override-file> [<path-prefix>]] > Packages
-
-Options:
-  -t, --type <type>        scan for <type> packages (default is 'deb').
-  -a, --arch <arch>        architecture to scan for.
-  -m, --multiversion       allow multiple versions of a single package.
-  -e, --extra-override <file>
-                           use extra override file.
-  -M, --medium <medium>    add X-Medium field for dselect multicd access method
-  -?, --help               show this help message.
-      --version            show the version.
-"), $Dpkg::PROGNAME;
-}
-
-sub load_override
-{
-    my $override = shift;
-    my $comp_file = Dpkg::Compression::FileHandle->new(filename => $override);
-
-    while (<$comp_file>) {
-	s/\#.*//;
-	s/\s+$//;
-	next unless $_;
-
-	my ($p, $priority, $section, $maintainer) = split(/\s+/, $_, 4);
-
-	if (not defined($packages{$p})) {
-	    push(@spuriousover, $p);
-	    next;
-	}
-
-	for my $package (@{$packages{$p}}) {
-	    if ($maintainer) {
-		if ($maintainer =~ m/(.+?)\s*=\>\s*(.+)/) {
-		    my $oldmaint = $1;
-		    my $newmaint = $2;
-		    my $debmaint = $$package{Maintainer};
-		    if (none { $debmaint eq $_ } split m{\s*//\s*}, $oldmaint) {
-			push(@changedmaint,
-			     sprintf(_g('  %s (package says %s, not %s)'),
-			             $p, $$package{Maintainer}, $oldmaint));
-		    } else {
-			$$package{Maintainer} = $newmaint;
-		    }
-		} elsif ($$package{Maintainer} eq $maintainer) {
-		    push(@samemaint, "  $p ($maintainer)");
-		} else {
-		    warning(_g('unconditional maintainer override for %s'), $p);
-		    $$package{Maintainer} = $maintainer;
-		}
-	    }
-	    $$package{Priority} = $priority;
-	    $$package{Section} = $section;
-	}
-	$overridden{$p} = 1;
-    }
-
-    close($comp_file);
-}
-
-sub load_override_extra
-{
-    my $extra_override = shift;
-    my $comp_file = Dpkg::Compression::FileHandle->new(filename => $extra_override);
-
-    while (<$comp_file>) {
-	s/\#.*//;
-	s/\s+$//;
-	next unless $_;
-
-	my ($pr, $field, $value) = split(/\s+/, $_, 3);
-
-        my ($p, $parch) = split(/\//, $pr, 2);
-        if (defined($options{arch}) and defined($parch)) {
-            next unless ($options{arch} eq $parch);
-        }
-
-	next unless defined($packages{$p});
-
-	for my $package (@{$packages{$p}}) {
-	    $$package{$field} = $value;
-	}
-    }
-
-    close($comp_file);
-}
-
-{
-    local $SIG{__WARN__} = sub { usageerr($_[0]) };
-    GetOptions(\%options, @options_spec);
-}
-
-if (not (@ARGV >= 1 and @ARGV <= 3)) {
-    usageerr(_g('one to three arguments expected'));
-}
-
-my $type = defined($options{type}) ? $options{type} : 'deb';
-my $arch = $options{arch};
-
-my @find_args;
-if ($options{arch}) {
-     @find_args = ('(', '-name', "*_all.$type", '-o',
-			'-name', "*_${arch}.$type", ')');
-}
-else {
-     @find_args = ('-name', "*.$type");
-}
-
-my ($binarydir, $override, $pathprefix) = @ARGV;
-
-if (not -d $binarydir) {
-    error(_g('binary dir %s not found'), $binarydir);
-}
-if (defined $override and not -e $override) {
-    error(_g('override file %s not found'), $override);
-}
-
-$pathprefix //= '';
-
-my $find_h = IO::Handle->new();
-open($find_h, '-|', 'find', '-L', "$binarydir/", @find_args, '-print')
-     or syserr(_g("couldn't open %s for reading"), $binarydir);
-FILE:
-    while (<$find_h>) {
-	chomp;
-	my $fn = $_;
-	my $output;
-	my $pid = spawn(exec => [ 'dpkg-deb', '-I', $fn, 'control' ],
-	                to_pipe => \$output);
-	my $fields = Dpkg::Control->new(type => CTRL_INDEX_PKG);
-	$fields->parse($output, $fn)
-	    or error(_g("couldn't parse control information from %s"), $fn);
-	wait_child($pid, no_check => 1);
-	if ($?) {
-	    warning(_g("\`dpkg-deb -I %s control' exited with %d, skipping package"),
-	            $fn, $?);
-	    next;
-	}
-
-	defined($fields->{'Package'})
-	    or error(_g('no Package field in control file of %s'), $fn);
-	my $p = $fields->{'Package'};
-
-	if (defined($packages{$p}) and not $options{multiversion}) {
-	    foreach (@{$packages{$p}}) {
-		if (version_compare_relation($fields->{'Version'}, REL_GT,
-					     $_->{'Version'}))
-                {
-		    warning(_g('package %s (filename %s) is repeat but newer version;'),
-		            $p, $fn);
-		    warning(_g('used that one and ignored data from %s!'),
-		            $_->{Filename});
-		    $packages{$p} = [];
-		} else {
-		    warning(_g('package %s (filename %s) is repeat;'), $p, $fn);
-		    warning(_g('ignored that one and using data from %s!'),
-		            $_->{Filename});
-		    next FILE;
-		}
-	    }
-	}
-	warning(_g('package %s (filename %s) has Filename field!'), $p, $fn)
-	    if defined($fields->{'Filename'});
-
-	$fields->{'Filename'} = "$pathprefix$fn";
-
-        my $sums = Dpkg::Checksums->new();
-	$sums->add_from_file($fn);
-        foreach my $alg (checksums_get_list()) {
-            if ($alg eq 'md5') {
-	        $fields->{'MD5sum'} = $sums->get_checksum($fn, $alg);
-            } else {
-                $fields->{$alg} = $sums->get_checksum($fn, $alg);
-            }
-        }
-	$fields->{'Size'} = $sums->get_size($fn);
-        $fields->{'X-Medium'} = $options{medium} if defined $options{medium};
-
-	push @{$packages{$p}}, $fields;
-    }
-close($find_h);
-
-load_override($override) if defined $override;
-load_override_extra($options{'extra-override'}) if defined $options{'extra-override'};
-
-my @missingover=();
-
-my $records_written = 0;
-for my $p (sort keys %packages) {
-    if (defined($override) and not defined($overridden{$p})) {
-        push(@missingover,$p);
-    }
-    for my $package (@{$packages{$p}}) {
-         print("$package\n") or syserr(_g('failed when writing stdout'));
-         $records_written++;
-    }
-}
-close(STDOUT) or syserr(_g("couldn't close stdout"));
-
-if (@changedmaint) {
-    warning(_g('Packages in override file with incorrect old maintainer value:'));
-    warning($_) foreach (@changedmaint);
-}
-if (@samemaint) {
-    warning(_g('Packages specifying same maintainer as override file:'));
-    warning($_) foreach (@samemaint);
-}
-if (@missingover) {
-    warning(_g('Packages in archive but missing from override file:'));
-    warning('  %s', join(' ', @missingover));
-}
-if (@spuriousover) {
-    warning(_g('Packages in override file but not in archive:'));
-    warning('  %s', join(' ', @spuriousover));
-}
-
-info(_g('Wrote %s entries to output Packages file.'), $records_written);

util/dpkg.patch

@@ -0,0 +1,49 @@
+From 09efec567e6b0baa835c1921c222a5a967cf193a Mon Sep 17 00:00:00 2001
+From: Dennis Dmitriev <ddmitriev@mirantis.com>
+Date: Tue, 19 Aug 2014 18:31:33 +0300
+Subject: [PATCH] Allow correctly parcing indices/override.* files
+
+Allow parcing indices/override.* files when architecture
+prefixes are used in package names there. It allows to use
+dpkg-scanpackages with Ubuntu repositories.
+
+Originally, dpkg-scanpackages just skip the following lines because it
+compares package names without any modification, and cannot find the
+apropriate package because 'a2jmidid' != 'a2jmidid/amd64'
+For example, several lines from override.precise.extra.main:
+...
+a2jmidid        Bugs    https://bugs.launchpad.net/ubuntu/+filebug
+a2jmidid        Origin  Ubuntu
+a2jmidid/amd64  Task    ubuntustudio-generation, ubuntustudio-recording
+a2jmidid/armel  Task    ubuntustudio-generation, ubuntustudio-recording
+a2jmidid/armhf  Task    ubuntustudio-generation, ubuntustudio-recording
+a2jmidid/i386   Task    ubuntustudio-generation, ubuntustudio-recording
+a2jmidid/powerpc        Task    ubuntustudio-generation, ubuntustudio-recording
+...
+
+Related-Bug: 1358785
+---
+ scripts/dpkg-scanpackages.pl | 7 ++++++-
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/scripts/dpkg-scanpackages.pl b/scripts/dpkg-scanpackages.pl
+index 9a3d59c..3d6724b 100755
+--- a/scripts/dpkg-scanpackages.pl
++++ b/scripts/dpkg-scanpackages.pl
+@@ -143,7 +143,12 @@ sub load_override_extra
+ 	s/\s+$//;
+ 	next unless $_;
+ 
+-	my ($p, $field, $value) = split(/\s+/, $_, 3);
++	my ($pr, $field, $value) = split(/\s+/, $_, 3);
++
++        my ($p, $parch) = split(/\//, $pr, 2);
++        if (defined($options{arch}) and defined($parch)) {
++            next unless ($options{arch} eq $parch);
++        }
+ 
+ 	next unless defined($packages{$p});
+ 
+-- 
+1.8.5.5
+

util/partial_ubuntu.sh

@@ -54,6 +54,8 @@ if ! apt-get $apt_altstate_opts update; then
 	exit 1
 fi
 
+if ! hash wget 2>/dev/null; then apt-get -y install wget; fi
+
 if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/requirements-deb.txt -O $apt_altstate/requirements-deb.txt; then
     echo "`basename $0`: failed to fetch requirements-deb.txt"
     exit 1

util/regenerate_ubuntu_repo

@@ -21,12 +21,12 @@ package_udeb=${BINDIR}/debian-installer/binary-${ARCH}/Packages
 cd ${REPO_PATH}
 
 # Scan *.deb packages
-$BINROOT/util/dpkg-scanpackages  -m --extra-override ${extraoverride} -a ${ARCH} pool/main ${binoverride}  > ${package_deb}.tmp 2>/dev/null
+dpkg-scanpackages  -m --extra-override ${extraoverride} -a ${ARCH} pool/main ${binoverride}  > ${package_deb}.tmp 2>/dev/null
 
 gzip -9c ${package_deb}.tmp > ${package_deb}.gz.tmp
 
 # Scan *.udeb packages
-$BINROOT/util/dpkg-scanpackages -t udeb -m -a ${ARCH} pool/debian-installer ${binoverride_udeb}  > ${package_udeb}.tmp 2>/dev/null
+dpkg-scanpackages -t udeb -m -a ${ARCH} pool/debian-installer ${binoverride_udeb}  > ${package_udeb}.tmp 2>/dev/null
 
 gzip -9c ${package_udeb}.tmp > ${package_udeb}.gz.tmp