Tested on Fuel node.
This commit is contained in:
Vitaly Parakhin
parent
cb06b7a82d
commit
6103b51c7a
|
|
@ -6,16 +6,21 @@ LOG_ROOT="/var/log/mirror-sync"
|
|||
|
||||
# Attempt to autodetect current Fuel version
|
||||
if hash fuel 2>/dev/null; then
|
||||
# working on Fuel master node
|
||||
# working on Fuel node
|
||||
FUEL_VERSION=`fuel --fuel-version 2>&1 | awk -v sq="'" '/^release:/ { gsub(sq,""); print $2 }'`
|
||||
if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_VERSION}/config.mk -O /dev/null; then
|
||||
FUEL_BRANCH=master
|
||||
else
|
||||
FUEL_BRANCH=$FUEL_VERSION
|
||||
fi
|
||||
# On Fuel node we will be using dockerized Ubuntu for dependencies resolving.
|
||||
# Assuming that Docker is up and running.
|
||||
DOCKER_MODE=true
|
||||
else
|
||||
# working outside Fuel master node
|
||||
echo "Cannot detect current Fuel version, using defaults. Please configure settings in config/common.cfg"
|
||||
echo "Cannot detect current Fuel version, using defaults. Please configure FUEL_* settings in config/common.cfg"
|
||||
sleep 3
|
||||
FUEL_VERSION=6.1
|
||||
FUEL_BRANCH=master
|
||||
DOCKER_MODE=false
|
||||
fi
|
||||
|
|
@ -2,8 +2,8 @@
|
|||
source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg"
|
||||
|
||||
# Sync source
|
||||
UPSTREAM="osci-mirror-kha.kha.mirantis.net"
|
||||
UPSTREAM_DIR="mirror-sync/mos/ubuntu"
|
||||
UPSTREAM="mirror.fuel-infra.org"
|
||||
UPSTREAM_DIR="mirror/mos/ubuntu"
|
||||
UPSTREAM_DIR_HTTP="mos/ubuntu"
|
||||
|
||||
# NOTE! Partial repository sync feature requires apt support on your OS!
|
||||
|
|
@ -35,7 +35,7 @@ FETCH_DIFF="no" # Fetch diffs of 'Packages' (speed's up client update)
|
|||
FETCH_INDICES="no" # Fetch indices
|
||||
|
||||
# Misc
|
||||
DEBUG="no"
|
||||
DEBUG="yes"
|
||||
QUIET="no"
|
||||
DISABLE_CLEANUP="no"
|
||||
LOG_FILE="${LOG_ROOT:-"/var/log/mirror-sync"}/ubuntu-update.log"
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg"
|
||||
|
||||
# Sync source
|
||||
UPSTREAM="osci-mirror-kha.kha.mirantis.net"
|
||||
UPSTREAM_DIR="mirror-sync/mos/ubuntu"
|
||||
UPSTREAM="mirror.fuel-infra.org"
|
||||
UPSTREAM_DIR="mirror/mos/ubuntu"
|
||||
UPSTREAM_DIR_HTTP="mos/ubuntu"
|
||||
|
||||
# NOTE! Partial repository sync feature requires apt support on your OS!
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ FETCH_DIFF="no" # Fetch diffs of 'Packages' (speed's up client update)
|
|||
FETCH_INDICES="yes" # Fetch indices
|
||||
|
||||
# Misc
|
||||
DEBUG="no"
|
||||
DEBUG="yes"
|
||||
QUIET="no"
|
||||
DISABLE_CLEANUP="no"
|
||||
LOG_FILE="${LOG_ROOT:-"/var/log/mirror-sync"}/ubuntu-update.log"
|
||||
|
|
|
|||
15
deb-mirror
15
deb-mirror
|
|
@ -451,8 +451,6 @@ if [[ "$PARTIAL_UPSTREAM" = "1" ]]; then
|
|||
|
||||
info "Resolving dependencies for partial mirror"
|
||||
|
||||
[ -f /usr/bin/apt-get ] || fatal "APT not found on this system, creation of partial repository will fail"
|
||||
|
||||
# Detect kernel version of debian-installer
|
||||
fuel_config_temp=`mktemp --suffix="-fuel-config-temp"`
|
||||
if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/config.mk -O $fuel_config_temp; then
|
||||
|
|
@ -473,7 +471,14 @@ rm -f "$fuel_config_temp"
|
|||
export apt_altstate=`mktemp -d --suffix="-apt-altstate"`
|
||||
export FUEL_BRANCH
|
||||
|
||||
$BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
|
||||
if [[ "$DOCKER_MODE" = "true" ]]; then
|
||||
# docker pull ubuntu:latest
|
||||
docker -D run --net=bridge --rm -a stdout -a stderr -i -t --privileged -e FUEL_BRANCH=$FUEL_BRANCH -e BINROOT=$BINROOT -e apt_altstate=$apt_altstate -v $BINROOT:$BINROOT:rw -v $apt_altstate:$apt_altstate:rw ubuntu:latest $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
|
||||
# cleanup ubuntu container
|
||||
docker images | awk '/ubuntu/ { print $3; }' | xargs --no-run-if-empty docker rmi -f
|
||||
else
|
||||
$BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
|
||||
fi
|
||||
|
||||
# Create download lists for deb and udeb
|
||||
awk 'FNR==NR {arr[$0];next} $3 in arr' $apt_altstate/deb "$deb_md5" > $apt_altstate/deb_md5
|
||||
|
|
@ -605,8 +610,10 @@ EOF
|
|||
|
||||
# Build partial mirror
|
||||
info "Generating metadata for partial mirror"
|
||||
info "Apply fix for upstream dpkg-scanpackages"
|
||||
patch -N /usr/bin/dpkg-scanpackages < $BINROOT/util/dpkg.patch
|
||||
export BINROOT
|
||||
$BINROOT/util/regenerate_ubuntu_repo ${PARTIAL_UPSTREAM_PATH} ${UBUNTU_RELEASE} || failure "Failed to generate partial mirror"
|
||||
$BINROOT/util/regenerate_ubuntu_repo ${PARTIAL_UPSTREAM_PATH} ${UBUNTU_RELEASE} || fatal "Failed to generate partial mirror"
|
||||
rm -rf $LOCAL_DIR
|
||||
fi # "$PARTIAL_UPSTREAM" = "1"
|
||||
info "Done"
|
||||
|
|
|
|||
|
|
@ -1,289 +0,0 @@
|
|||
#!/usr/bin/perl
|
||||
#
|
||||
# dpkg-scanpackages
|
||||
#
|
||||
# Copyright © 2006-2012 Guillem Jover <guillem@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
use warnings;
|
||||
use strict;
|
||||
|
||||
use IO::Handle;
|
||||
use IO::File;
|
||||
use Getopt::Long qw(:config posix_default bundling no_ignorecase);
|
||||
|
||||
use Dpkg ();
|
||||
use Dpkg::Gettext;
|
||||
use Dpkg::ErrorHandling;
|
||||
#use Dpkg::Util qw(:list);
|
||||
use Dpkg::Control;
|
||||
use Dpkg::Version;
|
||||
use Dpkg::Checksums;
|
||||
use Dpkg::Compression::FileHandle;
|
||||
use Dpkg::IPC;
|
||||
|
||||
textdomain('dpkg-dev');
|
||||
|
||||
# Do not pollute STDOUT with info messages
|
||||
report_options(info_fh => \*STDERR);
|
||||
|
||||
my (@samemaint, @changedmaint);
|
||||
my @spuriousover;
|
||||
my %packages;
|
||||
my %overridden;
|
||||
|
||||
my %options = (help => sub { usage(); exit 0; },
|
||||
version => \&version,
|
||||
type => undef,
|
||||
arch => undef,
|
||||
multiversion => 0,
|
||||
'extra-override'=> undef,
|
||||
medium => undef,
|
||||
);
|
||||
|
||||
my @options_spec = (
|
||||
'help|?',
|
||||
'version',
|
||||
'type|t=s',
|
||||
'arch|a=s',
|
||||
'multiversion|m!',
|
||||
'extra-override|e=s',
|
||||
'medium|M=s',
|
||||
);
|
||||
|
||||
sub version {
|
||||
printf _g("Debian %s version.\n"), $Dpkg::PROGNAME;
|
||||
exit;
|
||||
}
|
||||
|
||||
sub usage {
|
||||
printf _g(
|
||||
"Usage: %s [<option>...] <binary-path> [<override-file> [<path-prefix>]] > Packages
|
||||
|
||||
Options:
|
||||
-t, --type <type> scan for <type> packages (default is 'deb').
|
||||
-a, --arch <arch> architecture to scan for.
|
||||
-m, --multiversion allow multiple versions of a single package.
|
||||
-e, --extra-override <file>
|
||||
use extra override file.
|
||||
-M, --medium <medium> add X-Medium field for dselect multicd access method
|
||||
-?, --help show this help message.
|
||||
--version show the version.
|
||||
"), $Dpkg::PROGNAME;
|
||||
}
|
||||
|
||||
sub load_override
|
||||
{
|
||||
my $override = shift;
|
||||
my $comp_file = Dpkg::Compression::FileHandle->new(filename => $override);
|
||||
|
||||
while (<$comp_file>) {
|
||||
s/\#.*//;
|
||||
s/\s+$//;
|
||||
next unless $_;
|
||||
|
||||
my ($p, $priority, $section, $maintainer) = split(/\s+/, $_, 4);
|
||||
|
||||
if (not defined($packages{$p})) {
|
||||
push(@spuriousover, $p);
|
||||
next;
|
||||
}
|
||||
|
||||
for my $package (@{$packages{$p}}) {
|
||||
if ($maintainer) {
|
||||
if ($maintainer =~ m/(.+?)\s*=\>\s*(.+)/) {
|
||||
my $oldmaint = $1;
|
||||
my $newmaint = $2;
|
||||
my $debmaint = $$package{Maintainer};
|
||||
if (none { $debmaint eq $_ } split m{\s*//\s*}, $oldmaint) {
|
||||
push(@changedmaint,
|
||||
sprintf(_g(' %s (package says %s, not %s)'),
|
||||
$p, $$package{Maintainer}, $oldmaint));
|
||||
} else {
|
||||
$$package{Maintainer} = $newmaint;
|
||||
}
|
||||
} elsif ($$package{Maintainer} eq $maintainer) {
|
||||
push(@samemaint, " $p ($maintainer)");
|
||||
} else {
|
||||
warning(_g('unconditional maintainer override for %s'), $p);
|
||||
$$package{Maintainer} = $maintainer;
|
||||
}
|
||||
}
|
||||
$$package{Priority} = $priority;
|
||||
$$package{Section} = $section;
|
||||
}
|
||||
$overridden{$p} = 1;
|
||||
}
|
||||
|
||||
close($comp_file);
|
||||
}
|
||||
|
||||
sub load_override_extra
|
||||
{
|
||||
my $extra_override = shift;
|
||||
my $comp_file = Dpkg::Compression::FileHandle->new(filename => $extra_override);
|
||||
|
||||
while (<$comp_file>) {
|
||||
s/\#.*//;
|
||||
s/\s+$//;
|
||||
next unless $_;
|
||||
|
||||
my ($pr, $field, $value) = split(/\s+/, $_, 3);
|
||||
|
||||
my ($p, $parch) = split(/\//, $pr, 2);
|
||||
if (defined($options{arch}) and defined($parch)) {
|
||||
next unless ($options{arch} eq $parch);
|
||||
}
|
||||
|
||||
next unless defined($packages{$p});
|
||||
|
||||
for my $package (@{$packages{$p}}) {
|
||||
$$package{$field} = $value;
|
||||
}
|
||||
}
|
||||
|
||||
close($comp_file);
|
||||
}
|
||||
|
||||
{
|
||||
local $SIG{__WARN__} = sub { usageerr($_[0]) };
|
||||
GetOptions(\%options, @options_spec);
|
||||
}
|
||||
|
||||
if (not (@ARGV >= 1 and @ARGV <= 3)) {
|
||||
usageerr(_g('one to three arguments expected'));
|
||||
}
|
||||
|
||||
my $type = defined($options{type}) ? $options{type} : 'deb';
|
||||
my $arch = $options{arch};
|
||||
|
||||
my @find_args;
|
||||
if ($options{arch}) {
|
||||
@find_args = ('(', '-name', "*_all.$type", '-o',
|
||||
'-name', "*_${arch}.$type", ')');
|
||||
}
|
||||
else {
|
||||
@find_args = ('-name', "*.$type");
|
||||
}
|
||||
|
||||
my ($binarydir, $override, $pathprefix) = @ARGV;
|
||||
|
||||
if (not -d $binarydir) {
|
||||
error(_g('binary dir %s not found'), $binarydir);
|
||||
}
|
||||
if (defined $override and not -e $override) {
|
||||
error(_g('override file %s not found'), $override);
|
||||
}
|
||||
|
||||
$pathprefix //= '';
|
||||
|
||||
my $find_h = IO::Handle->new();
|
||||
open($find_h, '-|', 'find', '-L', "$binarydir/", @find_args, '-print')
|
||||
or syserr(_g("couldn't open %s for reading"), $binarydir);
|
||||
FILE:
|
||||
while (<$find_h>) {
|
||||
chomp;
|
||||
my $fn = $_;
|
||||
my $output;
|
||||
my $pid = spawn(exec => [ 'dpkg-deb', '-I', $fn, 'control' ],
|
||||
to_pipe => \$output);
|
||||
my $fields = Dpkg::Control->new(type => CTRL_INDEX_PKG);
|
||||
$fields->parse($output, $fn)
|
||||
or error(_g("couldn't parse control information from %s"), $fn);
|
||||
wait_child($pid, no_check => 1);
|
||||
if ($?) {
|
||||
warning(_g("\`dpkg-deb -I %s control' exited with %d, skipping package"),
|
||||
$fn, $?);
|
||||
next;
|
||||
}
|
||||
|
||||
defined($fields->{'Package'})
|
||||
or error(_g('no Package field in control file of %s'), $fn);
|
||||
my $p = $fields->{'Package'};
|
||||
|
||||
if (defined($packages{$p}) and not $options{multiversion}) {
|
||||
foreach (@{$packages{$p}}) {
|
||||
if (version_compare_relation($fields->{'Version'}, REL_GT,
|
||||
$_->{'Version'}))
|
||||
{
|
||||
warning(_g('package %s (filename %s) is repeat but newer version;'),
|
||||
$p, $fn);
|
||||
warning(_g('used that one and ignored data from %s!'),
|
||||
$_->{Filename});
|
||||
$packages{$p} = [];
|
||||
} else {
|
||||
warning(_g('package %s (filename %s) is repeat;'), $p, $fn);
|
||||
warning(_g('ignored that one and using data from %s!'),
|
||||
$_->{Filename});
|
||||
next FILE;
|
||||
}
|
||||
}
|
||||
}
|
||||
warning(_g('package %s (filename %s) has Filename field!'), $p, $fn)
|
||||
if defined($fields->{'Filename'});
|
||||
|
||||
$fields->{'Filename'} = "$pathprefix$fn";
|
||||
|
||||
my $sums = Dpkg::Checksums->new();
|
||||
$sums->add_from_file($fn);
|
||||
foreach my $alg (checksums_get_list()) {
|
||||
if ($alg eq 'md5') {
|
||||
$fields->{'MD5sum'} = $sums->get_checksum($fn, $alg);
|
||||
} else {
|
||||
$fields->{$alg} = $sums->get_checksum($fn, $alg);
|
||||
}
|
||||
}
|
||||
$fields->{'Size'} = $sums->get_size($fn);
|
||||
$fields->{'X-Medium'} = $options{medium} if defined $options{medium};
|
||||
|
||||
push @{$packages{$p}}, $fields;
|
||||
}
|
||||
close($find_h);
|
||||
|
||||
load_override($override) if defined $override;
|
||||
load_override_extra($options{'extra-override'}) if defined $options{'extra-override'};
|
||||
|
||||
my @missingover=();
|
||||
|
||||
my $records_written = 0;
|
||||
for my $p (sort keys %packages) {
|
||||
if (defined($override) and not defined($overridden{$p})) {
|
||||
push(@missingover,$p);
|
||||
}
|
||||
for my $package (@{$packages{$p}}) {
|
||||
print("$package\n") or syserr(_g('failed when writing stdout'));
|
||||
$records_written++;
|
||||
}
|
||||
}
|
||||
close(STDOUT) or syserr(_g("couldn't close stdout"));
|
||||
|
||||
if (@changedmaint) {
|
||||
warning(_g('Packages in override file with incorrect old maintainer value:'));
|
||||
warning($_) foreach (@changedmaint);
|
||||
}
|
||||
if (@samemaint) {
|
||||
warning(_g('Packages specifying same maintainer as override file:'));
|
||||
warning($_) foreach (@samemaint);
|
||||
}
|
||||
if (@missingover) {
|
||||
warning(_g('Packages in archive but missing from override file:'));
|
||||
warning(' %s', join(' ', @missingover));
|
||||
}
|
||||
if (@spuriousover) {
|
||||
warning(_g('Packages in override file but not in archive:'));
|
||||
warning(' %s', join(' ', @spuriousover));
|
||||
}
|
||||
|
||||
info(_g('Wrote %s entries to output Packages file.'), $records_written);
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
From 09efec567e6b0baa835c1921c222a5a967cf193a Mon Sep 17 00:00:00 2001
|
||||
From: Dennis Dmitriev <ddmitriev@mirantis.com>
|
||||
Date: Tue, 19 Aug 2014 18:31:33 +0300
|
||||
Subject: [PATCH] Allow correctly parcing indices/override.* files
|
||||
|
||||
Allow parcing indices/override.* files when architecture
|
||||
prefixes are used in package names there. It allows to use
|
||||
dpkg-scanpackages with Ubuntu repositories.
|
||||
|
||||
Originally, dpkg-scanpackages just skip the following lines because it
|
||||
compares package names without any modification, and cannot find the
|
||||
apropriate package because 'a2jmidid' != 'a2jmidid/amd64'
|
||||
For example, several lines from override.precise.extra.main:
|
||||
...
|
||||
a2jmidid Bugs https://bugs.launchpad.net/ubuntu/+filebug
|
||||
a2jmidid Origin Ubuntu
|
||||
a2jmidid/amd64 Task ubuntustudio-generation, ubuntustudio-recording
|
||||
a2jmidid/armel Task ubuntustudio-generation, ubuntustudio-recording
|
||||
a2jmidid/armhf Task ubuntustudio-generation, ubuntustudio-recording
|
||||
a2jmidid/i386 Task ubuntustudio-generation, ubuntustudio-recording
|
||||
a2jmidid/powerpc Task ubuntustudio-generation, ubuntustudio-recording
|
||||
...
|
||||
|
||||
Related-Bug: 1358785
|
||||
---
|
||||
scripts/dpkg-scanpackages.pl | 7 ++++++-
|
||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/scripts/dpkg-scanpackages.pl b/scripts/dpkg-scanpackages.pl
|
||||
index 9a3d59c..3d6724b 100755
|
||||
--- a/scripts/dpkg-scanpackages.pl
|
||||
+++ b/scripts/dpkg-scanpackages.pl
|
||||
@@ -143,7 +143,12 @@ sub load_override_extra
|
||||
s/\s+$//;
|
||||
next unless $_;
|
||||
|
||||
- my ($p, $field, $value) = split(/\s+/, $_, 3);
|
||||
+ my ($pr, $field, $value) = split(/\s+/, $_, 3);
|
||||
+
|
||||
+ my ($p, $parch) = split(/\//, $pr, 2);
|
||||
+ if (defined($options{arch}) and defined($parch)) {
|
||||
+ next unless ($options{arch} eq $parch);
|
||||
+ }
|
||||
|
||||
next unless defined($packages{$p});
|
||||
|
||||
--
|
||||
1.8.5.5
|
||||
|
||||
|
|
@ -54,6 +54,8 @@ if ! apt-get $apt_altstate_opts update; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if ! hash wget 2>/dev/null; then apt-get -y install wget; fi
|
||||
|
||||
if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/requirements-deb.txt -O $apt_altstate/requirements-deb.txt; then
|
||||
echo "`basename $0`: failed to fetch requirements-deb.txt"
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -21,12 +21,12 @@ package_udeb=${BINDIR}/debian-installer/binary-${ARCH}/Packages
|
|||
cd ${REPO_PATH}
|
||||
|
||||
# Scan *.deb packages
|
||||
$BINROOT/util/dpkg-scanpackages -m --extra-override ${extraoverride} -a ${ARCH} pool/main ${binoverride} > ${package_deb}.tmp 2>/dev/null
|
||||
dpkg-scanpackages -m --extra-override ${extraoverride} -a ${ARCH} pool/main ${binoverride} > ${package_deb}.tmp 2>/dev/null
|
||||
|
||||
gzip -9c ${package_deb}.tmp > ${package_deb}.gz.tmp
|
||||
|
||||
# Scan *.udeb packages
|
||||
$BINROOT/util/dpkg-scanpackages -t udeb -m -a ${ARCH} pool/debian-installer ${binoverride_udeb} > ${package_udeb}.tmp 2>/dev/null
|
||||
dpkg-scanpackages -t udeb -m -a ${ARCH} pool/debian-installer ${binoverride_udeb} > ${package_udeb}.tmp 2>/dev/null
|
||||
|
||||
gzip -9c ${package_udeb}.tmp > ${package_udeb}.gz.tmp
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue