Fix apt-sources configuration for debian-minimal

The apt-sources element did not work with debian-minimal, because
the later one overwrote the /etc/apt/sources.list file created by
the apt-sources element.

Two changes were made:
o the debian-minimal uses now files inside the /etc/apt/sources.list.d
  directory.  Therefore there is no possibilty for clashes between those
  two elements any more.
o instead of only adding backports, also the updates and the security
  repository is added by default which gives perfect initial
  configuration for a stable system.
  If you want to use local mirrors with other naming schemas or an
  unstable tree, there is the possibility to fully specify the
  repositories.

Change-Id: I69dbaa34be3db3d667e6bd8450ef4ce04a751c70
Signed-off-by: Andreas Florath <andreas@florath.net>

elements/debian-minimal/README.rst

@@ -2,20 +2,44 @@
 debian-minimal
 ==============
 
-Create a minimal image based on Debian. We default to unstable but DIB_RELEASE
+Create a minimal image based on Debian. We default to unstable but `DIB_RELEASE`
-is mapped to any series of Debian.
+can be set to any series of Debian.
 
-Note that the default Debian series is `unstable`, and the default
+There are two ways to configure apt-sources:
-mirrors for Debian can be problematic for `unstable`. Because apt does
-not handle changing Packages files well across multiple out of sync
-mirrors, it is recommended that you choose a single mirror of debian,
-and pass it in via `DIB_DISTRIBUTION_MIRROR`.
 
-By default only `main` component is used. If `DIB_DEBIAN_COMPONENTS` (comma
+1. Using the standard way of defining the default, backports, updates
-separated) from the `debootstrap` element has been set, that list of
+   and security repositories is the default. In this case you can
-components will be used instead.
+   overwrite the two environment variables to adapt the behavior:
+   `DIB_DISTRIBUTION_MIRROR`: the mirror to use
+      default: http://httpredir.debian.org/debian
+   `DIB_DEBIAN_COMPONENTS`: (default) `main`
+      a comma separated list of components. For Debian this can be
+      e.g. `main,contrib,non-free`.
 
-Backports are included unless `DIB_RELEASE` is `unstable`.
+   Note that the default Debian series is `unstable`, and the default
+   mirrors for Debian can be problematic for `unstable`. Because apt
+   does not handle changing Packages files well across multiple out of
+   sync mirrors, it is recommended that you choose a single mirror of
+   Debian, and pass it in via `DIB_DISTRIBUTION_MIRROR`.
+
+   By default only `main` component is used. If
+   `DIB_DEBIAN_COMPONENTS` (comma separated) from the `debootstrap`
+   element has been set, that list of components will be used instead.
+
+   Backports, updates and security are included unless `DIB_RELEASE`
+   is `unstable`.
+
+2. Complete configuration given in the variable
+   `DIB_APT_SOURCES_CONF`.
+   Each line contains exactly one entry for the sources.list.d
+   directory.
+   The first word must be the logical name (which is used as file name
+   with `.list` automatically appended), followed by a colon `:`,
+   followed by the complete repository specification.
+   Example:
+   DIB_APT_SOURCES_CONF=\
+       "default:deb http://10.0.0.10/ stretch main contrib
+        mysecurity:deb http://10.0.0.10/ stretch-security main contrib"
 
 If necessary, a custom apt keyring and debootstrap script can be
 supplied to the `debootstrap` command via `DIB_APT_KEYRING` and

elements/debian-minimal/environment.d/10-debian-minimal.bash

@@ -1,3 +1,18 @@
 export DISTRO_NAME=debian
 export DIB_RELEASE=${DIB_RELEASE:-unstable}
 export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://httpredir.debian.org/debian}
+export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
+export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ }
+
+DIB_APT_SOURCES_CONF_DEFAULT=\
+"default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}
+backports:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-backports ${DIB_DEBIAN_COMPONENTS_WS}
+updates:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-updates ${DIB_DEBIAN_COMPONENTS_WS}
+security:deb http://security.debian.org/ ${DIB_RELEASE}/updates ${DIB_DEBIAN_COMPONENTS_WS}
+"
+
+if [ "${DIB_RELEASE}" = "unstable" ]; then
+    DIB_APT_SOURCES_CONF_DEFAULT="default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}"
+fi
+
+export DIB_APT_SOURCES_CONF=${DIB_APT_SOURCES_CONF:-${DIB_APT_SOURCES_CONF_DEFAULT}}

elements/debian-minimal/pre-install.d/01-debian-apt-update

@@ -1,42 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
-components=${DIB_DEBIAN_COMPONENTS//,/ }
-
-# We should manage this in a betterer way
-cat << EOF >/etc/apt/sources.list
-deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE $components
-EOF
-
-if [ $DIB_RELEASE != unstable ] ; then
-    cat << EOF >>/etc/apt/sources.list
-deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports $components
-EOF
-fi
-
-# Need to update to retrieve the signed Release file
-apt-get update
-
-apt-get clean
-apt-get dist-upgrade -y

elements/debian-minimal/pre-install.d/02-debian-apt-update

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# Copyright (c) 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# The filename needs to be 02-...: because the install-package script
+# is installed in the dpkg/pre-install/01-dpkg and that has to be executed
+# first.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Writes the apt sources files.
+# The description is passed in via line coded elements.
+# (The approach using associative arrays for configuration faild,
+#  because it looks that there is no way to handle defaults in
+#  this case - and additionally we run with '-u'.)
+function apt_sources_write {
+    local APT_SOURCES_CONF="$1"
+
+    mkdir -p /etc/apt/sources.list.d
+
+    echo "${APT_SOURCES_CONF}" \
+        | while read line; do
+            local name=$(echo ${line} | cut -d ":" -f 1)
+            local value=$(echo ${line} | cut -d ":" -f 2-)
+            echo "$value" >>/etc/apt/sources.list.d/${name}.list
+        done
+}
+
+apt_sources_write "${DIB_APT_SOURCES_CONF}"
+
+# Need to update to retrieve the signed Release file
+apt-get update
+
+apt-get clean
+install-packages -u

elements/debian/test-elements/build-succeeds/environment.d/09-debian-stable

@@ -0,0 +1,4 @@
+# We don't want to build against Debian unstable
+# (Needs to be a 09- because must be executed before
+# debian-minimal 10-debian-minimal.bash.)
+export DIB_RELEASE="stable"

elements/debian/test-elements/build-succeeds/environment.d/10-debian-stable

@@ -1,2 +0,0 @@
-# We don't want to build against Debian unstable
-export DIB_RELEASE="stable"

elements/debootstrap/root.d/08-debootstrap

@@ -71,6 +71,12 @@ else
         $DIB_DISTRIBUTION_MIRROR \
         ${DIB_DEBIAN_DEBOOTSTRAP_SCRIPT:-}"
 
+    # debootstrap creates sometimes a not-usable
+    # /etc/apt/sources.list: DIB wants to set up its own anyway
+    # and the old has to go.
+    sudo rm -fr ${TARGET_ROOT}/etc/apt/sources.list \
+        ${TARGET_ROOT}/etc/apt/sources.list.d
+
     echo Caching debootstrap result in $DEBOOTSTRAP_TARBALL
     if [ "${DIB_DEBOOTSTRAP_CACHE:-0}" != "0" ]; then
         sudo tar --numeric-owner -C $TARGET_ROOT -zcf $DEBOOTSTRAP_TARBALL --exclude='./tmp/*' .