Merge "Modify mini-mirror to build from yaml"
This commit is contained in:
Zuul
Gerrit Code Review
commit
980be240ec
|
|
@ -15,12 +15,13 @@
|
|||
FROM ubuntu:18.04 as aptly
|
||||
|
||||
ARG APTLY_CONFIG_PATH=etc/aptly.conf
|
||||
ARG MIRROR_SOURCE_DIR=sources
|
||||
ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
|
||||
ARG RELEASE_SIGN_KEY_PATH=etc
|
||||
ARG RELEASE_SIGN_KEY_PASSPHRASE
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y gnupg wget
|
||||
RUN apt-get install -y gnupg wget jq python-pip
|
||||
RUN pip install yq
|
||||
|
||||
COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
|
||||
COPY tools/install_aptly.sh /opt/install_aptly.sh
|
||||
|
|
@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
|
|||
|
||||
RUN /opt/install_aptly.sh
|
||||
|
||||
COPY "${MIRROR_SOURCE_DIR}" /opt/sources
|
||||
COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
|
||||
COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
|
||||
COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,13 @@
|
|||
FROM ubuntu:16.04 as aptly
|
||||
|
||||
ARG APTLY_CONFIG_PATH=etc/aptly.conf
|
||||
ARG MIRROR_SOURCE_DIR=sources
|
||||
ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
|
||||
ARG RELEASE_SIGN_KEY_PATH=etc
|
||||
ARG RELEASE_SIGN_KEY_PASSPHRASE
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y wget
|
||||
RUN apt-get install -y wget jq python-pip
|
||||
RUN pip install yq
|
||||
|
||||
COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
|
||||
COPY tools/install_aptly.sh /opt/install_aptly.sh
|
||||
|
|
@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
|
|||
|
||||
RUN /opt/install_aptly.sh
|
||||
|
||||
COPY "${MIRROR_SOURCE_DIR}" /opt/sources
|
||||
COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
|
||||
COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
|
||||
COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -11,50 +11,39 @@ Build Requirements
|
|||
Add mirror sources
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Mini-mirror requires a directory at build-time that contains the repositories
|
||||
and packages that will be mirrored.
|
||||
Mini-mirror requires a YAML file at build-time that contains the repositories
|
||||
and packages that will be mirrored as different YAML documents.
|
||||
|
||||
.. code::
|
||||
.. code:: yaml
|
||||
---
|
||||
name: <Repository name (i.e. the directory a source serves from)>
|
||||
url: <URL link to the source repository>
|
||||
key_url: <URL link to the key for the source repository>
|
||||
codename: *<Override codename for the release file>
|
||||
label: *<Override label for the release file>
|
||||
aptly_config: | # *Inline aptly config JSON file to replace default
|
||||
{ }
|
||||
components: # List of Components
|
||||
- <Component>
|
||||
subrepos: # List of repositories within the source repository
|
||||
- distribution: <Distribution name of the repository>
|
||||
packages: # <List of all packages>
|
||||
- name: <Package name>
|
||||
version: *<Version of package to pin to>
|
||||
...
|
||||
---
|
||||
# Additional repository document here
|
||||
...
|
||||
|
||||
sources/
|
||||
| -- source1-prefix/
|
||||
|-- source-name/
|
||||
|-- source.txt
|
||||
|-- packages.txt
|
||||
| -- source2-prefix/
|
||||
|-- source-name/
|
||||
|-- source.txt
|
||||
|-- packages.txt
|
||||
*Optional
|
||||
|
||||
Sources are defined as directories containing the files:
|
||||
|
||||
* source-prefix - a prefix to separate sources that have conflicting
|
||||
distribution names (i.e. the directory a source serves from).
|
||||
* source-name - the name of a source; used for record-keeping.
|
||||
* source.txt - contains location and metadata information for a source.
|
||||
* packages.txt - contains a list of packages, formatted as `package queries <https://www.aptly.info/doc/feature/query/>`_
|
||||
for a source.
|
||||
|
||||
Example ``source.txt`` format:
|
||||
|
||||
.. code::
|
||||
|
||||
source_url source_key_url dist components
|
||||
|
||||
Example ``packages.txt`` format:
|
||||
|
||||
.. code::
|
||||
|
||||
package1
|
||||
package2
|
||||
package3 (>=3.6)
|
||||
|
||||
To specify the location of your sources directory, export the following
|
||||
To specify the location of your sources YAML file, export the following
|
||||
environment variable:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
export MIRROR_SOURCE_DIR=/path/to/sources
|
||||
export MIRROR_SOURCE_FILE=/path/to/sources.yaml
|
||||
|
||||
Generate a signing key
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -92,8 +81,8 @@ environment variable:
|
|||
.. NOTE::
|
||||
|
||||
Mini-mirror can be configured on a per-repo basis by adding an Aptly config
|
||||
file to the root directory of a source. This overrides the Aptly config
|
||||
file taken from ``APTLY_CONFIG_PATH``.
|
||||
file to the .aptly_config key in the YAML document. This overrides
|
||||
the Aptly config file taken from ``APTLY_CONFIG_PATH``.
|
||||
|
||||
Proxy
|
||||
~~~~~
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ SCRIPT_DIR=$(dirname "${SCRIPT}")
|
|||
## Only build from main folder
|
||||
cd "${SCRIPT_DIR}"/.. || exit
|
||||
|
||||
IMAGE="mini-mirror"
|
||||
PROJECT_PATH="mini-mirror"
|
||||
IMAGE=${IMAGE:-mini-mirror}
|
||||
VERSION=${VERSION:-latest}
|
||||
DISTRO=${DISTRO:-ubuntu_xenial}
|
||||
REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"}
|
||||
|
|
@ -31,7 +32,7 @@ HTTPS_PROXY=${HTTPS_PROXY:-""}
|
|||
NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"}
|
||||
|
||||
APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"}
|
||||
MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"}
|
||||
MIRROR_SOURCE_FILE=${MIRROR_SOURCE_FILE:-"mini-mirror-sources.yaml"}
|
||||
RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"}
|
||||
RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""}
|
||||
|
||||
|
|
@ -42,7 +43,7 @@ APTLY_INSTALL_FROM=${APTLY_INSTALL_FROM:-"source"}
|
|||
APTLY_REPO=${APTLY_REPO:-"https://github.com/smstone/aptly.git"}
|
||||
APTLY_REFSPEC=${APTLY_REFSPEC:-"allow-custom-codename"}
|
||||
|
||||
docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
|
||||
docker build -f "${PROJECT_PATH}"/Dockerfile."${DISTRO}" --network=host \
|
||||
-t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \
|
||||
--build-arg http_proxy="${HTTP_PROXY}" \
|
||||
--build-arg https_proxy="${HTTPS_PROXY}" \
|
||||
|
|
@ -51,12 +52,12 @@ docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
|
|||
--build-arg no_proxy="${NO_PROXY}" \
|
||||
--build-arg NO_PROXY="${NO_PROXY}" \
|
||||
--build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \
|
||||
--build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \
|
||||
--build-arg MIRROR_SOURCE_FILE="${MIRROR_SOURCE_FILE}" \
|
||||
--build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \
|
||||
--build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \
|
||||
--build-arg APTLY_INSTALL_FROM="${APTLY_INSTALL_FROM}" \
|
||||
--build-arg APTLY_REPO="${APTLY_REPO}" \
|
||||
--build-arg APTLY_REFSPEC="${APTLY_REFSPEC}" \
|
||||
${extra_build_args} "${IMAGE}"
|
||||
${extra_build_args} "${PROJECT_PATH}"
|
||||
|
||||
cd - || exit
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
"rootDir": "/opt/.aptly",
|
||||
"downloadConcurrency": 4,
|
||||
"downloadSpeedLimit": 0,
|
||||
"architectures": [],
|
||||
"architectures": ["amd64"],
|
||||
"dependencyFollowSuggests": false,
|
||||
"dependencyFollowRecommends": false,
|
||||
"dependencyFollowAllVariants": false,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
---
|
||||
name: aptly-example
|
||||
url: http://repo.aptly.info/
|
||||
key_url: https://www.aptly.info/pubkey.txt
|
||||
components:
|
||||
- main
|
||||
subrepos:
|
||||
- distribution: squeeze
|
||||
packages:
|
||||
- name: aptly
|
||||
...
|
||||
# Example additional config for adding docker
|
||||
#---
|
||||
#name: docker
|
||||
#url: https://download.docker.com/linux/ubuntu
|
||||
#key_url: https://download.docker.com/linux/ubuntu/gpg
|
||||
#aptly_config: |
|
||||
# {
|
||||
# "rootDir": "/opt/.aptly",
|
||||
# "downloadConcurrency": 20,
|
||||
# "downloadSpeedLimit": 0,
|
||||
# "architectures": ["amd64"],
|
||||
# "dependencyFollowSuggests": false,
|
||||
# "dependencyFollowRecommends": false,
|
||||
# "dependencyFollowAllVariants": false,
|
||||
# "dependencyFollowSource": false,
|
||||
# "dependencyVerboseResolve": true,
|
||||
# "gpgDisableSign": true,
|
||||
# "gpgDisableVerify": true,
|
||||
# "gpgProvider": "gpg",
|
||||
# "downloadSourcePackages": false,
|
||||
# "skipLegacyPool": true,
|
||||
# "ppaDistributorID": "ubuntu",
|
||||
# "ppaCodename": ""
|
||||
# }
|
||||
#components:
|
||||
# - stable
|
||||
#subrepos:
|
||||
# - distribution: xenial
|
||||
# packages:
|
||||
# - name: docker-ce
|
||||
# version: 17.03.3~ce-0~ubuntu-xenial_amd64
|
||||
#...
|
||||
|
|
@ -1 +0,0 @@
|
|||
aptly
|
||||
|
|
@ -1 +0,0 @@
|
|||
http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main
|
||||
|
|
@ -14,23 +14,42 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -ex
|
||||
|
||||
if [ ! -z "$1" ]; then
|
||||
if [[ ! -z "$1" ]]; then
|
||||
gpg --import /opt/release.gpg
|
||||
fi
|
||||
|
||||
for source_prefix in /opt/sources/*; do
|
||||
for source in $source_prefix/*; do
|
||||
read -r -a info < "${source}"/source.txt
|
||||
repo=${info[0]}
|
||||
key=${info[1]}
|
||||
dist=${info[2]}
|
||||
components=${info[*]:3}
|
||||
sources=$(yq "." /opt/mini-mirror-sources.yaml | jq -s '.')
|
||||
|
||||
# Loop to iterate over each document in the YAML file.
|
||||
# By base64 encoding and then decoding the output from jq we are able to
|
||||
# cleanly iterate over the output even if it contains newlines, etc.
|
||||
for source in $(echo "${sources}" | jq -r '.[] | @base64' ); do
|
||||
_source() {
|
||||
echo "${source}" | base64 --decode | jq -r "${*}"
|
||||
}
|
||||
|
||||
source_name=$(_source '.name')
|
||||
repo=$(_source '.url')
|
||||
key=$(_source '.key_url')
|
||||
components=$(_source '.components')
|
||||
label=$(_source '.label')
|
||||
codename=$(_source '.code_name')
|
||||
|
||||
# Loop to iterate over the `subrepo` list in the document
|
||||
for subrepo in $(_source '.subrepos[] | @base64'); do
|
||||
_subrepo() {
|
||||
echo "${subrepo}" | base64 --decode | jq -r "${*}"
|
||||
}
|
||||
|
||||
dist=$(_subrepo '.distribution')
|
||||
|
||||
# Use source specific aptly config when provided
|
||||
if [ -f "${source}"/aptly.conf ]; then
|
||||
conf="${source}"/aptly.conf
|
||||
source_conf=$(_source '.aptly_config')
|
||||
if [[ "$source_conf" != "null" ]]; then
|
||||
echo "${source_conf}" > aptly.conf
|
||||
conf=$(pwd)/aptly.conf
|
||||
else
|
||||
conf=/etc/aptly.conf
|
||||
fi
|
||||
|
|
@ -38,11 +57,21 @@ for source_prefix in /opt/sources/*; do
|
|||
# Create package query from well-defined package list.
|
||||
#
|
||||
# package1
|
||||
# package2 ==> package1 | package2 | package3
|
||||
# package2 ==> package1 | package2 (=1.0) | package3
|
||||
# package3
|
||||
#
|
||||
packages=$(awk -v ORS=" | " '{ print $1 }' "${source}"/packages.txt)
|
||||
packages="${packages::-3}"
|
||||
|
||||
# Grab packages from .subrepo
|
||||
packages=$(_subrepo '.packages')
|
||||
# Convert any found versions to strings
|
||||
str_versions=$(echo "${packages}" | jq ' .[] | if .version != null then {name: .name, version: .version | tostring} else {name: .name} end')
|
||||
# Format packages <pkg> and versions <ver> to "<pkg> (=@<ver>"
|
||||
formatted_packages=$(echo "${str_versions}" | jq -r '. | join(" (=@")')
|
||||
# Substitute "@<ver>" with "<ver>)" so the new format is "<pkg> (=<ver>)"
|
||||
# and bring the packages on to one line separated by "@"
|
||||
wrap_versions=$(echo "${formatted_packages}" | sed -r "s/@(.*)/\1\)/g" | tr "\n" "@")
|
||||
# Substitute the "@" between packages with " | "
|
||||
package_query=$(echo "${wrap_versions}" | sed -r "s/@/ \| /g" | sed -r "s/ \| $//g")
|
||||
|
||||
# Import source key
|
||||
wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \
|
||||
|
|
@ -51,13 +80,14 @@ for source_prefix in /opt/sources/*; do
|
|||
# Create a mirror of each component from a source's repository, update it,
|
||||
# and publish a snapshot of it.
|
||||
mirrors=()
|
||||
for component in $components; do
|
||||
name="${source}-${component}"
|
||||
# Loop to iterate over the `component` list in the document
|
||||
for component in $(echo "${components}" | jq -r '.[]' ); do
|
||||
name="${source_name}-${dist}-${component}"
|
||||
mirrors+=("$name")
|
||||
|
||||
aptly mirror create \
|
||||
-config="${conf}" \
|
||||
-filter="${packages}" \
|
||||
-filter="${package_query}" \
|
||||
-filter-with-deps \
|
||||
"${name}" "${repo}" "${dist}" "${component}"
|
||||
|
||||
|
|
@ -65,28 +95,35 @@ for source_prefix in /opt/sources/*; do
|
|||
aptly snapshot create -config="${conf}" "${name}" from mirror "${name}"
|
||||
done
|
||||
|
||||
# preserve the codename and label of the source repository
|
||||
codename=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Codename: //p')
|
||||
label=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Label: //p')
|
||||
# If the codename or label have not been specified then acquire them from
|
||||
# the mirror
|
||||
if [[ "${codename}" == "null" ]]; then
|
||||
codename=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Codename: //p')
|
||||
fi
|
||||
if [[ "${label}" == "null" ]]; then
|
||||
label=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Label: //p')
|
||||
fi
|
||||
|
||||
# Publish snapshot and sign if a key passphrase is provided.
|
||||
com_list=$(echo "${components[@]}" | tr ' ' ',')
|
||||
if [ ! -z "$1" ]; then
|
||||
com_list=$(echo "${components}" | jq -r '. | join(",")')
|
||||
if [[ ! -z "$1" ]]; then
|
||||
aptly publish snapshot \
|
||||
-batch=true \
|
||||
-config="${conf}" \
|
||||
-component="${com_list}" \
|
||||
-distribution="${dist}" \
|
||||
-passphrase="${1}" \
|
||||
-codename="${codename}" \
|
||||
-label="${label}" \
|
||||
"${mirrors[@]}" "${source_prefix:13}"
|
||||
"${mirrors[@]}" "${source_name}"
|
||||
else
|
||||
aptly publish snapshot \
|
||||
-config="${conf}" \
|
||||
-component="${com_list}" \
|
||||
-distribution="${dist}" \
|
||||
-codename="${codename}" \
|
||||
-label="${label}" \
|
||||
"${mirrors[@]}" "${source_prefix:13}"
|
||||
"${mirrors[@]}" "${source_name}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in New Issue