Add support for Ubuntu bionic base image
Added support to buid shipyard and airflow images using either a xenial or Ubuntu bionic base image. The default base image is set to bionic. Change-Id: I6ad4d42dede081586b3ccea87a42e250979ac106
This commit is contained in:
Ahmad Mahmoudi
parent
72ca47e5c9
commit
24f6a01e0b
54
.zuul.yaml
54
.zuul.yaml
@ -25,18 +25,21 @@
|
||||
- airship-shipyard-chart-build-latest-htk
|
||||
- airship-shipyard-whitespace-lint-gate
|
||||
- airship-shipyard-airskiff-deployment
|
||||
- airship-shipyard-image-gate-ubuntu
|
||||
- airship-shipyard-image-gate-ubuntu_xenial
|
||||
- airship-shipyard-image-gate-ubuntu_bionic
|
||||
- airship-shipyard-image-gate-opensuse
|
||||
gate:
|
||||
jobs:
|
||||
- openstack-tox-pep8
|
||||
- airship-shipyard-chart-build-gate
|
||||
- airship-shipyard-whitespace-lint-gate
|
||||
- airship-shipyard-image-gate-ubuntu
|
||||
- airship-shipyard-image-gate-ubuntu_xenial
|
||||
- airship-shipyard-image-gate-ubuntu_bionic
|
||||
- airship-shipyard-image-gate-opensuse
|
||||
post:
|
||||
jobs:
|
||||
- airship-shipyard-docker-build-post-ubuntu
|
||||
- airship-shipyard-docker-build-post-ubuntu_xenial
|
||||
- airship-shipyard-docker-build-post-ubuntu_bionic
|
||||
- airship-shipyard-docker-build-post-opensuse
|
||||
- shipyard-upload-git-mirror
|
||||
|
||||
@ -44,7 +47,7 @@
|
||||
name: airship-shipyard-single-node
|
||||
nodes:
|
||||
- name: primary
|
||||
label: ubuntu-xenial
|
||||
label: ubuntu-bionic
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-chart-build-gate
|
||||
@ -83,9 +86,9 @@
|
||||
- ^releasenotes/.*$
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-image-gate-ubuntu
|
||||
name: airship-shipyard-image-gate-ubuntu_xenial
|
||||
description: |
|
||||
Run shipyard-image build for ubuntu.
|
||||
Run shipyard-image build for ubuntu_xenial.
|
||||
parent: airship-shipyard-image-base
|
||||
vars:
|
||||
publish: false
|
||||
@ -94,6 +97,18 @@
|
||||
dynamic:
|
||||
patch_set: true
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-image-gate-ubuntu_bionic
|
||||
description: |
|
||||
Run shipyard-image build for ubuntu_bionic.
|
||||
parent: airship-shipyard-image-base
|
||||
vars:
|
||||
publish: false
|
||||
distro: ubuntu_bionic
|
||||
tags:
|
||||
dynamic:
|
||||
patch_set: true
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-airskiff-deployment
|
||||
nodeset: airship-shipyard-single-node
|
||||
@ -110,7 +125,7 @@
|
||||
- airship/treasuremap
|
||||
vars:
|
||||
CLONE_SHIPYARD: false
|
||||
irrelevant-files:
|
||||
irrelevant-files: &irrelevant-files
|
||||
- ^.*\.rst$
|
||||
- ^doc/.*$
|
||||
- ^releasenotes/.*$
|
||||
@ -128,14 +143,11 @@
|
||||
patch_set: true
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-docker-build-post-ubuntu
|
||||
name: airship-shipyard-docker-build-post-ubuntu_xenial
|
||||
timeout: 1800
|
||||
run: tools/gate/playbooks/run-image.yaml
|
||||
nodeset: airship-shipyard-single-node
|
||||
irrelevant-files:
|
||||
- ^.*\.rst$
|
||||
- ^doc/.*$
|
||||
- ^releasenotes/.*$
|
||||
irrelevant-files: *irrelevant-files
|
||||
secrets:
|
||||
- airship_shipyard_quay_creds
|
||||
vars:
|
||||
@ -148,6 +160,24 @@
|
||||
static:
|
||||
- latest
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-docker-build-post-ubuntu_bionic
|
||||
timeout: 1800
|
||||
run: tools/gate/playbooks/run-image.yaml
|
||||
nodeset: airship-shipyard-single-node
|
||||
irrelevant-files: *irrelevant-files
|
||||
secrets:
|
||||
- airship_shipyard_quay_creds
|
||||
vars:
|
||||
publish: true
|
||||
distro: ubuntu_bionic
|
||||
tags:
|
||||
dynamic:
|
||||
branch: true
|
||||
commit: true
|
||||
static:
|
||||
- latest
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-docker-build-post-opensuse
|
||||
timeout: 1800
|
||||
|
||||
2
Makefile
2
Makefile
@ -34,7 +34,7 @@ USE_PROXY ?= false
|
||||
AIRFLOW_SRC ?=
|
||||
AIRFLOW_HOME ?=
|
||||
DISTRO_BASE_IMAGE ?=
|
||||
DISTRO ?= ubuntu_xenial
|
||||
DISTRO ?= ubuntu_bionic
|
||||
|
||||
IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
|
||||
IMAGE_DIR:=images/$(IMAGE_NAME)
|
||||
|
||||
@ -31,15 +31,15 @@ labels:
|
||||
|
||||
images:
|
||||
tags:
|
||||
airflow: quay.io/airshipit/airflow:latest-ubuntu_xenial
|
||||
shipyard: quay.io/airshipit/shipyard:latest-ubuntu_xenial
|
||||
airflow: quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
shipyard: quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
|
||||
shipyard_db_init: docker.io/postgres:9.5
|
||||
shipyard_db_auxiliary: docker.io/postgres:9.5
|
||||
shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_xenial
|
||||
shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
airflow_db_init: docker.io/postgres:9.5
|
||||
rabbit_init: docker.io/rabbitmq:3.7-management
|
||||
airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_xenial
|
||||
airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
ks_user: docker.io/openstackhelm/heat:ocata
|
||||
ks_service: docker.io/openstackhelm/heat:ocata
|
||||
ks_endpoints: docker.io/openstackhelm/heat:ocata
|
||||
|
||||
@ -184,7 +184,7 @@ command. In this example we will execute the ``get actions`` command::
|
||||
-e 'OS_PASSWORD=password' -e 'OS_PROJECT_DOMAIN_NAME=default' \
|
||||
-e 'OS_PROJECT_NAME=service' -e 'OS_USERNAME=shipyard' \
|
||||
-e 'OS_USER_DOMAIN_NAME=default' -e 'OS_IDENTITY_API_VERSION=3' \
|
||||
--rm --net=host airshipit/shipyard:latest-ubuntu_xenial get actions
|
||||
--rm --net=host airshipit/shipyard:latest-ubuntu_bionic get actions
|
||||
|
||||
The output will resemble the following::
|
||||
|
||||
@ -208,7 +208,7 @@ Definition
|
||||
Preparation
|
||||
The site is assembled, racking, and wiring is completed, and the hardware is
|
||||
readied for operation. The ``Genesis Node`` is preinstalled with an
|
||||
(Ubuntu 16.04) image. Airship is deployed; See
|
||||
(Ubuntu 18.04) image. Airship is deployed; See
|
||||
:ref:`shipyard_deployment_guide`
|
||||
|
||||
At this point, Airship is ready for use. This is when the Shipyard API is
|
||||
|
||||
@ -45,7 +45,7 @@ Following parameters need to be passed as environment/shell variable to make
|
||||
command:
|
||||
|
||||
DISTRO
|
||||
parameter to identify distro specific Dockerfile, ubuntu_xenial (Default)
|
||||
parameter to identify distro specific Dockerfile, ubuntu_bionic (Default)
|
||||
|
||||
DISTRO_BASE_IMAGE
|
||||
parameter to use different base image other than what's used in DISTRO
|
||||
|
||||
142
images/airflow/Dockerfile.ubuntu_bionic
Normal file
142
images/airflow/Dockerfile.ubuntu_bionic
Normal file
@ -0,0 +1,142 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Docker image to run Airflow on Kubernetes
|
||||
ARG FROM=ubuntu:18.04
|
||||
FROM ${FROM}
|
||||
|
||||
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
|
||||
org.opencontainers.image.url='https://airshipit.org' \
|
||||
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
|
||||
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
|
||||
org.opencontainers.image.vendor='The Airship Authors' \
|
||||
org.opencontainers.image.licenses='Apache-2.0'
|
||||
|
||||
# Do not prompt user for choices on installation/configuration of packages
|
||||
# Set port 8080 for Airflow Web
|
||||
# Set port 5555 for Airflow Flower
|
||||
# Set port 8793 for Airflow Worker
|
||||
ENV container docker
|
||||
ENV WEB_PORT 8080
|
||||
ENV FLOWER_PORT 5555
|
||||
ENV WORKER_PORT 8793
|
||||
ENV SLUGIFY_USES_TEXT_UNIDECODE yes
|
||||
|
||||
# Expose port for applications
|
||||
EXPOSE $WEB_PORT
|
||||
EXPOSE $FLOWER_PORT
|
||||
EXPOSE $WORKER_PORT
|
||||
|
||||
# Set ARG for usage during build
|
||||
ARG AIRFLOW_HOME=/usr/local/airflow
|
||||
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG ctx_base=src/bin
|
||||
|
||||
# Kubectl version
|
||||
ARG KUBECTL_VERSION=1.16.2
|
||||
|
||||
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
||||
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
||||
|
||||
RUN set -ex && \
|
||||
apt-get -qq update && \
|
||||
apt-get -y install \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
g++ \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libpq-dev \
|
||||
locales \
|
||||
netcat \
|
||||
netbase \
|
||||
python3 \
|
||||
python3-setuptools \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-dateutil \
|
||||
make \
|
||||
--no-install-recommends \
|
||||
&& python3 -m pip install -U pip \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base
|
||||
|
||||
# Things that change mostly infrequently
|
||||
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
|
||||
&& curl -L -o /usr/local/bin/kubectl \
|
||||
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
|
||||
&& chmod +x /usr/local/bin/kubectl
|
||||
|
||||
# Dependency requirements
|
||||
# Note - removing snakebite (python 2 vs. 3). See:
|
||||
# https://github.com/puckel/docker-airflow/issues/77
|
||||
# Install Airflow directly to allow overriding source
|
||||
COPY images/airflow/requirements.txt /tmp/
|
||||
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
|
||||
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
|
||||
&& pip3 uninstall -y snakebite || true
|
||||
|
||||
# Copy scripts used in the container:
|
||||
COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
|
||||
|
||||
# Copy configuration (e.g. logging config for Airflow):
|
||||
COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
|
||||
|
||||
# Change permissions
|
||||
RUN chown -R airflow: ${AIRFLOW_HOME}
|
||||
|
||||
# Setting the version explicitly for PBR
|
||||
ENV PBR_VERSION 0.1a1
|
||||
|
||||
# Shipyard
|
||||
#
|
||||
# Shipyard provides core functionality used by the Airflow plugins/operators
|
||||
# Since Shipyard and Airflow are built together as images, this should prevent
|
||||
# stale or out-of-date code between these parts.
|
||||
# Shipyard requirements, source and installation
|
||||
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
|
||||
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
|
||||
|
||||
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
|
||||
RUN cd /tmp/shipyard \
|
||||
&& python3 setup.py install
|
||||
|
||||
# Note: The value for the dags and plugins directories that are sourced
|
||||
# from the values.yaml of the Shipyard Helm chart need to align with these
|
||||
# directories. If they do not, airflow will not find the intended dags and
|
||||
# plugins.
|
||||
#
|
||||
# Note: In the case of building images using the provided Makefile, a test is
|
||||
# run against the built-in dags provided with Airflow. Since there is no Helm
|
||||
# chart to reconfigure the airflow.cfg with these directories, these dags and
|
||||
# plugins are not known to Airflow during the image test.
|
||||
#
|
||||
# Copy the plugins and dags that will be used by this Airflow image:
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
|
||||
|
||||
# Set work directory
|
||||
USER airflow
|
||||
WORKDIR ${AIRFLOW_HOME}
|
||||
|
||||
# Execute entrypoint
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
107
images/shipyard/Dockerfile.ubuntu_bionic
Normal file
107
images/shipyard/Dockerfile.ubuntu_bionic
Normal file
@ -0,0 +1,107 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARG FROM=ubuntu:18.04
|
||||
FROM ${FROM}
|
||||
|
||||
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
|
||||
org.opencontainers.image.url='https://airshipit.org' \
|
||||
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
|
||||
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
|
||||
org.opencontainers.image.vendor='The Airship Authors' \
|
||||
org.opencontainers.image.licenses='Apache-2.0'
|
||||
|
||||
ENV container docker
|
||||
ENV PORT 9000
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV LANG C.UTF-8
|
||||
# Setting the version explicitly for PBR
|
||||
ENV PBR_VERSION 0.1a1
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG ctx_base=src/bin
|
||||
|
||||
# Expose port 9000 for application
|
||||
EXPOSE $PORT
|
||||
|
||||
RUN set -ex && \
|
||||
apt-get -qq update && \
|
||||
apt-get -y install \
|
||||
ca-certificates \
|
||||
curl \
|
||||
netbase \
|
||||
python3-dev \
|
||||
python3-setuptools \
|
||||
--no-install-recommends \
|
||||
&& apt-get autoremove -yqq --purge \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base
|
||||
|
||||
# Create shipyard user
|
||||
RUN useradd -ms /bin/bash shipyard \
|
||||
&& mkdir -p /home/shipyard/shipyard \
|
||||
&& mkdir -p /home/shipyard/shipyard_client
|
||||
|
||||
# Copy entrypoint.sh to /home/shipyard
|
||||
COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh
|
||||
# Change permissions and set up directories
|
||||
RUN chown -R shipyard: /home/shipyard \
|
||||
&& chmod +x /home/shipyard/entrypoint.sh
|
||||
|
||||
# Requirements and Shipyard source
|
||||
COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
|
||||
COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
|
||||
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
|
||||
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
|
||||
|
||||
# Build
|
||||
RUN set -ex \
|
||||
&& buildDeps=' \
|
||||
gcc \
|
||||
git \
|
||||
libssl-dev \
|
||||
make \
|
||||
python3-pip \
|
||||
' \
|
||||
&& apt-get -qq update \
|
||||
&& apt-get -y install -y $buildDeps --no-install-recommends \
|
||||
&& python3 -m pip install -U pip \
|
||||
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard_client \
|
||||
&& python3 setup.py install \
|
||||
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard \
|
||||
&& python3 setup.py install \
|
||||
&& apt-get purge -y --auto-remove $buildDeps \
|
||||
&& apt-get autoremove -yqq --purge \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base
|
||||
|
||||
# Entrypoint
|
||||
ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
|
||||
CMD ["server"]
|
||||
# Set user to shipyard
|
||||
USER shipyard
|
||||
@ -1,73 +0,0 @@
|
||||
# Shipyard
|
||||
Shipyard is the directed acyclic graph controller for Kubernetes and
|
||||
OpenStack control plane life cycle management, and a component of the
|
||||
Airship Undercloud Platform (UCP).
|
||||
|
||||
Shipyard provides the entrypoint for the following aspects of the
|
||||
control plane established by the Airship:
|
||||
|
||||
<dl>
|
||||
<dt>
|
||||
Designs and Secrets
|
||||
</dt>
|
||||
<dd>
|
||||
Site designs, including the configuration of bare metal host
|
||||
nodes, network design, operating systems, Kubernetes nodes,
|
||||
Armada manifests, Helm charts, and any other descriptors that
|
||||
define the build out of a group of servers enter the Airship via
|
||||
Shipyard. Secrets, such as passwords and certificates use the
|
||||
same mechanism. <br />
|
||||
The designs and secrets are stored in Airship's Deckhand,
|
||||
providing for version history and secure storage among other
|
||||
document-based conveniences.
|
||||
</dd>
|
||||
<dt>
|
||||
Actions
|
||||
</dt>
|
||||
<dd>
|
||||
Interaction with the site's control plane is done via
|
||||
invocation of actions in Shipyard. Each action is backed by
|
||||
a workflow implemented as a directed acyclic graph (DAG) that
|
||||
runs using Apache Airflow. Shipyard provides a mechanism to
|
||||
monitor and control the execution of the workflow.
|
||||
</dd>
|
||||
</dl>
|
||||
|
||||
Find more documentation for Shipyard on
|
||||
[Read the Docs](https://airship-shipyard.readthedocs.io)
|
||||
|
||||
|
||||
## Integration Points:
|
||||
[OpenStack Identity (Keystone)](https://github.com/openstack/keystone)
|
||||
provides authentication and support for role based authorization
|
||||
\
|
||||
[Apache Airflow](https://airflow.incubator.apache.org/) provides the
|
||||
framework and automation of workflows provided by Shipyard
|
||||
\
|
||||
[PostgreSQL](https://www.postgresql.org/) is used to persist
|
||||
information to correlate workflows with users and history of workflow
|
||||
commands
|
||||
\
|
||||
[Deckhand](https://github.com/openstack/airship-deckhand) supplies storage
|
||||
and management of site designs and secrets
|
||||
\
|
||||
[Drydock](https://github.com/openstack/airship-drydock) is orchestrated by
|
||||
Shipyard to perform bare metal node provisioning
|
||||
\
|
||||
[Promenade](https://github.com/openstack/airship-promenade) is indirectly
|
||||
orchestrated by Shipyard to configure and join Kubernetes nodes
|
||||
\
|
||||
[Armada](https://github.com/openstack/airship-armada) is orchestrated by
|
||||
Shipyard to deploy and test Kubernetes workloads
|
||||
|
||||
|
||||
## Getting Started:
|
||||
|
||||
[Shipyard @ Openstack Gerrit](https://review.openstack.org/#/q/project:openstack/airship-shipyard)
|
||||
\
|
||||
[Helm chart](https://github.com/openstack/airship-shipyard/tree/master/charts/shipyard)
|
||||
|
||||
|
||||
## See also:
|
||||
|
||||
[Airship in a Bottle](https://github.com/openstack/airship-in-a-bottle)
|
||||
82
src/bin/shipyard_airflow/README.rst
Normal file
82
src/bin/shipyard_airflow/README.rst
Normal file
@ -0,0 +1,82 @@
|
||||
Shipyard
|
||||
========
|
||||
|
||||
Shipyard is the directed acyclic graph controller for Kubernetes and
|
||||
OpenStack control plane life cycle management, and a component of the
|
||||
Airship Undercloud Platform (UCP).
|
||||
|
||||
Shipyard provides the entrypoint for the following aspects of the
|
||||
control plane established by the Airship:
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<dl>
|
||||
|
||||
::
|
||||
|
||||
<dt>
|
||||
Designs and Secrets
|
||||
</dt>
|
||||
<dd>
|
||||
Site designs, including the configuration of bare metal host
|
||||
nodes, network design, operating systems, Kubernetes nodes,
|
||||
Armada manifests, Helm charts, and any other descriptors that
|
||||
define the build out of a group of servers enter the Airship via
|
||||
Shipyard. Secrets, such as passwords and certificates use the
|
||||
same mechanism. <br />
|
||||
The designs and secrets are stored in Airship's Deckhand,
|
||||
providing for version history and secure storage among other
|
||||
document-based conveniences.
|
||||
</dd>
|
||||
<dt>
|
||||
Actions
|
||||
</dt>
|
||||
<dd>
|
||||
Interaction with the site's control plane is done via
|
||||
invocation of actions in Shipyard. Each action is backed by
|
||||
a workflow implemented as a directed acyclic graph (DAG) that
|
||||
runs using Apache Airflow. Shipyard provides a mechanism to
|
||||
monitor and control the execution of the workflow.
|
||||
</dd>
|
||||
|
||||
.. raw:: html
|
||||
|
||||
</dl>
|
||||
|
||||
Find more documentation for Shipyard on `Read the
|
||||
Docs <https://airship-shipyard.readthedocs.io>`__
|
||||
|
||||
Integration Points:
|
||||
-------------------
|
||||
|
||||
| `OpenStack Identity
|
||||
(Keystone) <https://github.com/openstack/keystone>`__ provides
|
||||
authentication and support for role based authorization
|
||||
| `Apache Airflow <https://airflow.incubator.apache.org/>`__ provides
|
||||
the framework and automation of workflows provided by Shipyard
|
||||
| `PostgreSQL <https://www.postgresql.org/>`__ is used to persist
|
||||
information to correlate workflows with users and history of workflow
|
||||
commands
|
||||
| `Deckhand <https://github.com/openstack/airship-deckhand>`__ supplies
|
||||
storage and management of site designs and secrets
|
||||
| `Drydock <https://github.com/openstack/airship-drydock>`__ is
|
||||
orchestrated by Shipyard to perform bare metal node provisioning
|
||||
| `Promenade <https://github.com/openstack/airship-promenade>`__ is
|
||||
indirectly orchestrated by Shipyard to configure and join Kubernetes
|
||||
nodes
|
||||
| `Armada <https://github.com/openstack/airship-armada>`__ is
|
||||
orchestrated by Shipyard to deploy and test Kubernetes workloads
|
||||
|
||||
Getting Started:
|
||||
----------------
|
||||
|
||||
| `Shipyard @ Openstack
|
||||
Gerrit <https://review.openstack.org/#/q/project:openstack/airship-shipyard>`__
|
||||
| `Helm
|
||||
chart <https://github.com/openstack/airship-shipyard/tree/master/charts/shipyard>`__
|
||||
|
||||
See also:
|
||||
---------
|
||||
|
||||
`Airship in a
|
||||
Bottle <https://github.com/openstack/airship-in-a-bottle>`__
|
||||
@ -1,7 +1,7 @@
|
||||
[metadata]
|
||||
name = shipyard_airflow
|
||||
summary = Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management
|
||||
description-file = README.md
|
||||
description-file = README.rst
|
||||
author = The Airship Authors
|
||||
home-page = https://airship-shipyard.readthedocs.io/
|
||||
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
### Airflow Task State Sample Dag
|
||||
"""
|
||||
import airflow
|
||||
from airflow import DAG
|
||||
from airflow.operators import TaskStateOperator
|
||||
from airflow.operators.bash_operator import BashOperator
|
||||
from datetime import timedelta
|
||||
|
||||
default_args = {
|
||||
'owner': 'airflow',
|
||||
'depends_on_past': False,
|
||||
'start_date': airflow.utils.dates.days_ago(2),
|
||||
'email': ['airflow@example.com'],
|
||||
'email_on_failure': False,
|
||||
'email_on_retry': False,
|
||||
'retries': 1,
|
||||
'retry_delay': timedelta(minutes=1),
|
||||
}
|
||||
|
||||
dag = DAG('airflow_task_state',
|
||||
default_args=default_args,
|
||||
schedule_interval=None)
|
||||
|
||||
# Get Task State
|
||||
t1 = TaskStateOperator(
|
||||
task_id='airflow_task_state',
|
||||
airflow_dag_id='openstack_cli',
|
||||
airflow_task_id='endpoint_list_task',
|
||||
airflow_execution_date='2017-07-02T21:30:33.519582',
|
||||
dag=dag)
|
||||
|
||||
# Use XCOM to Retrieve Task State
|
||||
t2 = BashOperator(
|
||||
task_id='pull',
|
||||
bash_command=("echo {{ ti.xcom_pull(task_ids='airflow_task_state',"
|
||||
" key='task_state') }}"),
|
||||
xcom_push=True,
|
||||
dag=dag)
|
||||
|
||||
t2.set_upstream(t1)
|
||||
@ -1,5 +1,5 @@
|
||||
# Testing
|
||||
amqp==2.5.1
|
||||
amqp==2.5.2
|
||||
pytest==3.5.0
|
||||
pytest-cov==2.5.1
|
||||
responses==0.10.2
|
||||
|
||||
@ -12,8 +12,18 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- hosts: primary
|
||||
- hosts: ubuntu-bionic
|
||||
tasks:
|
||||
# Stop systemd-resolved service before starting docker.
|
||||
- name: stop systemd-resolved service
|
||||
systemd:
|
||||
state: stopped
|
||||
enabled: no
|
||||
masked: yes
|
||||
daemon_reload: yes
|
||||
name: systemd-resolved
|
||||
become: yes
|
||||
|
||||
- name: Clone Required Repositories
|
||||
shell: |
|
||||
export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
|
||||
@ -27,6 +37,12 @@
|
||||
args:
|
||||
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
|
||||
|
||||
- name: Setup AppArmor
|
||||
shell: |
|
||||
./tools/deployment/airskiff/developer/015-setup-apparmor.sh
|
||||
args:
|
||||
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
|
||||
|
||||
- name: Build Shipyard and Airflow with submitted changes
|
||||
shell: |
|
||||
make
|
||||
|
||||
@ -63,7 +63,7 @@
|
||||
- apt_key:
|
||||
url: https://download.docker.com/linux/ubuntu/gpg
|
||||
- apt_repository:
|
||||
repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker xenial stable
|
||||
repo: "deb http://{{ zuul_site_mirror_fqdn }}/deb-docker {% if distro.startswith('ubuntu_') %} {{ distro | regex_replace('ubuntu_') }} {% else %} xenial {% endif %} stable"
|
||||
- apt:
|
||||
name: "{{ item }}"
|
||||
allow_unauthenticated: True
|
||||
|
||||
@ -38,7 +38,8 @@ set -e
|
||||
# Source Base Docker Command
|
||||
SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"}
|
||||
NAMESPACE="${NAMESPACE:-ucp}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-ubuntu_xenial}"
|
||||
DISTRO="${DISTRO:-ubuntu_bionic}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
|
||||
# set default value for OS_PASSWORD if it's not set
|
||||
# this doesn't actually get exported to environment
|
||||
# unless the script is sourced
|
||||
|
||||
@ -19,7 +19,8 @@
|
||||
# before executing this script if they differ from the default values.
|
||||
#
|
||||
NAMESPACE="${NAMESPACE:-ucp}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-ubuntu_xenial}"
|
||||
DISTRO="${DISTRO:-ubuntu_bionic}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
|
||||
|
||||
# Define Base Docker Command
|
||||
base_docker_command=$(cat << EndOfCommand
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user