Add IMAGE_ELEMENT_YAML and get_image_element_array

These new variables are a list of elements chosen for the build along
with their full paths.  For Python elements, IMAGE_ELEMENT_YAML is a
YAML formatted list that can be easily parsed.  For bash elements,
"get_image_element_array" will produce an associative-array of the
same (working around lack of array export in Bash).

This list is intended for consumption of elements who need to copy
files from other elements, such as pkg-map and svc-map.  As discussed
in I2a29861c67de2d25c595cb35d850e92807d26ac6, this list has already
been pruned and had overrides processed, so it is safe to simply walk
over this list with no further processing.

Since we're presenting the element list in a couple of different ways,
we combine it all into the element-info script.  It will output an
eval-able string that declares the appropriate variables.

I've added some inline documentation so they still appear in grep.
The documentation is updated with examples, and moved to a more
appropriate location as a sub-section of the element sytle guide.

To test this out, use the associative-array in generate_hooks, where
we can now find the element's directory without searching.

Change-Id: Ibbd07d082ec827441def2d3f6240df3efdc6eae3

diskimage_builder/element_dependencies.py

@@ -19,6 +19,7 @@ import errno
 import logging
 import os
 import sys
+import yaml
 
 import diskimage_builder.logging_config
 
@@ -188,6 +189,10 @@ def main(argv):
                         default=False,
                         help=('(DEPRECATED) print expanded dependencies '
                               'of all args'))
+    parser.add_argument('--env', '-e', action='store_true',
+                        default=False,
+                        help=('Output eval-able bash strings for '
+                              'IMAGE_ELEMENT variables'))
 
     args = parser.parse_args(argv[1:])
 
@@ -197,5 +202,33 @@ def main(argv):
         logger.warning("expand-dependencies flag is deprecated,  "
                        "and is now on by default.", file=sys.stderr)
 
-    print(' '.join(expand_dependencies(args.elements, all_elements)))
+    elements = expand_dependencies(args.elements, all_elements)
+
+    if args.env:
+        # first the "legacy" environment variable that just lists the
+        # elements
+        print("export IMAGE_ELEMENT='%s'" % ' '.join(elements))
+
+        # Then YAML
+        output = {}
+        for element in elements:
+            output[element] = all_elements[element].path
+        print("export IMAGE_ELEMENT_YAML='%s'" % yaml.safe_dump(output))
+
+        # Then bash array.  Unfortunately, bash can't export array
+        # variables.  So we take a compromise and produce an exported
+        # function that outputs the string to re-create the array.
+        # You can then simply do
+        #  eval declare -A element_array=$(get_image_element_array)
+        # and you have it.
+        output = ""
+        for element in elements:
+            output += '[%s]=%s ' % (element, all_elements[element].path)
+        print("function get_image_element_array {\n"
+              "  echo \"%s\"\n"
+              "};\n"
+              "export -f get_image_element_array;" % output)
+    else:
+        print(' '.join(elements))
+
     return 0

doc/source/developer/developing_elements.rst

@@ -178,10 +178,6 @@ To set environment variables for other hooks, add a file to your element
 This directory contains bash script snippets that are sourced before running
 scripts in each phase.
 
-DIB exposes an internal ``$IMAGE_ELEMENT`` variable which provides elements
-access to the full set of elements that are included in the image build. This
-can be used to process local in-element files across all the elements
-(``pkg-map`` for example).
 
 Dependencies
 ^^^^^^^^^^^^
@@ -319,6 +315,37 @@ diskimage-builder, making it possible to use the same element to track more
 then one branch of a git repository or to get source for a local cache. See
 :doc:`../elements/source-repositories/README` for more information.
 
+Finding other elements
+----------------------
+
+DIB exposes an internal ``$IMAGE_ELEMENT_YAML`` variable which
+provides elements access to the full set of included elements and
+their paths.  This can be used to process local in-element files
+across all the elements (``pkg-map`` for example).
+
+.. code-block:: python
+
+   import os
+   import yaml
+
+   elements = yaml.load(os.getenv('IMAGE_ELEMENT_YAML'))
+   for element, path in elements:
+      ...
+
+For elements written in Bash, there is a function
+``get_image_element_array`` that can be used to instantiate an
+associative-array of elements and paths (note arrays can not be
+exported in bash).
+
+.. code-block:: bash
+
+   # note eval to expand the result of the get function
+   eval declare -A image_elements=($(get_image_element_array))
+   for i in ${!image_elements[$i]}; do
+     element=$i
+     path=${image_elements[$i]}
+   done
+
 Debugging elements
 ------------------
 

lib/common-functions

@@ -75,19 +75,27 @@ function copy_hooks_not_overwrite () {
 }
 
 function generate_hooks () {
-  mkdir -p $TMP_HOOKS_PATH
-  for _ELEMENT in $IMAGE_ELEMENT ; do
-    for dir in ${ELEMENTS_PATH//:/ } ; do
-      [ -d $dir/$_ELEMENT ] || continue
-      for _DIR in $(find $dir/$_ELEMENT -follow -mindepth 1 -maxdepth 1 -type d -not -name tests); do
-        copy_hooks_not_overwrite $_DIR
-      done
-      for _FILE in $(find $dir/$_ELEMENT -follow -maxdepth 1 -type f); do
-        cp -t $TMP_HOOKS_PATH -a $_FILE
-      done
-      break
+    local dir
+    local file
+
+    eval declare -A image_elements=($(get_image_element_array))
+
+    mkdir -p $TMP_HOOKS_PATH
+
+    for i in "${!image_elements[@]}"; do
+        local element=$i
+        local element_dir=${image_elements[$i]}
+
+        for dir in $(find $element_dir \
+                          -follow -mindepth 1 -maxdepth 1 \
+                          -type d -not -name tests); do
+            copy_hooks_not_overwrite $dir
+        done
+        for file in $(find $element_dir \
+                           -follow -maxdepth 1 -type f); do
+            cp -t $TMP_HOOKS_PATH -a $file
+        done
     done
-  done
 }
 
 # Call the supplied break-in routine if the named point is listed in the break
@@ -196,8 +204,29 @@ function arg_to_elements() {
   fi
   echo "Building elements: $IMAGE_ELEMENT"
 
-  IMAGE_ELEMENT=$($SCRIPT_HOME/element-info $IMAGE_ELEMENT)
-  export IMAGE_ELEMENT
+  # element-info will output bash code to create
+  #  * IMAGE_ELEMENT
+  #     legacy list of elements
+  #
+  #  * IMAGE_ELEMENT_YAML
+  #     YAML dictionary with key=element, value=path
+  #
+  #      import os
+  #      import yaml
+  #      yaml.load(os.getenv('IMAGE_ELEMENT_YAML')
+  #
+  #  * function get_image_element_array
+  #     Function to create Bash associative-array.  Since bash can not
+  #     export array variables, we provide a function to populate the
+  #     variables.
+  #
+  #      # we need the eval, it expands the string for the array create
+  #      eval declare -A image_elements=($(get_image_element_array))
+  #      for i in ${!image_elements[@]}; do
+  #        element=$i
+  #        path=${image_elements[$i]
+  #      done
+  eval "$($SCRIPT_HOME/element-info --env $IMAGE_ELEMENT)"
 
   echo "Expanded element dependencies to: $IMAGE_ELEMENT"
 }

releasenotes/notes/element-vars-c6bf2e6795002f01.yaml

@@ -0,0 +1,6 @@
+---
+features:
+
+  - Elements that need access to the other elements being used during
+    the build should use the new ``IMAGE_ELEMENT_YAML`` environment
+    variable and it's Bash equivalent ``get_image_element_array``.