Use clouds.yaml for puppet

This change introduces the capability to load clouds.yaml file in
the base Puppet::Provider::Openstack::Auth module, so that each
providers can look up credentials from clouds.yaml instead of rc file.
When SRBAC is enforced, services require appropriate scope for each
operation and this makes it difficult to use rc files which can store
only one credential per file. Usage of clouds.yaml allows us to store
multiple credentials in a single file and switch scopes according to
the API request used.

The new implementation loads the clouds.yaml file for admin user, which
is created by puppet-keystoe. It also allows overriding the credential
by a user-created clouds.file.

We expect clouds.yaml file is created under /etc/openstack, which is
the location openstackclient searches to look up clouds.yaml. To avoid
unexpected conjunction with existing files, the files used by puppet
are located in an independent 'puppet' directory at this moment.

Change-Id: I7587f6e0c2486cbfaf2cbafeb64e9db56a817106

lib/puppet/provider/openstack/auth.rb

@@ -5,12 +5,31 @@ module Puppet::Provider::Openstack::Auth
 
   RCFILENAME = "#{ENV['HOME']}/openrc"
 
+  CLOUDSFILENAMES = [
+    # This allows overrides by users
+    "/etc/openstack/puppet/clouds.yaml",
+    # This is created by puppet-keystone
+    "/etc/openstack/puppet/admin-clouds.yaml",
+    ]
+
   def get_os_vars_from_env
     env = {}
     ENV.each { |k,v| env.merge!(k => v) if k =~ /^OS_/ }
     return env
   end
 
+  def get_os_vars_from_cloudsfile(scope)
+    cloudsfile = clouds_filenames.detect { |f| File.exists? f}
+    unless cloudsfile.nil?
+      {
+        'OS_CLOUD'              => scope,
+        'OS_CLIENT_CONFIG_FILE' => cloudsfile
+      }
+    else
+      {}
+    end
+  end
+
   def get_os_vars_from_rcfile(filename)
     env = {}
     rcfile = [filename, '/root/openrc'].detect { |f| File.exists? f }
@@ -32,13 +51,30 @@ module Puppet::Provider::Openstack::Auth
     RCFILENAME
   end
 
+  def clouds_filenames
+    CLOUDSFILENAMES
+  end
+
   def request(service, action, properties=nil, options={}, scope='project')
     properties ||= []
+
+    # First, check environments
     set_credentials(@credentials, get_os_vars_from_env)
+
     unless @credentials.set? and (!@credentials.scope_set? or @credentials.scope == scope)
+      # Then look for clouds.yaml
       @credentials.unset
-      set_credentials(@credentials, get_os_vars_from_rcfile(rc_filename))
+      clouds_env = get_os_vars_from_cloudsfile(scope)
+      if ! clouds_env.empty?
+        set_credentials(@credentials, clouds_env)
+      else
+        # If it fails then check rc files, to keep backword compatibility.
+        warning('Usage of rc file is deprecated and will be removed in a future release.')
+        @credentials.unset
+        set_credentials(@credentials, get_os_vars_from_rcfile(rc_filename))
+      end
     end
+
     unless @credentials.set? and (!@credentials.scope_set? or @credentials.scope == scope)
       raise(Puppet::Error::OpenstackAuthInputError, 'Insufficient credentials to authenticate')
     end

lib/puppet/provider/openstack/credentials.rb

@@ -35,7 +35,10 @@ class Puppet::Provider::Openstack::Credentials
     env = {}
     self.instance_variables.each do |var|
       name = var.to_s.sub(/^@/,'OS_').upcase
-      env.merge!(name => self.instance_variable_get(var))
+      value = self.instance_variable_get(var)
+      unless value.nil?
+        env.merge!(name => value)
+      end
     end
     env
   end
@@ -62,7 +65,7 @@ class Puppet::Provider::Openstack::Credentials
     self.instance_variables.each do |var|
       if var.to_s != '@identity_api_version' &&
         self.instance_variable_defined?(var.to_s)
-        set(var.to_s.sub(/^@/,''), '')
+        set(var.to_s.sub(/^@/,''), nil)
       end
     end
   end

spec/unit/provider/openstack/auth_spec.rb

@@ -85,6 +85,43 @@ describe Puppet::Provider::Openstack::Auth do
     end
   end
 
+  describe '#get_os_vars_from_cloudsfile' do
+    context 'with a clouds.yaml present' do
+      it 'provides a hash' do
+        File.expects(:exists?).with('/etc/openstack/puppet/clouds.yaml').returns(true)
+
+        response = klass.get_os_vars_from_cloudsfile('project')
+        expect(response).to eq({
+          'OS_CLOUD'              => 'project',
+          'OS_CLIENT_CONFIG_FILE' => '/etc/openstack/puppet/clouds.yaml'
+        })
+      end
+    end
+
+    context 'with a admin-clouds.yaml present' do
+      it 'provides a hash' do
+        File.expects(:exists?).with('/etc/openstack/puppet/clouds.yaml').returns(false)
+        File.expects(:exists?).with('/etc/openstack/puppet/admin-clouds.yaml').returns(true)
+
+        response = klass.get_os_vars_from_cloudsfile('project')
+        expect(response).to eq({
+          'OS_CLOUD'              => 'project',
+          'OS_CLIENT_CONFIG_FILE' => '/etc/openstack/puppet/admin-clouds.yaml'
+        })
+      end
+    end
+
+    context 'with a clouds.yaml not present' do
+      it 'provides an empty hash' do
+        File.expects(:exists?).with('/etc/openstack/puppet/clouds.yaml').returns(false)
+        File.expects(:exists?).with('/etc/openstack/puppet/admin-clouds.yaml').returns(false)
+
+        response = klass.get_os_vars_from_cloudsfile('project')
+        expect(response).to eq({})
+      end
+    end
+  end
+
   describe '#get_os_vars_from_rcfile' do
     context 'with a valid RC file' do
       it 'provides a hash' do
@@ -209,6 +246,28 @@ describe Puppet::Provider::Openstack::Auth do
       end
     end
 
+    context 'with clouds.yaml file' do
+      it 'is successful' do
+        # return incomplete creds from env
+        klass.expects(:get_os_vars_from_env)
+             .returns({ 'OS_USERNAME' => 'incompleteusername',
+                        'OS_AUTH_URL' => 'incompleteauthurl' })
+        File.expects(:exists?).with('/etc/openstack/puppet/clouds.yaml').returns(true)
+        klass.expects(:openstack)
+             .with('project', 'list', '--quiet', '--format', 'csv', ['--long'])
+             .returns('"ID","Name","Description","Enabled"
+"1cb05cfed7c24279be884ba4f6520262","test","Test tenant",True
+')
+        response = provider.class.request('project', 'list', ['--long'])
+        expect(response.first[:description]).to eq("Test tenant")
+        expect(klass.instance_variable_get(:@credentials).to_env).to eq({
+          'OS_IDENTITY_API_VERSION' => '3',
+          'OS_CLOUD'                => 'project',
+          'OS_CLIENT_CONFIG_FILE'   => '/etc/openstack/puppet/clouds.yaml',
+        })
+      end
+    end
+
     context 'with a RC file containing user credentials' do
       it 'is successful' do
         # return incomplete creds from env
@@ -216,6 +275,8 @@ describe Puppet::Provider::Openstack::Auth do
              .returns({ 'OS_USERNAME' => 'incompleteusername',
                         'OS_AUTH_URL' => 'incompleteauthurl' })
         mock = "export OS_USERNAME='test'\nexport OS_PASSWORD='abc123'\nexport OS_PROJECT_NAME='test'\nexport OS_AUTH_URL='http://127.0.0.1:5000'\nexport OS_NOT_VALID='notvalid'"
+        File.expects(:exists?).with("/etc/openstack/puppet/clouds.yaml").returns(false)
+        File.expects(:exists?).with("/etc/openstack/puppet/admin-clouds.yaml").returns(false)
         File.expects(:exists?).with("#{ENV['HOME']}/openrc").returns(true)
         File.expects(:open).with("#{ENV['HOME']}/openrc").returns(StringIO.new(mock))
         klass.expects(:openstack)
@@ -241,6 +302,8 @@ describe Puppet::Provider::Openstack::Auth do
         klass.expects(:get_os_vars_from_env)
              .returns({ 'OS_TOKEN' => 'incomplete' })
         mock = "export OS_TOKEN='test'\nexport OS_ENDPOINT='abc123'\nexport OS_NOT_VALID='notvalid'\n"
+        File.expects(:exists?).with("/etc/openstack/puppet/clouds.yaml").returns(false)
+        File.expects(:exists?).with("/etc/openstack/puppet/admin-clouds.yaml").returns(false)
         File.expects(:exists?).with("#{ENV['HOME']}/openrc").returns(true)
         File.expects(:open).with("#{ENV['HOME']}/openrc").returns(StringIO.new(mock))
         klass.expects(:openstack)

spec/unit/provider/openstack/credentials_spec.rb

@@ -120,18 +120,18 @@ describe Puppet::Provider::Openstack::Credentials do
         creds.cloud = 'openstack'
         creds.client_config_file = '/etc/openstack/clouds.yaml'
         creds.unset
-        expect(creds.auth_url).to eq('')
-        expect(creds.password).to eq('')
-        expect(creds.project_name).to eq('')
-        expect(creds.domain_name).to eq('')
-        expect(creds.system_scope).to eq('')
-        expect(creds.username).to eq('')
-        expect(creds.token).to eq('')
-        expect(creds.endpoint).to eq('')
-        expect(creds.region_name).to eq('')
+        expect(creds.auth_url).to eq(nil)
+        expect(creds.password).to eq(nil)
+        expect(creds.project_name).to eq(nil)
+        expect(creds.domain_name).to eq(nil)
+        expect(creds.system_scope).to eq(nil)
+        expect(creds.username).to eq(nil)
+        expect(creds.token).to eq(nil)
+        expect(creds.endpoint).to eq(nil)
+        expect(creds.region_name).to eq(nil)
         expect(creds.identity_api_version).to eq('identity_api_version')
-        expect(creds.cloud).to eq('')
-        expect(creds.client_config_file).to eq('')
+        expect(creds.cloud).to eq(nil)
+        expect(creds.client_config_file).to eq(nil)
         newcreds = Puppet::Provider::Openstack::CredentialsV3.new
         expect(newcreds.identity_api_version).to eq('3')
       end