go: conf.d support + some refactoring

Refactor how configs are loaded and passed to daemons.

Add conf.d support. So you can have an /etc/swift/object-server.conf.d
or /etc/swift/object-server/1.conf.d, and any .conf files under it
will be loaded as one config.

conf.d files are combined in filename lexigraphical order, with any
sections in later files replace existing section entries.

Change-Id: I78d7e5449cd0b62df9dfdb25616a2d4c91159f8c

go/bench/dbench.go

@@ -126,7 +126,7 @@ func RunDBench(args []string) {
 		os.Exit(1)
 	}
 
-	benchconf, err := hummingbird.LoadIniFile(args[0])
+	benchconf, err := hummingbird.LoadConfig(args[0])
 	if err != nil {
 		fmt.Println("Error parsing ini file:", err)
 		os.Exit(1)

go/bench/main.go

@@ -139,7 +139,7 @@ func RunBench(args []string) {
 		os.Exit(1)
 	}
 
-	benchconf, err := hummingbird.LoadIniFile(args[0])
+	benchconf, err := hummingbird.LoadConfig(args[0])
 	if err != nil {
 		fmt.Println("Error parsing ini file:", err)
 		os.Exit(1)
@@ -227,7 +227,7 @@ func RunThrash(args []string) {
 		os.Exit(1)
 	}
 
-	thrashconf, err := hummingbird.LoadIniFile(args[0])
+	thrashconf, err := hummingbird.LoadConfig(args[0])
 	if err != nil {
 		fmt.Println("Error parsing ini file:", err)
 		os.Exit(1)

go/cmd/hummingbird.go

@@ -73,8 +73,10 @@ func findConfig(name string) string {
 	configName := strings.Split(name, "-")[0]
 	configSearch := []string{
 		fmt.Sprintf("/etc/hummingbird/%s-server.conf", configName),
+		fmt.Sprintf("/etc/hummingbird/%s-server.conf.d", configName),
 		fmt.Sprintf("/etc/hummingbird/%s-server", configName),
 		fmt.Sprintf("/etc/swift/%s-server.conf", configName),
+		fmt.Sprintf("/etc/swift/%s-server.conf.d", configName),
 		fmt.Sprintf("/etc/swift/%s-server", configName),
 	}
 	for _, config := range configSearch {

go/hummingbird/conf.go

@@ -0,0 +1,168 @@
+//  Copyright (c) 2015 Rackspace
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+//  implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package hummingbird
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/vaughan0/go-ini"
+)
+
+// Config represents an ini file.
+type Config struct{ ini.File }
+
+// Get fetches a value from the Config, looking in the DEFAULT section if not found in the specific section.  Also ignores "set " key prefixes, like paste.
+func (f Config) Get(section string, key string) (string, bool) {
+	if value, ok := f.File.Get(section, key); ok {
+		return value, true
+	} else if value, ok := f.File.Get("DEFAULT", key); ok {
+		return value, true
+	} else if value, ok := f.File.Get(section, "set "+key); ok {
+		return value, true
+	} else if value, ok := f.File.Get("DEFAULT", "set "+key); ok {
+		return value, true
+	}
+	return "", false
+}
+
+// GetDefault returns a value from the config, or returns the default setting if the entry doesn't exist.
+func (f Config) GetDefault(section string, key string, dfl string) string {
+	if value, ok := f.Get(section, key); ok {
+		return value
+	}
+	return dfl
+}
+
+// GetBool loads a true/false value from the config, with support for things like "yes", "true", "1", "t", etc.
+func (f Config) GetBool(section string, key string, dfl bool) bool {
+	if value, ok := f.Get(section, key); ok {
+		return LooksTrue(value)
+	}
+	return dfl
+}
+
+// GetInt loads an entry from the config, parsed as an integer value.
+func (f Config) GetInt(section string, key string, dfl int64) int64 {
+	if value, ok := f.Get(section, key); ok {
+		if val, err := strconv.ParseInt(value, 10, 64); err == nil {
+			return val
+		}
+		panic(fmt.Sprintf("Error parsing integer %s/%s from config.", section, key))
+	}
+	return dfl
+}
+
+// GetLimit loads an entry from the config in the format of %d/%d.
+func (f Config) GetLimit(section string, key string, dfla int64, dflb int64) (int64, int64) {
+	if value, ok := f.Get(section, key); ok {
+		fmt.Sscanf(value, "%d/%d", &dfla, &dflb)
+	}
+	return dfla, dflb
+}
+
+// HasSection determines whether or not the section exists in the ini file.
+func (f Config) HasSection(section string) bool {
+	return f.File[section] != nil
+}
+
+// LoadConfig loads an ini from a path.  The path should be a *.conf file or a *.conf.d directory.
+func LoadConfig(path string) (Config, error) {
+	file := Config{make(ini.File)}
+	if fi, err := os.Stat(path); err != nil {
+		return file, err
+	} else if fi.IsDir() {
+		files, err := filepath.Glob(filepath.Join(path, "*.conf"))
+		if err != nil {
+			return file, err
+		}
+		sort.Strings(files)
+		for _, subfile := range files {
+			sf, err := LoadConfig(subfile)
+			if err != nil {
+				return file, err
+			}
+			for sec, val := range sf.File {
+				file.File[sec] = val
+			}
+		}
+		return file, nil
+	}
+	return file, file.LoadFile(path)
+}
+
+// LoadConfigs finds and loads any configs that exist for the given path.  Multiple configs are supported for things like SAIO setups.
+func LoadConfigs(path string) ([]Config, error) {
+	configPaths := []string{}
+	configs := []Config{}
+	if fi, err := os.Stat(path); err == nil && fi.IsDir() && !strings.HasSuffix(path, ".conf.d") {
+		if multiConfigs, err := filepath.Glob(filepath.Join(path, "*.conf")); err == nil {
+			configPaths = append(configPaths, multiConfigs...)
+		}
+		if multiConfigs, err := filepath.Glob(filepath.Join(path, "*.conf.d")); err == nil {
+			configPaths = append(configPaths, multiConfigs...)
+		}
+	} else {
+		configPaths = append(configPaths, path)
+	}
+	for _, p := range configPaths {
+		if config, err := LoadConfig(p); err == nil {
+			configs = append(configs, config)
+		}
+	}
+	if len(configs) == 0 {
+		return nil, errors.New("Unable to find any configs")
+	}
+	return configs, nil
+}
+
+// StringConfig returns an Config from a string, for use in tests.
+func StringConfig(data string) (Config, error) {
+	file := Config{make(ini.File)}
+	return file, file.Load(bytes.NewBufferString(data))
+}
+
+// UidFromConf returns the uid and gid for the user set in the first config found.
+func UidFromConf(path string) (uint32, uint32, error) {
+	configs, err := LoadConfigs(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	for _, config := range configs {
+		username := config.GetDefault("DEFAULT", "user", "swift")
+		usr, err := user.Lookup(username)
+		if err != nil {
+			return 0, 0, err
+		}
+		uid, err := strconv.ParseUint(usr.Uid, 10, 32)
+		if err != nil {
+			return 0, 0, err
+		}
+		gid, err := strconv.ParseUint(usr.Gid, 10, 32)
+		if err != nil {
+			return 0, 0, err
+		}
+		return uint32(uid), uint32(gid), nil
+	}
+	return 0, 0, fmt.Errorf("Unable to find config")
+}

go/hummingbird/conf_test.go

@@ -0,0 +1,126 @@
+//  Copyright (c) 2015 Rackspace
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+//  implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package hummingbird
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfig(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "INI")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempFile.Name())
+	tempFile.WriteString("[DEFAULT]\ndefvalue=1\n[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\nlimit=3/5\n")
+	iniFile, err := LoadConfig(tempFile.Name())
+	require.Equal(t, true, iniFile.GetBool("stuff", "truevalue", false))
+	require.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true))
+	require.Equal(t, true, iniFile.GetBool("stuff", "defaultvalue", true))
+	require.Equal(t, int64(3), iniFile.GetInt("stuff", "intvalue", 2))
+	require.Equal(t, int64(2), iniFile.GetInt("stuff", "missingvalue", 2))
+	require.Equal(t, "false", iniFile.GetDefault("stuff", "falsevalue", "true"))
+	require.Equal(t, "true", iniFile.GetDefault("stuff", "missingvalue", "true"))
+	require.Equal(t, "LOG_LOCAL1", iniFile.GetDefault("stuff", "log_facility", "LOG_LOCAL0"))
+	require.Equal(t, int64(1), iniFile.GetInt("stuff", "defvalue", 0))
+	limit1, limit2 := iniFile.GetLimit("stuff", "limit", 1, 1)
+	require.Equal(t, limit1, int64(3))
+	require.Equal(t, limit2, int64(5))
+}
+
+func TestConfD(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempDir)
+	ioutil.WriteFile(filepath.Join(tempDir, "0.conf"), []byte("[stuff]\ntruevalue=true\n[otherstuff]\nintvalue=3\n"), 0666)
+	ioutil.WriteFile(filepath.Join(tempDir, "1.conf"), []byte("[stuff]\nfalsevalue=false\n"), 0666)
+	iniFile, err := LoadConfig(tempDir)
+	require.Nil(t, err)
+	require.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true))      // falsevalue was set by later conf
+	require.Equal(t, false, iniFile.GetBool("stuff", "truevalue", false))      // truevalue from earlier conf was unset
+	require.Equal(t, int(3), int(iniFile.GetInt("otherstuff", "intvalue", 0))) // otherstuff from earlier conf was preserved
+}
+
+func TestLoadConfigs(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempDir)
+	require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift1"), 0755))
+	ioutil.WriteFile(filepath.Join(tempDir, "etcswift1", "object-server.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666)
+	require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift2", "object-server.conf.d"), 0755))
+	ioutil.WriteFile(filepath.Join(tempDir, "etcswift2", "object-server.conf.d", "default.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666)
+	require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift3", "object-server", "2.conf.d"), 0755))
+	ioutil.WriteFile(filepath.Join(tempDir, "etcswift3", "object-server", "1.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666)
+	ioutil.WriteFile(filepath.Join(tempDir, "etcswift3", "object-server", "2.conf.d", "default.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666)
+
+	for _, configPath := range []string{
+		filepath.Join(tempDir, "etcswift1", "object-server.conf"),
+		filepath.Join(tempDir, "etcswift2", "object-server.conf.d"),
+		filepath.Join(tempDir, "etcswift3", "object-server"),
+	} {
+		configs, err := LoadConfigs(configPath)
+		require.Nil(t, err)
+		for _, config := range configs {
+			require.Equal(t, true, config.GetBool("stuff", "truevalue", false))
+		}
+	}
+}
+
+func TestUidFromConf(t *testing.T) {
+	usr, err := user.Current()
+	require.Nil(t, err)
+	tempFile, err := ioutil.TempFile("", "INI")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempFile.Name())
+	defer tempFile.Close()
+	fmt.Fprintf(tempFile, "[DEFAULT]\nuser=%s\n", usr.Username)
+
+	currentUid, err := strconv.ParseUint(usr.Uid, 10, 32)
+	require.Nil(t, err)
+	currentGid, err := strconv.ParseUint(usr.Gid, 10, 32)
+	require.Nil(t, err)
+	uid, gid, err := UidFromConf(tempFile.Name())
+	require.Nil(t, err)
+	require.Equal(t, uint32(currentUid), uint32(uid))
+	require.Equal(t, uint32(currentGid), uint32(gid))
+}
+
+func TestUidFromConfFailure(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "INI")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempFile.Name())
+	defer tempFile.Close()
+	fmt.Fprintf(tempFile, "[DEFAULT]\nuser=SomeUserWhoShouldntExist\n")
+	_, _, err = UidFromConf(tempFile.Name())
+	require.NotNil(t, err)
+}
+
+func TestHasSection(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "INI")
+	require.Nil(t, err)
+	defer os.RemoveAll(tempFile.Name())
+	tempFile.WriteString("[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\n")
+	iniFile, err := LoadConfig(tempFile.Name())
+	require.Nil(t, err)
+	require.True(t, iniFile.HasSection("stuff"))
+	require.False(t, iniFile.HasSection("otherstuff"))
+}

go/hummingbird/memcachering.go

@@ -63,25 +63,25 @@ type memcacheRing struct {
 }
 
 func NewMemcacheRing(conf string) (*memcacheRing, error) {
-	iniFile, err := LoadIniFile(conf)
+	config, err := LoadConfig(conf)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to load conf file: %s", conf)
 	}
-	return NewMemcacheRingFromIniFile(iniFile)
+	return NewMemcacheRingFromConfig(config)
 }
 
-func NewMemcacheRingFromIniFile(iniFile IniFile) (*memcacheRing, error) {
+func NewMemcacheRingFromConfig(config Config) (*memcacheRing, error) {
 	ring := &memcacheRing{}
 	ring.ring = make(map[string]string)
 	ring.serverKeys = make([]string, 0)
 	ring.servers = make(map[string]*server)
 
-	ring.maxFreeConnectionsPerServer = iniFile.GetInt("memcache", "max_free_connections_per_server", 100)
-	ring.connTimeout = iniFile.GetInt("memcache", "conn_timeout", 100)
-	ring.responseTimeout = iniFile.GetInt("memcache", "response_timeout", 100)
-	ring.nodeWeight = iniFile.GetInt("memcache", "node_weight", 50)
-	ring.tries = iniFile.GetInt("memcache", "tries", 5)
-	for _, s := range strings.Split(iniFile.GetDefault("memcache", "memcache_servers", ""), ",") {
+	ring.maxFreeConnectionsPerServer = config.GetInt("memcache", "max_free_connections_per_server", 100)
+	ring.connTimeout = config.GetInt("memcache", "conn_timeout", 100)
+	ring.responseTimeout = config.GetInt("memcache", "response_timeout", 100)
+	ring.nodeWeight = config.GetInt("memcache", "node_weight", 50)
+	ring.tries = config.GetInt("memcache", "tries", 5)
+	for _, s := range strings.Split(config.GetDefault("memcache", "memcache_servers", ""), ",") {
 		err := ring.addServer(s)
 		if err != nil {
 			return nil, err

go/hummingbird/memcachering_test.go

@@ -18,7 +18,7 @@ import (
 func TestLocalHost(t *testing.T) {
 	// construct ini and check for running memcache servers
 	var buffer bytes.Buffer
-	iniFile := IniFile{File: make(ini.File)}
+	iniFile := Config{File: make(ini.File)}
 	for i := 0; i < 4; i++ {
 		server := fmt.Sprintf("localhost:%d", 10000+i)
 		if i > 0 {
@@ -39,7 +39,7 @@ func TestLocalHost(t *testing.T) {
 	section["dial_timeout"] = "1000"
 	section["max_free_connections_per_server"] = "3"
 	section["memcache_servers"] = buffer.String()
-	ring, err := NewMemcacheRingFromIniFile(iniFile)
+	ring, err := NewMemcacheRingFromConfig(iniFile)
 	if err != nil {
 		t.Fatal("Unable to get memcache ring")
 	}
@@ -49,7 +49,7 @@ func TestLocalHost(t *testing.T) {
 func TestUnixSocket(t *testing.T) {
 	// construct ini and start memcache servers
 	var buffer bytes.Buffer
-	iniFile := IniFile{File: make(ini.File)}
+	iniFile := Config{File: make(ini.File)}
 	for i := 0; i < 4; i++ {
 		sock := fmt.Sprintf("/tmp/test-memecachering-%d", i)
 		if err := os.Remove(sock); err != nil {
@@ -85,7 +85,7 @@ func TestUnixSocket(t *testing.T) {
 	section["dial_timeout"] = "1000"
 	section["max_free_connections_per_server"] = "3"
 	section["memcache_servers"] = buffer.String()
-	ring, err := NewMemcacheRingFromIniFile(iniFile)
+	ring, err := NewMemcacheRingFromConfig(iniFile)
 	if err != nil {
 		t.Fatal("Unable to get memcache ring")
 	}

go/hummingbird/server.go

@@ -24,7 +24,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"runtime/debug"
 	"strconv"
 	"sync"
@@ -224,7 +223,7 @@ func RetryListen(ip string, port int) (net.Listener, error) {
 }
 
 type Server interface {
-	GetHandler() http.Handler
+	GetHandler(Config) http.Handler
 }
 
 /*
@@ -234,7 +233,7 @@ type Server interface {
 
 	Graceful shutdown/restart gives any open connections 5 minutes to complete, then exits.
 */
-func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysLogLike, error), flags *flag.FlagSet) {
+func RunServers(GetServer func(Config, *flag.FlagSet) (string, int, Server, SysLogLike, error), flags *flag.FlagSet) {
 	var servers []*HummingbirdServer
 
 	if flags.NArg() != 0 {
@@ -242,12 +241,14 @@ func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysL
 		return
 	}
 	configFile := flags.Lookup("c").Value.(flag.Getter).Get().(string)
-	configFiles, err := filepath.Glob(filepath.Join(configFile, "*.conf"))
-	if err != nil || len(configFiles) <= 0 {
-		configFiles = []string{configFile}
+	configs, err := LoadConfigs(configFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error finding configs: %v\n", err)
+		return
 	}
-	for _, configFile := range configFiles {
-		ip, port, server, logger, err := GetServer(configFile, flags)
+
+	for _, config := range configs {
+		ip, port, server, logger, err := GetServer(config, flags)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%v\n", err)
 			os.Exit(1)
@@ -260,7 +261,7 @@ func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysL
 		}
 		srv := HummingbirdServer{
 			Server: http.Server{
-				Handler:      server.GetHandler(),
+				Handler:      server.GetHandler(config),
 				ReadTimeout:  24 * time.Hour,
 				WriteTimeout: 24 * time.Hour,
 			},
@@ -305,7 +306,7 @@ type Daemon interface {
 	LogError(format string, args ...interface{})
 }
 
-func RunDaemon(GetDaemon func(string, *flag.FlagSet) (Daemon, error), flags *flag.FlagSet) {
+func RunDaemon(GetDaemon func(Config, *flag.FlagSet) (Daemon, error), flags *flag.FlagSet) {
 	var daemons []Daemon
 
 	if flags.NArg() != 0 {
@@ -314,15 +315,16 @@ func RunDaemon(GetDaemon func(string, *flag.FlagSet) (Daemon, error), flags *fla
 	}
 
 	configFile := flags.Lookup("c").Value.(flag.Getter).Get().(string)
-	configFiles, err := filepath.Glob(filepath.Join(configFile, "*.conf"))
-	if err != nil || len(configFiles) <= 0 {
-		configFiles = []string{configFile}
+	configs, err := LoadConfigs(configFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error finding configs: %v\n", err)
+		return
 	}
 
 	once := flags.Lookup("once").Value.(flag.Getter).Get() == true
 
-	for _, configFile := range configFiles {
-		if daemon, err := GetDaemon(configFile, flags); err == nil {
+	for _, config := range configs {
+		if daemon, err := GetDaemon(config, flags); err == nil {
 			if once {
 				daemon.Run()
 				fmt.Fprintf(os.Stderr, "Daemon pass completed.\n")

go/hummingbird/utils.go

@@ -25,7 +25,6 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
-	"os/user"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -36,7 +35,6 @@ import (
 	"time"
 
 	"github.com/cactus/go-statsd-client/statsd"
-	ini "github.com/vaughan0/go-ini"
 )
 
 var (
@@ -63,85 +61,6 @@ type httpRange struct {
 
 var GMT *time.Location
 
-type IniFile struct{ ini.File }
-
-func (f IniFile) Get(section string, key string) (string, bool) {
-	if value, ok := f.File.Get(section, key); ok {
-		return value, true
-	} else if value, ok := f.File.Get("DEFAULT", key); ok {
-		return value, true
-	} else if value, ok := f.File.Get(section, "set "+key); ok {
-		return value, true
-	} else if value, ok := f.File.Get("DEFAULT", "set "+key); ok {
-		return value, true
-	}
-	return "", false
-}
-
-func (f IniFile) GetDefault(section string, key string, dfl string) string {
-	if value, ok := f.Get(section, key); ok {
-		return value
-	}
-	return dfl
-}
-
-func (f IniFile) GetBool(section string, key string, dfl bool) bool {
-	if value, ok := f.Get(section, key); ok {
-		return LooksTrue(value)
-	}
-	return dfl
-}
-
-func (f IniFile) GetInt(section string, key string, dfl int64) int64 {
-	if value, ok := f.Get(section, key); ok {
-		if val, err := strconv.ParseInt(value, 10, 64); err == nil {
-			return val
-		}
-		panic(fmt.Sprintf("Error parsing integer %s/%s from config.", section, key))
-	}
-	return dfl
-}
-
-func (f IniFile) GetLimit(section string, key string, dfla int64, dflb int64) (int64, int64) {
-	if value, ok := f.Get(section, key); ok {
-		fmt.Sscanf(value, "%d/%d", &dfla, &dflb)
-	}
-	return dfla, dflb
-}
-
-func (f IniFile) HasSection(section string) bool {
-	return f.File[section] != nil
-}
-
-func LoadIniFile(filename string) (IniFile, error) {
-	file := IniFile{make(ini.File)}
-	return file, file.LoadFile(filename)
-}
-
-func UidFromConf(serverConf string) (uint32, uint32, error) {
-	if ini, err := LoadIniFile(serverConf); err == nil {
-		username := ini.GetDefault("DEFAULT", "user", "swift")
-		usr, err := user.Lookup(username)
-		if err != nil {
-			return 0, 0, err
-		}
-		uid, err := strconv.ParseUint(usr.Uid, 10, 32)
-		if err != nil {
-			return 0, 0, err
-		}
-		gid, err := strconv.ParseUint(usr.Gid, 10, 32)
-		if err != nil {
-			return 0, 0, err
-		}
-		return uint32(uid), uint32(gid), nil
-	} else {
-		if matches, err := filepath.Glob(serverConf + "/*.conf"); err == nil && len(matches) > 0 {
-			return UidFromConf(matches[0])
-		}
-	}
-	return 0, 0, fmt.Errorf("Unable to find config file")
-}
-
 func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error {
 	partDir := filepath.Dir(filename)
 	tmpFile, err := ioutil.TempFile(partDir, ".tmp-o-file")
@@ -572,7 +491,7 @@ func GetHashPrefixAndSuffix() (prefix string, suffix string, err error) {
 	config_locations := []string{"/etc/hummingbird/hummingbird.conf", "/etc/swift/swift.conf"}
 
 	for _, loc := range config_locations {
-		if conf, e := LoadIniFile(loc); e == nil {
+		if conf, e := LoadConfig(loc); e == nil {
 			var ok bool
 			prefix, _ = conf.Get("swift-hash", "swift_hash_path_prefix")
 			if suffix, ok = conf.Get("swift-hash", "swift_hash_path_suffix"); !ok {

go/hummingbird/utils_test.go

@@ -17,11 +17,8 @@ package hummingbird
 
 import (
 	"bytes"
-	"fmt"
 	"io/ioutil"
 	"os"
-	"os/user"
-	"strconv"
 	"testing"
 	"time"
 
@@ -218,22 +215,6 @@ func TestUrlencode(t *testing.T) {
 	assert.True(t, Urlencode("鐋댋") == "%E9%90%8B%EB%8C%8B")
 }
 
-func TestIniFile(t *testing.T) {
-	tempFile, err := ioutil.TempFile("", "INI")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tempFile.Name())
-	tempFile.WriteString("[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\n")
-	iniFile, err := LoadIniFile(tempFile.Name())
-	assert.Equal(t, true, iniFile.GetBool("stuff", "truevalue", false))
-	assert.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true))
-	assert.Equal(t, true, iniFile.GetBool("stuff", "defaultvalue", true))
-	assert.Equal(t, int64(3), iniFile.GetInt("stuff", "intvalue", 2))
-	assert.Equal(t, int64(2), iniFile.GetInt("stuff", "missingvalue", 2))
-	assert.Equal(t, "false", iniFile.GetDefault("stuff", "falsevalue", "true"))
-	assert.Equal(t, "true", iniFile.GetDefault("stuff", "missingvalue", "true"))
-	assert.Equal(t, "LOG_LOCAL1", iniFile.GetDefault("stuff", "log_facility", "LOG_LOCAL0"))
-}
-
 func TestIsMount(t *testing.T) {
 	isMount, err := IsMount("/proc")
 	assert.Nil(t, err)
@@ -300,32 +281,3 @@ func TestGetHashPrefixAndSuffix(t *testing.T) {
 		t.Error("Error prefix and suffix not being set")
 	}
 }
-
-func TestUidFromConf(t *testing.T) {
-	usr, err := user.Current()
-	assert.Nil(t, err)
-	tempFile, err := ioutil.TempFile("", "INI")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tempFile.Name())
-	defer tempFile.Close()
-	fmt.Fprintf(tempFile, "[DEFAULT]\nuser=%s\n", usr.Username)
-
-	currentUid, err := strconv.ParseUint(usr.Uid, 10, 32)
-	assert.Nil(t, err)
-	currentGid, err := strconv.ParseUint(usr.Gid, 10, 32)
-	assert.Nil(t, err)
-	uid, gid, err := UidFromConf(tempFile.Name())
-	assert.Nil(t, err)
-	assert.Equal(t, uint32(currentUid), uint32(uid))
-	assert.Equal(t, uint32(currentGid), uint32(gid))
-}
-
-func TestUidFromConfFailure(t *testing.T) {
-	tempFile, err := ioutil.TempFile("", "INI")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tempFile.Name())
-	defer tempFile.Close()
-	fmt.Fprintf(tempFile, "[DEFAULT]\nuser=SomeUserWhoShouldntExist\n")
-	_, _, err = UidFromConf(tempFile.Name())
-	assert.NotNil(t, err)
-}

go/objectserver/auditor.go

@@ -326,12 +326,11 @@ func (d *AuditorDaemon) RunForever() {
 }
 
 // NewAuditor returns a new AuditorDaemon with the given conf.
-func NewAuditor(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error) {
-	d := &AuditorDaemon{}
-	serverconf, err := hummingbird.LoadIniFile(conf)
-	if err != nil || !serverconf.HasSection("object-auditor") {
-		return nil, fmt.Errorf("Unable to find auditor config: %s", conf)
+func NewAuditor(serverconf hummingbird.Config, flags *flag.FlagSet) (hummingbird.Daemon, error) {
+	if !serverconf.HasSection("object-auditor") {
+		return nil, fmt.Errorf("Unable to find object-auditor config section")
 	}
+	d := &AuditorDaemon{}
 	d.driveRoot = serverconf.GetDefault("object-auditor", "devices", "/srv/node")
 	d.checkMounts = serverconf.GetBool("object-auditor", "mount_check", true)
 	d.logger = hummingbird.SetupLogger(serverconf.GetDefault("object-auditor", "log_facility", "LOG_LOCAL0"), "object-auditor", "")

go/objectserver/auditor_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openstack/swift/go/hummingbird"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAuditHashPasses(t *testing.T) {
@@ -192,25 +193,23 @@ func (s *auditLogSaver) Debug(line string) error {
 }
 
 func makeAuditor(settings ...string) *Auditor {
-	conf, _ := ioutil.TempFile("", "")
-	conf.WriteString("[object-auditor]\n")
+	configString := "[object-auditor]\n"
 	for i := 0; i < len(settings); i += 2 {
-		fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1])
+		configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1])
 	}
-	defer conf.Close()
-	defer os.RemoveAll(conf.Name())
-	auditorDaemon, _ := NewAuditor(conf.Name(), &flag.FlagSet{})
+	conf, _ := hummingbird.StringConfig(configString)
+	auditorDaemon, _ := NewAuditor(conf, &flag.FlagSet{})
 	auditorDaemon.(*AuditorDaemon).logger = &auditLogSaver{}
 	return &Auditor{AuditorDaemon: auditorDaemon.(*AuditorDaemon), filesPerSecond: 1}
 }
 
 func TestFailsWithoutSection(t *testing.T) {
-	conf, _ := ioutil.TempFile("", "")
-	defer conf.Close()
-	defer os.RemoveAll(conf.Name())
-	auditorDaemon, err := NewAuditor(conf.Name(), &flag.FlagSet{})
+	conf, err := hummingbird.StringConfig("")
+	require.Nil(t, err)
+	auditorDaemon, err := NewAuditor(conf, &flag.FlagSet{})
+	require.NotNil(t, err)
 	assert.Nil(t, auditorDaemon)
-	assert.True(t, strings.HasPrefix(err.Error(), "Unable to find auditor config"))
+	assert.True(t, strings.HasPrefix(err.Error(), "Unable to find object-auditor"))
 }
 
 func TestAuditSuffixNotDir(t *testing.T) {

go/objectserver/main.go

@@ -455,7 +455,7 @@ func (server *ObjectServer) updateDeviceLocks(seconds int64) {
 	}
 }
 
-func (server *ObjectServer) GetHandler() http.Handler {
+func (server *ObjectServer) GetHandler(config hummingbird.Config) http.Handler {
 	commonHandlers := alice.New(middleware.ClearHandler, server.LogRequest, middleware.ValidateRequest, server.AcquireDevice)
 	router := hummingbird.NewRouter()
 	router.Get("/healthcheck", commonHandlers.ThenFunc(server.HealthcheckHandler))
@@ -477,7 +477,7 @@ func (server *ObjectServer) GetHandler() http.Handler {
 	return alice.New(middleware.GrepObject).Then(router)
 }
 
-func GetServer(conf string, flags *flag.FlagSet) (bindIP string, bindPort int, serv hummingbird.Server, logger hummingbird.SysLogLike, err error) {
+func GetServer(serverconf hummingbird.Config, flags *flag.FlagSet) (bindIP string, bindPort int, serv hummingbird.Server, logger hummingbird.SysLogLike, err error) {
 	server := &ObjectServer{driveRoot: "/srv/node", hashPathPrefix: "", hashPathSuffix: "",
 		allowedHeaders: map[string]bool{"Content-Disposition": true,
 			"Content-Encoding":      true,
@@ -491,11 +491,6 @@ func GetServer(conf string, flags *flag.FlagSet) (bindIP string, bindPort int, s
 		return "", 0, nil, nil, err
 	}
 
-	serverconf, err := hummingbird.LoadIniFile(conf)
-	if err != nil {
-		return "", 0, nil, nil, errors.New(fmt.Sprintf("Unable to load %s", conf))
-	}
-
 	objEngineName := serverconf.GetDefault("app:object-server", "object_engine", DefaultObjEngine)
 	if newEngineFactory, err := FindEngine(objEngineName); err != nil {
 		return "", 0, nil, nil, errors.New("Unable to find object engine")

go/objectserver/main_test.go

@@ -63,21 +63,16 @@ func makeObjectServer(settings ...string) (*TestServer, error) {
 	if err != nil {
 		return nil, err
 	}
-	conf, err := ioutil.TempFile(driveRoot, "")
+	configString := fmt.Sprintf("[app:object-server]\ndevices=%s\nmount_check=false\n", driveRoot)
+	for i := 0; i < len(settings); i += 2 {
+		configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1])
+	}
+	conf, err := hummingbird.StringConfig(configString)
 	if err != nil {
 		return nil, err
 	}
-	conf.WriteString("[app:object-server]\n")
-	fmt.Fprintf(conf, "devices=%s\n", driveRoot)
-	fmt.Fprintf(conf, "mount_check=false\n")
-	for i := 0; i < len(settings); i += 2 {
-		fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1])
-	}
-	if err := conf.Close(); err != nil {
-		return nil, err
-	}
-	_, _, server, _, _ := GetServer(conf.Name(), &flag.FlagSet{})
-	ts := httptest.NewServer(server.GetHandler())
+	_, _, server, _, _ := GetServer(conf, &flag.FlagSet{})
+	ts := httptest.NewServer(server.GetHandler(conf))
 	u, err := url.Parse(ts.URL)
 	if err != nil {
 		return nil, err
@@ -313,7 +308,7 @@ func TestDisconnectOnPut(t *testing.T) {
 
 	resp := &fakeResponse{}
 
-	ts.objServer.GetHandler().ServeHTTP(resp, req)
+	ts.Server.Config.Handler.ServeHTTP(resp, req)
 	assert.Equal(t, resp.status, 499)
 }
 

go/objectserver/objengine.go

@@ -61,7 +61,7 @@ type ObjectEngine interface {
 }
 
 // ObjectEngineConstructor> is a function that, given configs and flags, returns an ObjectEngine
-type ObjectEngineConstructor func(hummingbird.IniFile, *flag.FlagSet) (ObjectEngine, error)
+type ObjectEngineConstructor func(hummingbird.Config, *flag.FlagSet) (ObjectEngine, error)
 
 type engineFactoryEntry struct {
 	name        string

go/objectserver/objengine_test.go

@@ -26,7 +26,7 @@ import (
 
 func TestObjectEngineRegistry(t *testing.T) {
 	testErr := errors.New("Not implemented")
-	constructor := func(hummingbird.IniFile, *flag.FlagSet) (ObjectEngine, error) {
+	constructor := func(hummingbird.Config, *flag.FlagSet) (ObjectEngine, error) {
 		return nil, testErr
 	}
 
@@ -34,7 +34,7 @@ func TestObjectEngineRegistry(t *testing.T) {
 
 	fconstructor, err := FindEngine("test")
 	require.Nil(t, err)
-	eng, err := fconstructor(hummingbird.IniFile{}, nil)
+	eng, err := fconstructor(hummingbird.Config{}, nil)
 	require.Nil(t, eng)
 	require.Equal(t, err, testErr)
 

go/objectserver/replicator.go

@@ -740,7 +740,11 @@ func (r *Replicator) RunForever() {
 	r.run(time.Tick(RunForeverInterval))
 }
 
-func NewReplicator(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error) {
+func NewReplicator(serverconf hummingbird.Config, flags *flag.FlagSet) (hummingbird.Daemon, error) {
+	if !serverconf.HasSection("object-replicator") {
+		return nil, fmt.Errorf("Unable to find object-auditor config section")
+	}
+
 	replicator := &Replicator{
 		partitionTimesAdd:         make(chan float64),
 		replicationCountIncrement: make(chan uint64),
@@ -752,10 +756,6 @@ func NewReplicator(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to get hash prefix and suffix")
 	}
-	serverconf, err := hummingbird.LoadIniFile(conf)
-	if err != nil || !serverconf.HasSection("object-replicator") {
-		return nil, fmt.Errorf("Unable to find replicator config: %s", conf)
-	}
 	replicator.reconCachePath = serverconf.GetDefault("object-auditor", "recon_cache_path", "/var/cache/swift")
 	replicator.checkMounts = serverconf.GetBool("object-replicator", "mount_check", true)
 	replicator.driveRoot = serverconf.GetDefault("object-replicator", "devices", "/srv/node")

go/objectserver/replicator_test.go

@@ -87,14 +87,12 @@ func makeReplicator(settings ...string) *Replicator {
 }
 
 func makeReplicatorWithFlags(settings []string, flags *flag.FlagSet) *Replicator {
-	conf, _ := ioutil.TempFile("", "")
-	conf.WriteString("[object-replicator]\nmount_check=false\n")
+	configString := "[object-replicator]\nmount_check=false\n"
 	for i := 0; i < len(settings); i += 2 {
-		fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1])
+		configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1])
 	}
-	defer conf.Close()
-	defer os.RemoveAll(conf.Name())
-	replicator, _ := NewReplicator(conf.Name(), flags)
+	conf, _ := hummingbird.StringConfig(configString)
+	replicator, _ := NewReplicator(conf, flags)
 	rep := replicator.(*Replicator)
 	rep.concurrencySem = make(chan struct{}, 1)
 	return rep
@@ -194,12 +192,6 @@ func (r *FakeHandoffRing) GetJobNodes(partition uint64, localDevice int) (respon
 	return []*hummingbird.Device{r.dev}, true
 }
 
-func TestReplicatorInitFail(t *testing.T) {
-	replicator, err := NewReplicator("nonexistentfile", &flag.FlagSet{})
-	assert.Nil(t, replicator)
-	assert.NotNil(t, err)
-}
-
 func TestReplicatorVmDuration(t *testing.T) {
 	replicator := makeReplicator("vm_test_mode", "true")
 	assert.Equal(t, 2000*time.Millisecond, replicator.timePerPart)

go/objectserver/swiftobjeng.go

@@ -355,7 +355,7 @@ func (f *SwiftObjectFactory) RegisterHandlers(addRoute func(method, path string,
 }
 
 // SwiftEngineConstructor creates a SwiftObjectFactory given the object server configs.
-func SwiftEngineConstructor(config hummingbird.IniFile, flags *flag.FlagSet) (ObjectEngine, error) {
+func SwiftEngineConstructor(config hummingbird.Config, flags *flag.FlagSet) (ObjectEngine, error) {
 	driveRoot := config.GetDefault("app:object-server", "devices", "/srv/node")
 	reserve := config.GetInt("app:object-server", "fallocate_reserve", 0)
 	hashPathPrefix, hashPathSuffix, err := hummingbird.GetHashPrefixAndSuffix()

go/probe/base.go

@@ -156,24 +156,19 @@ func NewEnvironment(settings ...string) *Environment {
 		host, ports, _ := net.SplitHostPort(u.Host)
 		port, _ := strconv.Atoi(ports)
 
-		conf, _ := ioutil.TempFile("", "")
-		conf.WriteString("[DEFAULT]\nmount_check=false\n")
-		fmt.Fprintf(conf, "devices=%s\n", driveRoot)
-		fmt.Fprintf(conf, "bind_port=%d\n", port)
-		fmt.Fprintf(conf, "bind_ip=%s\n", host)
+		configString := "[DEFAULT]\nmount_check=false\n"
+		configString += fmt.Sprintf("devices=%s\n", driveRoot)
+		configString += fmt.Sprintf("bind_port=%d\n", port)
+		configString += fmt.Sprintf("bind_ip=%s\n", host)
 		for i := 0; i < len(settings); i += 2 {
-			fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1])
+			configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1])
 		}
-		conf.WriteString("[app:object-server]\n")
-		conf.WriteString("[object-replicator]\n")
-		conf.WriteString("[object-auditor]\n")
-		defer conf.Close()
-		defer os.RemoveAll(conf.Name())
-
-		_, _, server, _, _ := objectserver.GetServer(conf.Name(), &flag.FlagSet{})
-		ts.Config.Handler = server.GetHandler()
-		replicator, _ := objectserver.NewReplicator(conf.Name(), &flag.FlagSet{})
-		auditor, _ := objectserver.NewAuditor(conf.Name(), &flag.FlagSet{})
+		configString += "[app:object-server]\n[object-replicator]\n[object-auditor]\n"
+		conf, _ := hummingbird.StringConfig(configString)
+		_, _, server, _, _ := objectserver.GetServer(conf, &flag.FlagSet{})
+		ts.Config.Handler = server.GetHandler(conf)
+		replicator, _ := objectserver.NewReplicator(conf, &flag.FlagSet{})
+		auditor, _ := objectserver.NewAuditor(conf, &flag.FlagSet{})
 		replicator.(*objectserver.Replicator).Ring = env.ring
 		env.ring.(*FakeRing).devices = append(env.ring.(*FakeRing).devices, &hummingbird.Device{
 			Id: i, Device: "sda", Ip: host, Port: port, Region: 0, ReplicationIp: host, ReplicationPort: port, Weight: 1, Zone: i,

go/proxyserver/main.go

@@ -16,7 +16,6 @@
 package proxyserver
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"net/http"
@@ -68,7 +67,7 @@ func (server *ProxyServer) LogRequest(next http.Handler) http.Handler {
 	return http.HandlerFunc(fn)
 }
 
-func (server *ProxyServer) GetHandler() http.Handler {
+func (server *ProxyServer) GetHandler(config hummingbird.Config) http.Handler {
 	router := hummingbird.NewRouter()
 	router.Get("/healthcheck", http.HandlerFunc(server.HealthcheckHandler))
 
@@ -108,21 +107,17 @@ func (server *ProxyServer) GetHandler() http.Handler {
 	).Then(router)
 }
 
-func GetServer(conf string, flags *flag.FlagSet) (string, int, hummingbird.Server, hummingbird.SysLogLike, error) {
+func GetServer(serverconf hummingbird.Config, flags *flag.FlagSet) (string, int, hummingbird.Server, hummingbird.SysLogLike, error) {
 	var err error
 	server := &ProxyServer{}
 	server.C, err = client.NewProxyDirectClient()
 	if err != nil {
 		return "", 0, nil, nil, err
 	}
-	server.mc, err = hummingbird.NewMemcacheRingFromIniFile(hummingbird.IniFile{})
+	server.mc, err = hummingbird.NewMemcacheRingFromConfig(serverconf)
 	if err != nil {
 		return "", 0, nil, nil, err
 	}
-	serverconf, err := hummingbird.LoadIniFile(conf)
-	if err != nil {
-		return "", 0, nil, nil, errors.New(fmt.Sprintf("Unable to load %s", conf))
-	}
 
 	bindIP := serverconf.GetDefault("DEFAULT", "bind_ip", "0.0.0.0")
 	bindPort := serverconf.GetInt("DEFAULT", "bind_port", 8080)