diff --git a/go/bench/dbench.go b/go/bench/dbench.go index 9c0e13a25a..eb8e449b66 100644 --- a/go/bench/dbench.go +++ b/go/bench/dbench.go @@ -126,7 +126,7 @@ func RunDBench(args []string) { os.Exit(1) } - benchconf, err := hummingbird.LoadIniFile(args[0]) + benchconf, err := hummingbird.LoadConfig(args[0]) if err != nil { fmt.Println("Error parsing ini file:", err) os.Exit(1) diff --git a/go/bench/main.go b/go/bench/main.go index 4af3e4acc4..f80d5766ae 100644 --- a/go/bench/main.go +++ b/go/bench/main.go @@ -139,7 +139,7 @@ func RunBench(args []string) { os.Exit(1) } - benchconf, err := hummingbird.LoadIniFile(args[0]) + benchconf, err := hummingbird.LoadConfig(args[0]) if err != nil { fmt.Println("Error parsing ini file:", err) os.Exit(1) @@ -227,7 +227,7 @@ func RunThrash(args []string) { os.Exit(1) } - thrashconf, err := hummingbird.LoadIniFile(args[0]) + thrashconf, err := hummingbird.LoadConfig(args[0]) if err != nil { fmt.Println("Error parsing ini file:", err) os.Exit(1) diff --git a/go/cmd/hummingbird.go b/go/cmd/hummingbird.go index 6a113b0c49..0e013825f9 100644 --- a/go/cmd/hummingbird.go +++ b/go/cmd/hummingbird.go @@ -73,8 +73,10 @@ func findConfig(name string) string { configName := strings.Split(name, "-")[0] configSearch := []string{ fmt.Sprintf("/etc/hummingbird/%s-server.conf", configName), + fmt.Sprintf("/etc/hummingbird/%s-server.conf.d", configName), fmt.Sprintf("/etc/hummingbird/%s-server", configName), fmt.Sprintf("/etc/swift/%s-server.conf", configName), + fmt.Sprintf("/etc/swift/%s-server.conf.d", configName), fmt.Sprintf("/etc/swift/%s-server", configName), } for _, config := range configSearch { diff --git a/go/hummingbird/conf.go b/go/hummingbird/conf.go new file mode 100644 index 0000000000..094cc8b106 --- /dev/null +++ b/go/hummingbird/conf.go @@ -0,0 +1,168 @@ +// Copyright (c) 2015 Rackspace +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hummingbird + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/user" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/vaughan0/go-ini" +) + +// Config represents an ini file. +type Config struct{ ini.File } + +// Get fetches a value from the Config, looking in the DEFAULT section if not found in the specific section. Also ignores "set " key prefixes, like paste. +func (f Config) Get(section string, key string) (string, bool) { + if value, ok := f.File.Get(section, key); ok { + return value, true + } else if value, ok := f.File.Get("DEFAULT", key); ok { + return value, true + } else if value, ok := f.File.Get(section, "set "+key); ok { + return value, true + } else if value, ok := f.File.Get("DEFAULT", "set "+key); ok { + return value, true + } + return "", false +} + +// GetDefault returns a value from the config, or returns the default setting if the entry doesn't exist. +func (f Config) GetDefault(section string, key string, dfl string) string { + if value, ok := f.Get(section, key); ok { + return value + } + return dfl +} + +// GetBool loads a true/false value from the config, with support for things like "yes", "true", "1", "t", etc. +func (f Config) GetBool(section string, key string, dfl bool) bool { + if value, ok := f.Get(section, key); ok { + return LooksTrue(value) + } + return dfl +} + +// GetInt loads an entry from the config, parsed as an integer value. +func (f Config) GetInt(section string, key string, dfl int64) int64 { + if value, ok := f.Get(section, key); ok { + if val, err := strconv.ParseInt(value, 10, 64); err == nil { + return val + } + panic(fmt.Sprintf("Error parsing integer %s/%s from config.", section, key)) + } + return dfl +} + +// GetLimit loads an entry from the config in the format of %d/%d. +func (f Config) GetLimit(section string, key string, dfla int64, dflb int64) (int64, int64) { + if value, ok := f.Get(section, key); ok { + fmt.Sscanf(value, "%d/%d", &dfla, &dflb) + } + return dfla, dflb +} + +// HasSection determines whether or not the section exists in the ini file. +func (f Config) HasSection(section string) bool { + return f.File[section] != nil +} + +// LoadConfig loads an ini from a path. The path should be a *.conf file or a *.conf.d directory. +func LoadConfig(path string) (Config, error) { + file := Config{make(ini.File)} + if fi, err := os.Stat(path); err != nil { + return file, err + } else if fi.IsDir() { + files, err := filepath.Glob(filepath.Join(path, "*.conf")) + if err != nil { + return file, err + } + sort.Strings(files) + for _, subfile := range files { + sf, err := LoadConfig(subfile) + if err != nil { + return file, err + } + for sec, val := range sf.File { + file.File[sec] = val + } + } + return file, nil + } + return file, file.LoadFile(path) +} + +// LoadConfigs finds and loads any configs that exist for the given path. Multiple configs are supported for things like SAIO setups. +func LoadConfigs(path string) ([]Config, error) { + configPaths := []string{} + configs := []Config{} + if fi, err := os.Stat(path); err == nil && fi.IsDir() && !strings.HasSuffix(path, ".conf.d") { + if multiConfigs, err := filepath.Glob(filepath.Join(path, "*.conf")); err == nil { + configPaths = append(configPaths, multiConfigs...) + } + if multiConfigs, err := filepath.Glob(filepath.Join(path, "*.conf.d")); err == nil { + configPaths = append(configPaths, multiConfigs...) + } + } else { + configPaths = append(configPaths, path) + } + for _, p := range configPaths { + if config, err := LoadConfig(p); err == nil { + configs = append(configs, config) + } + } + if len(configs) == 0 { + return nil, errors.New("Unable to find any configs") + } + return configs, nil +} + +// StringConfig returns an Config from a string, for use in tests. +func StringConfig(data string) (Config, error) { + file := Config{make(ini.File)} + return file, file.Load(bytes.NewBufferString(data)) +} + +// UidFromConf returns the uid and gid for the user set in the first config found. +func UidFromConf(path string) (uint32, uint32, error) { + configs, err := LoadConfigs(path) + if err != nil { + return 0, 0, err + } + for _, config := range configs { + username := config.GetDefault("DEFAULT", "user", "swift") + usr, err := user.Lookup(username) + if err != nil { + return 0, 0, err + } + uid, err := strconv.ParseUint(usr.Uid, 10, 32) + if err != nil { + return 0, 0, err + } + gid, err := strconv.ParseUint(usr.Gid, 10, 32) + if err != nil { + return 0, 0, err + } + return uint32(uid), uint32(gid), nil + } + return 0, 0, fmt.Errorf("Unable to find config") +} diff --git a/go/hummingbird/conf_test.go b/go/hummingbird/conf_test.go new file mode 100644 index 0000000000..84b7c17cc3 --- /dev/null +++ b/go/hummingbird/conf_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015 Rackspace +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hummingbird + +import ( + "fmt" + "io/ioutil" + "os" + "os/user" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfig(t *testing.T) { + tempFile, err := ioutil.TempFile("", "INI") + require.Nil(t, err) + defer os.RemoveAll(tempFile.Name()) + tempFile.WriteString("[DEFAULT]\ndefvalue=1\n[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\nlimit=3/5\n") + iniFile, err := LoadConfig(tempFile.Name()) + require.Equal(t, true, iniFile.GetBool("stuff", "truevalue", false)) + require.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true)) + require.Equal(t, true, iniFile.GetBool("stuff", "defaultvalue", true)) + require.Equal(t, int64(3), iniFile.GetInt("stuff", "intvalue", 2)) + require.Equal(t, int64(2), iniFile.GetInt("stuff", "missingvalue", 2)) + require.Equal(t, "false", iniFile.GetDefault("stuff", "falsevalue", "true")) + require.Equal(t, "true", iniFile.GetDefault("stuff", "missingvalue", "true")) + require.Equal(t, "LOG_LOCAL1", iniFile.GetDefault("stuff", "log_facility", "LOG_LOCAL0")) + require.Equal(t, int64(1), iniFile.GetInt("stuff", "defvalue", 0)) + limit1, limit2 := iniFile.GetLimit("stuff", "limit", 1, 1) + require.Equal(t, limit1, int64(3)) + require.Equal(t, limit2, int64(5)) +} + +func TestConfD(t *testing.T) { + tempDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + defer os.RemoveAll(tempDir) + ioutil.WriteFile(filepath.Join(tempDir, "0.conf"), []byte("[stuff]\ntruevalue=true\n[otherstuff]\nintvalue=3\n"), 0666) + ioutil.WriteFile(filepath.Join(tempDir, "1.conf"), []byte("[stuff]\nfalsevalue=false\n"), 0666) + iniFile, err := LoadConfig(tempDir) + require.Nil(t, err) + require.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true)) // falsevalue was set by later conf + require.Equal(t, false, iniFile.GetBool("stuff", "truevalue", false)) // truevalue from earlier conf was unset + require.Equal(t, int(3), int(iniFile.GetInt("otherstuff", "intvalue", 0))) // otherstuff from earlier conf was preserved +} + +func TestLoadConfigs(t *testing.T) { + tempDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + defer os.RemoveAll(tempDir) + require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift1"), 0755)) + ioutil.WriteFile(filepath.Join(tempDir, "etcswift1", "object-server.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666) + require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift2", "object-server.conf.d"), 0755)) + ioutil.WriteFile(filepath.Join(tempDir, "etcswift2", "object-server.conf.d", "default.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666) + require.Nil(t, os.MkdirAll(filepath.Join(tempDir, "etcswift3", "object-server", "2.conf.d"), 0755)) + ioutil.WriteFile(filepath.Join(tempDir, "etcswift3", "object-server", "1.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666) + ioutil.WriteFile(filepath.Join(tempDir, "etcswift3", "object-server", "2.conf.d", "default.conf"), []byte("[stuff]\ntruevalue=true\n"), 0666) + + for _, configPath := range []string{ + filepath.Join(tempDir, "etcswift1", "object-server.conf"), + filepath.Join(tempDir, "etcswift2", "object-server.conf.d"), + filepath.Join(tempDir, "etcswift3", "object-server"), + } { + configs, err := LoadConfigs(configPath) + require.Nil(t, err) + for _, config := range configs { + require.Equal(t, true, config.GetBool("stuff", "truevalue", false)) + } + } +} + +func TestUidFromConf(t *testing.T) { + usr, err := user.Current() + require.Nil(t, err) + tempFile, err := ioutil.TempFile("", "INI") + require.Nil(t, err) + defer os.RemoveAll(tempFile.Name()) + defer tempFile.Close() + fmt.Fprintf(tempFile, "[DEFAULT]\nuser=%s\n", usr.Username) + + currentUid, err := strconv.ParseUint(usr.Uid, 10, 32) + require.Nil(t, err) + currentGid, err := strconv.ParseUint(usr.Gid, 10, 32) + require.Nil(t, err) + uid, gid, err := UidFromConf(tempFile.Name()) + require.Nil(t, err) + require.Equal(t, uint32(currentUid), uint32(uid)) + require.Equal(t, uint32(currentGid), uint32(gid)) +} + +func TestUidFromConfFailure(t *testing.T) { + tempFile, err := ioutil.TempFile("", "INI") + require.Nil(t, err) + defer os.RemoveAll(tempFile.Name()) + defer tempFile.Close() + fmt.Fprintf(tempFile, "[DEFAULT]\nuser=SomeUserWhoShouldntExist\n") + _, _, err = UidFromConf(tempFile.Name()) + require.NotNil(t, err) +} + +func TestHasSection(t *testing.T) { + tempFile, err := ioutil.TempFile("", "INI") + require.Nil(t, err) + defer os.RemoveAll(tempFile.Name()) + tempFile.WriteString("[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\n") + iniFile, err := LoadConfig(tempFile.Name()) + require.Nil(t, err) + require.True(t, iniFile.HasSection("stuff")) + require.False(t, iniFile.HasSection("otherstuff")) +} diff --git a/go/hummingbird/memcachering.go b/go/hummingbird/memcachering.go index 1db2064c1a..1f60a31433 100644 --- a/go/hummingbird/memcachering.go +++ b/go/hummingbird/memcachering.go @@ -63,25 +63,25 @@ type memcacheRing struct { } func NewMemcacheRing(conf string) (*memcacheRing, error) { - iniFile, err := LoadIniFile(conf) + config, err := LoadConfig(conf) if err != nil { return nil, fmt.Errorf("Unable to load conf file: %s", conf) } - return NewMemcacheRingFromIniFile(iniFile) + return NewMemcacheRingFromConfig(config) } -func NewMemcacheRingFromIniFile(iniFile IniFile) (*memcacheRing, error) { +func NewMemcacheRingFromConfig(config Config) (*memcacheRing, error) { ring := &memcacheRing{} ring.ring = make(map[string]string) ring.serverKeys = make([]string, 0) ring.servers = make(map[string]*server) - ring.maxFreeConnectionsPerServer = iniFile.GetInt("memcache", "max_free_connections_per_server", 100) - ring.connTimeout = iniFile.GetInt("memcache", "conn_timeout", 100) - ring.responseTimeout = iniFile.GetInt("memcache", "response_timeout", 100) - ring.nodeWeight = iniFile.GetInt("memcache", "node_weight", 50) - ring.tries = iniFile.GetInt("memcache", "tries", 5) - for _, s := range strings.Split(iniFile.GetDefault("memcache", "memcache_servers", ""), ",") { + ring.maxFreeConnectionsPerServer = config.GetInt("memcache", "max_free_connections_per_server", 100) + ring.connTimeout = config.GetInt("memcache", "conn_timeout", 100) + ring.responseTimeout = config.GetInt("memcache", "response_timeout", 100) + ring.nodeWeight = config.GetInt("memcache", "node_weight", 50) + ring.tries = config.GetInt("memcache", "tries", 5) + for _, s := range strings.Split(config.GetDefault("memcache", "memcache_servers", ""), ",") { err := ring.addServer(s) if err != nil { return nil, err diff --git a/go/hummingbird/memcachering_test.go b/go/hummingbird/memcachering_test.go index 10e0da4fd0..66d788d21c 100644 --- a/go/hummingbird/memcachering_test.go +++ b/go/hummingbird/memcachering_test.go @@ -18,7 +18,7 @@ import ( func TestLocalHost(t *testing.T) { // construct ini and check for running memcache servers var buffer bytes.Buffer - iniFile := IniFile{File: make(ini.File)} + iniFile := Config{File: make(ini.File)} for i := 0; i < 4; i++ { server := fmt.Sprintf("localhost:%d", 10000+i) if i > 0 { @@ -39,7 +39,7 @@ func TestLocalHost(t *testing.T) { section["dial_timeout"] = "1000" section["max_free_connections_per_server"] = "3" section["memcache_servers"] = buffer.String() - ring, err := NewMemcacheRingFromIniFile(iniFile) + ring, err := NewMemcacheRingFromConfig(iniFile) if err != nil { t.Fatal("Unable to get memcache ring") } @@ -49,7 +49,7 @@ func TestLocalHost(t *testing.T) { func TestUnixSocket(t *testing.T) { // construct ini and start memcache servers var buffer bytes.Buffer - iniFile := IniFile{File: make(ini.File)} + iniFile := Config{File: make(ini.File)} for i := 0; i < 4; i++ { sock := fmt.Sprintf("/tmp/test-memecachering-%d", i) if err := os.Remove(sock); err != nil { @@ -85,7 +85,7 @@ func TestUnixSocket(t *testing.T) { section["dial_timeout"] = "1000" section["max_free_connections_per_server"] = "3" section["memcache_servers"] = buffer.String() - ring, err := NewMemcacheRingFromIniFile(iniFile) + ring, err := NewMemcacheRingFromConfig(iniFile) if err != nil { t.Fatal("Unable to get memcache ring") } diff --git a/go/hummingbird/server.go b/go/hummingbird/server.go index 28e026f18b..02a2ff4956 100644 --- a/go/hummingbird/server.go +++ b/go/hummingbird/server.go @@ -24,7 +24,6 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "runtime/debug" "strconv" "sync" @@ -224,7 +223,7 @@ func RetryListen(ip string, port int) (net.Listener, error) { } type Server interface { - GetHandler() http.Handler + GetHandler(Config) http.Handler } /* @@ -234,7 +233,7 @@ type Server interface { Graceful shutdown/restart gives any open connections 5 minutes to complete, then exits. */ -func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysLogLike, error), flags *flag.FlagSet) { +func RunServers(GetServer func(Config, *flag.FlagSet) (string, int, Server, SysLogLike, error), flags *flag.FlagSet) { var servers []*HummingbirdServer if flags.NArg() != 0 { @@ -242,12 +241,14 @@ func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysL return } configFile := flags.Lookup("c").Value.(flag.Getter).Get().(string) - configFiles, err := filepath.Glob(filepath.Join(configFile, "*.conf")) - if err != nil || len(configFiles) <= 0 { - configFiles = []string{configFile} + configs, err := LoadConfigs(configFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error finding configs: %v\n", err) + return } - for _, configFile := range configFiles { - ip, port, server, logger, err := GetServer(configFile, flags) + + for _, config := range configs { + ip, port, server, logger, err := GetServer(config, flags) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) @@ -260,7 +261,7 @@ func RunServers(GetServer func(string, *flag.FlagSet) (string, int, Server, SysL } srv := HummingbirdServer{ Server: http.Server{ - Handler: server.GetHandler(), + Handler: server.GetHandler(config), ReadTimeout: 24 * time.Hour, WriteTimeout: 24 * time.Hour, }, @@ -305,7 +306,7 @@ type Daemon interface { LogError(format string, args ...interface{}) } -func RunDaemon(GetDaemon func(string, *flag.FlagSet) (Daemon, error), flags *flag.FlagSet) { +func RunDaemon(GetDaemon func(Config, *flag.FlagSet) (Daemon, error), flags *flag.FlagSet) { var daemons []Daemon if flags.NArg() != 0 { @@ -314,15 +315,16 @@ func RunDaemon(GetDaemon func(string, *flag.FlagSet) (Daemon, error), flags *fla } configFile := flags.Lookup("c").Value.(flag.Getter).Get().(string) - configFiles, err := filepath.Glob(filepath.Join(configFile, "*.conf")) - if err != nil || len(configFiles) <= 0 { - configFiles = []string{configFile} + configs, err := LoadConfigs(configFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error finding configs: %v\n", err) + return } once := flags.Lookup("once").Value.(flag.Getter).Get() == true - for _, configFile := range configFiles { - if daemon, err := GetDaemon(configFile, flags); err == nil { + for _, config := range configs { + if daemon, err := GetDaemon(config, flags); err == nil { if once { daemon.Run() fmt.Fprintf(os.Stderr, "Daemon pass completed.\n") diff --git a/go/hummingbird/utils.go b/go/hummingbird/utils.go index b07d93178f..d7a0cd61b8 100644 --- a/go/hummingbird/utils.go +++ b/go/hummingbird/utils.go @@ -25,7 +25,6 @@ import ( "math/rand" "net/http" "os" - "os/user" "path/filepath" "runtime" "sort" @@ -36,7 +35,6 @@ import ( "time" "github.com/cactus/go-statsd-client/statsd" - ini "github.com/vaughan0/go-ini" ) var ( @@ -63,85 +61,6 @@ type httpRange struct { var GMT *time.Location -type IniFile struct{ ini.File } - -func (f IniFile) Get(section string, key string) (string, bool) { - if value, ok := f.File.Get(section, key); ok { - return value, true - } else if value, ok := f.File.Get("DEFAULT", key); ok { - return value, true - } else if value, ok := f.File.Get(section, "set "+key); ok { - return value, true - } else if value, ok := f.File.Get("DEFAULT", "set "+key); ok { - return value, true - } - return "", false -} - -func (f IniFile) GetDefault(section string, key string, dfl string) string { - if value, ok := f.Get(section, key); ok { - return value - } - return dfl -} - -func (f IniFile) GetBool(section string, key string, dfl bool) bool { - if value, ok := f.Get(section, key); ok { - return LooksTrue(value) - } - return dfl -} - -func (f IniFile) GetInt(section string, key string, dfl int64) int64 { - if value, ok := f.Get(section, key); ok { - if val, err := strconv.ParseInt(value, 10, 64); err == nil { - return val - } - panic(fmt.Sprintf("Error parsing integer %s/%s from config.", section, key)) - } - return dfl -} - -func (f IniFile) GetLimit(section string, key string, dfla int64, dflb int64) (int64, int64) { - if value, ok := f.Get(section, key); ok { - fmt.Sscanf(value, "%d/%d", &dfla, &dflb) - } - return dfla, dflb -} - -func (f IniFile) HasSection(section string) bool { - return f.File[section] != nil -} - -func LoadIniFile(filename string) (IniFile, error) { - file := IniFile{make(ini.File)} - return file, file.LoadFile(filename) -} - -func UidFromConf(serverConf string) (uint32, uint32, error) { - if ini, err := LoadIniFile(serverConf); err == nil { - username := ini.GetDefault("DEFAULT", "user", "swift") - usr, err := user.Lookup(username) - if err != nil { - return 0, 0, err - } - uid, err := strconv.ParseUint(usr.Uid, 10, 32) - if err != nil { - return 0, 0, err - } - gid, err := strconv.ParseUint(usr.Gid, 10, 32) - if err != nil { - return 0, 0, err - } - return uint32(uid), uint32(gid), nil - } else { - if matches, err := filepath.Glob(serverConf + "/*.conf"); err == nil && len(matches) > 0 { - return UidFromConf(matches[0]) - } - } - return 0, 0, fmt.Errorf("Unable to find config file") -} - func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { partDir := filepath.Dir(filename) tmpFile, err := ioutil.TempFile(partDir, ".tmp-o-file") @@ -572,7 +491,7 @@ func GetHashPrefixAndSuffix() (prefix string, suffix string, err error) { config_locations := []string{"/etc/hummingbird/hummingbird.conf", "/etc/swift/swift.conf"} for _, loc := range config_locations { - if conf, e := LoadIniFile(loc); e == nil { + if conf, e := LoadConfig(loc); e == nil { var ok bool prefix, _ = conf.Get("swift-hash", "swift_hash_path_prefix") if suffix, ok = conf.Get("swift-hash", "swift_hash_path_suffix"); !ok { diff --git a/go/hummingbird/utils_test.go b/go/hummingbird/utils_test.go index 532a9caf60..7607aeb01b 100644 --- a/go/hummingbird/utils_test.go +++ b/go/hummingbird/utils_test.go @@ -17,11 +17,8 @@ package hummingbird import ( "bytes" - "fmt" "io/ioutil" "os" - "os/user" - "strconv" "testing" "time" @@ -218,22 +215,6 @@ func TestUrlencode(t *testing.T) { assert.True(t, Urlencode("鐋댋") == "%E9%90%8B%EB%8C%8B") } -func TestIniFile(t *testing.T) { - tempFile, err := ioutil.TempFile("", "INI") - assert.Nil(t, err) - defer os.RemoveAll(tempFile.Name()) - tempFile.WriteString("[stuff]\ntruevalue=true\nfalsevalue=false\nintvalue=3\nset log_facility = LOG_LOCAL1\n") - iniFile, err := LoadIniFile(tempFile.Name()) - assert.Equal(t, true, iniFile.GetBool("stuff", "truevalue", false)) - assert.Equal(t, false, iniFile.GetBool("stuff", "falsevalue", true)) - assert.Equal(t, true, iniFile.GetBool("stuff", "defaultvalue", true)) - assert.Equal(t, int64(3), iniFile.GetInt("stuff", "intvalue", 2)) - assert.Equal(t, int64(2), iniFile.GetInt("stuff", "missingvalue", 2)) - assert.Equal(t, "false", iniFile.GetDefault("stuff", "falsevalue", "true")) - assert.Equal(t, "true", iniFile.GetDefault("stuff", "missingvalue", "true")) - assert.Equal(t, "LOG_LOCAL1", iniFile.GetDefault("stuff", "log_facility", "LOG_LOCAL0")) -} - func TestIsMount(t *testing.T) { isMount, err := IsMount("/proc") assert.Nil(t, err) @@ -300,32 +281,3 @@ func TestGetHashPrefixAndSuffix(t *testing.T) { t.Error("Error prefix and suffix not being set") } } - -func TestUidFromConf(t *testing.T) { - usr, err := user.Current() - assert.Nil(t, err) - tempFile, err := ioutil.TempFile("", "INI") - assert.Nil(t, err) - defer os.RemoveAll(tempFile.Name()) - defer tempFile.Close() - fmt.Fprintf(tempFile, "[DEFAULT]\nuser=%s\n", usr.Username) - - currentUid, err := strconv.ParseUint(usr.Uid, 10, 32) - assert.Nil(t, err) - currentGid, err := strconv.ParseUint(usr.Gid, 10, 32) - assert.Nil(t, err) - uid, gid, err := UidFromConf(tempFile.Name()) - assert.Nil(t, err) - assert.Equal(t, uint32(currentUid), uint32(uid)) - assert.Equal(t, uint32(currentGid), uint32(gid)) -} - -func TestUidFromConfFailure(t *testing.T) { - tempFile, err := ioutil.TempFile("", "INI") - assert.Nil(t, err) - defer os.RemoveAll(tempFile.Name()) - defer tempFile.Close() - fmt.Fprintf(tempFile, "[DEFAULT]\nuser=SomeUserWhoShouldntExist\n") - _, _, err = UidFromConf(tempFile.Name()) - assert.NotNil(t, err) -} diff --git a/go/objectserver/auditor.go b/go/objectserver/auditor.go index 173ef643bc..c31bfaffd5 100644 --- a/go/objectserver/auditor.go +++ b/go/objectserver/auditor.go @@ -326,12 +326,11 @@ func (d *AuditorDaemon) RunForever() { } // NewAuditor returns a new AuditorDaemon with the given conf. -func NewAuditor(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error) { - d := &AuditorDaemon{} - serverconf, err := hummingbird.LoadIniFile(conf) - if err != nil || !serverconf.HasSection("object-auditor") { - return nil, fmt.Errorf("Unable to find auditor config: %s", conf) +func NewAuditor(serverconf hummingbird.Config, flags *flag.FlagSet) (hummingbird.Daemon, error) { + if !serverconf.HasSection("object-auditor") { + return nil, fmt.Errorf("Unable to find object-auditor config section") } + d := &AuditorDaemon{} d.driveRoot = serverconf.GetDefault("object-auditor", "devices", "/srv/node") d.checkMounts = serverconf.GetBool("object-auditor", "mount_check", true) d.logger = hummingbird.SetupLogger(serverconf.GetDefault("object-auditor", "log_facility", "LOG_LOCAL0"), "object-auditor", "") diff --git a/go/objectserver/auditor_test.go b/go/objectserver/auditor_test.go index 0674bc9699..8dd27ed29f 100644 --- a/go/objectserver/auditor_test.go +++ b/go/objectserver/auditor_test.go @@ -28,6 +28,7 @@ import ( "github.com/openstack/swift/go/hummingbird" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAuditHashPasses(t *testing.T) { @@ -192,25 +193,23 @@ func (s *auditLogSaver) Debug(line string) error { } func makeAuditor(settings ...string) *Auditor { - conf, _ := ioutil.TempFile("", "") - conf.WriteString("[object-auditor]\n") + configString := "[object-auditor]\n" for i := 0; i < len(settings); i += 2 { - fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1]) + configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1]) } - defer conf.Close() - defer os.RemoveAll(conf.Name()) - auditorDaemon, _ := NewAuditor(conf.Name(), &flag.FlagSet{}) + conf, _ := hummingbird.StringConfig(configString) + auditorDaemon, _ := NewAuditor(conf, &flag.FlagSet{}) auditorDaemon.(*AuditorDaemon).logger = &auditLogSaver{} return &Auditor{AuditorDaemon: auditorDaemon.(*AuditorDaemon), filesPerSecond: 1} } func TestFailsWithoutSection(t *testing.T) { - conf, _ := ioutil.TempFile("", "") - defer conf.Close() - defer os.RemoveAll(conf.Name()) - auditorDaemon, err := NewAuditor(conf.Name(), &flag.FlagSet{}) + conf, err := hummingbird.StringConfig("") + require.Nil(t, err) + auditorDaemon, err := NewAuditor(conf, &flag.FlagSet{}) + require.NotNil(t, err) assert.Nil(t, auditorDaemon) - assert.True(t, strings.HasPrefix(err.Error(), "Unable to find auditor config")) + assert.True(t, strings.HasPrefix(err.Error(), "Unable to find object-auditor")) } func TestAuditSuffixNotDir(t *testing.T) { diff --git a/go/objectserver/main.go b/go/objectserver/main.go index 060d121542..2450bddfe8 100644 --- a/go/objectserver/main.go +++ b/go/objectserver/main.go @@ -455,7 +455,7 @@ func (server *ObjectServer) updateDeviceLocks(seconds int64) { } } -func (server *ObjectServer) GetHandler() http.Handler { +func (server *ObjectServer) GetHandler(config hummingbird.Config) http.Handler { commonHandlers := alice.New(middleware.ClearHandler, server.LogRequest, middleware.ValidateRequest, server.AcquireDevice) router := hummingbird.NewRouter() router.Get("/healthcheck", commonHandlers.ThenFunc(server.HealthcheckHandler)) @@ -477,7 +477,7 @@ func (server *ObjectServer) GetHandler() http.Handler { return alice.New(middleware.GrepObject).Then(router) } -func GetServer(conf string, flags *flag.FlagSet) (bindIP string, bindPort int, serv hummingbird.Server, logger hummingbird.SysLogLike, err error) { +func GetServer(serverconf hummingbird.Config, flags *flag.FlagSet) (bindIP string, bindPort int, serv hummingbird.Server, logger hummingbird.SysLogLike, err error) { server := &ObjectServer{driveRoot: "/srv/node", hashPathPrefix: "", hashPathSuffix: "", allowedHeaders: map[string]bool{"Content-Disposition": true, "Content-Encoding": true, @@ -491,11 +491,6 @@ func GetServer(conf string, flags *flag.FlagSet) (bindIP string, bindPort int, s return "", 0, nil, nil, err } - serverconf, err := hummingbird.LoadIniFile(conf) - if err != nil { - return "", 0, nil, nil, errors.New(fmt.Sprintf("Unable to load %s", conf)) - } - objEngineName := serverconf.GetDefault("app:object-server", "object_engine", DefaultObjEngine) if newEngineFactory, err := FindEngine(objEngineName); err != nil { return "", 0, nil, nil, errors.New("Unable to find object engine") diff --git a/go/objectserver/main_test.go b/go/objectserver/main_test.go index 8167400f2c..a1293c8a1a 100644 --- a/go/objectserver/main_test.go +++ b/go/objectserver/main_test.go @@ -63,21 +63,16 @@ func makeObjectServer(settings ...string) (*TestServer, error) { if err != nil { return nil, err } - conf, err := ioutil.TempFile(driveRoot, "") + configString := fmt.Sprintf("[app:object-server]\ndevices=%s\nmount_check=false\n", driveRoot) + for i := 0; i < len(settings); i += 2 { + configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1]) + } + conf, err := hummingbird.StringConfig(configString) if err != nil { return nil, err } - conf.WriteString("[app:object-server]\n") - fmt.Fprintf(conf, "devices=%s\n", driveRoot) - fmt.Fprintf(conf, "mount_check=false\n") - for i := 0; i < len(settings); i += 2 { - fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1]) - } - if err := conf.Close(); err != nil { - return nil, err - } - _, _, server, _, _ := GetServer(conf.Name(), &flag.FlagSet{}) - ts := httptest.NewServer(server.GetHandler()) + _, _, server, _, _ := GetServer(conf, &flag.FlagSet{}) + ts := httptest.NewServer(server.GetHandler(conf)) u, err := url.Parse(ts.URL) if err != nil { return nil, err @@ -313,7 +308,7 @@ func TestDisconnectOnPut(t *testing.T) { resp := &fakeResponse{} - ts.objServer.GetHandler().ServeHTTP(resp, req) + ts.Server.Config.Handler.ServeHTTP(resp, req) assert.Equal(t, resp.status, 499) } diff --git a/go/objectserver/objengine.go b/go/objectserver/objengine.go index 5e3b7dbefd..213f6c3e2c 100644 --- a/go/objectserver/objengine.go +++ b/go/objectserver/objengine.go @@ -61,7 +61,7 @@ type ObjectEngine interface { } // ObjectEngineConstructor> is a function that, given configs and flags, returns an ObjectEngine -type ObjectEngineConstructor func(hummingbird.IniFile, *flag.FlagSet) (ObjectEngine, error) +type ObjectEngineConstructor func(hummingbird.Config, *flag.FlagSet) (ObjectEngine, error) type engineFactoryEntry struct { name string diff --git a/go/objectserver/objengine_test.go b/go/objectserver/objengine_test.go index 18b3bae2b2..8405956c70 100644 --- a/go/objectserver/objengine_test.go +++ b/go/objectserver/objengine_test.go @@ -26,7 +26,7 @@ import ( func TestObjectEngineRegistry(t *testing.T) { testErr := errors.New("Not implemented") - constructor := func(hummingbird.IniFile, *flag.FlagSet) (ObjectEngine, error) { + constructor := func(hummingbird.Config, *flag.FlagSet) (ObjectEngine, error) { return nil, testErr } @@ -34,7 +34,7 @@ func TestObjectEngineRegistry(t *testing.T) { fconstructor, err := FindEngine("test") require.Nil(t, err) - eng, err := fconstructor(hummingbird.IniFile{}, nil) + eng, err := fconstructor(hummingbird.Config{}, nil) require.Nil(t, eng) require.Equal(t, err, testErr) diff --git a/go/objectserver/replicator.go b/go/objectserver/replicator.go index 0337bc98ab..815ace42a2 100644 --- a/go/objectserver/replicator.go +++ b/go/objectserver/replicator.go @@ -740,7 +740,11 @@ func (r *Replicator) RunForever() { r.run(time.Tick(RunForeverInterval)) } -func NewReplicator(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error) { +func NewReplicator(serverconf hummingbird.Config, flags *flag.FlagSet) (hummingbird.Daemon, error) { + if !serverconf.HasSection("object-replicator") { + return nil, fmt.Errorf("Unable to find object-auditor config section") + } + replicator := &Replicator{ partitionTimesAdd: make(chan float64), replicationCountIncrement: make(chan uint64), @@ -752,10 +756,6 @@ func NewReplicator(conf string, flags *flag.FlagSet) (hummingbird.Daemon, error) if err != nil { return nil, fmt.Errorf("Unable to get hash prefix and suffix") } - serverconf, err := hummingbird.LoadIniFile(conf) - if err != nil || !serverconf.HasSection("object-replicator") { - return nil, fmt.Errorf("Unable to find replicator config: %s", conf) - } replicator.reconCachePath = serverconf.GetDefault("object-auditor", "recon_cache_path", "/var/cache/swift") replicator.checkMounts = serverconf.GetBool("object-replicator", "mount_check", true) replicator.driveRoot = serverconf.GetDefault("object-replicator", "devices", "/srv/node") diff --git a/go/objectserver/replicator_test.go b/go/objectserver/replicator_test.go index 0f79a3c3c0..7e0cac55ac 100644 --- a/go/objectserver/replicator_test.go +++ b/go/objectserver/replicator_test.go @@ -87,14 +87,12 @@ func makeReplicator(settings ...string) *Replicator { } func makeReplicatorWithFlags(settings []string, flags *flag.FlagSet) *Replicator { - conf, _ := ioutil.TempFile("", "") - conf.WriteString("[object-replicator]\nmount_check=false\n") + configString := "[object-replicator]\nmount_check=false\n" for i := 0; i < len(settings); i += 2 { - fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1]) + configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1]) } - defer conf.Close() - defer os.RemoveAll(conf.Name()) - replicator, _ := NewReplicator(conf.Name(), flags) + conf, _ := hummingbird.StringConfig(configString) + replicator, _ := NewReplicator(conf, flags) rep := replicator.(*Replicator) rep.concurrencySem = make(chan struct{}, 1) return rep @@ -194,12 +192,6 @@ func (r *FakeHandoffRing) GetJobNodes(partition uint64, localDevice int) (respon return []*hummingbird.Device{r.dev}, true } -func TestReplicatorInitFail(t *testing.T) { - replicator, err := NewReplicator("nonexistentfile", &flag.FlagSet{}) - assert.Nil(t, replicator) - assert.NotNil(t, err) -} - func TestReplicatorVmDuration(t *testing.T) { replicator := makeReplicator("vm_test_mode", "true") assert.Equal(t, 2000*time.Millisecond, replicator.timePerPart) diff --git a/go/objectserver/swiftobjeng.go b/go/objectserver/swiftobjeng.go index 5320e78645..e769bfcf47 100644 --- a/go/objectserver/swiftobjeng.go +++ b/go/objectserver/swiftobjeng.go @@ -355,7 +355,7 @@ func (f *SwiftObjectFactory) RegisterHandlers(addRoute func(method, path string, } // SwiftEngineConstructor creates a SwiftObjectFactory given the object server configs. -func SwiftEngineConstructor(config hummingbird.IniFile, flags *flag.FlagSet) (ObjectEngine, error) { +func SwiftEngineConstructor(config hummingbird.Config, flags *flag.FlagSet) (ObjectEngine, error) { driveRoot := config.GetDefault("app:object-server", "devices", "/srv/node") reserve := config.GetInt("app:object-server", "fallocate_reserve", 0) hashPathPrefix, hashPathSuffix, err := hummingbird.GetHashPrefixAndSuffix() diff --git a/go/probe/base.go b/go/probe/base.go index 6fe6f6ec27..8b7c8afb04 100644 --- a/go/probe/base.go +++ b/go/probe/base.go @@ -156,24 +156,19 @@ func NewEnvironment(settings ...string) *Environment { host, ports, _ := net.SplitHostPort(u.Host) port, _ := strconv.Atoi(ports) - conf, _ := ioutil.TempFile("", "") - conf.WriteString("[DEFAULT]\nmount_check=false\n") - fmt.Fprintf(conf, "devices=%s\n", driveRoot) - fmt.Fprintf(conf, "bind_port=%d\n", port) - fmt.Fprintf(conf, "bind_ip=%s\n", host) + configString := "[DEFAULT]\nmount_check=false\n" + configString += fmt.Sprintf("devices=%s\n", driveRoot) + configString += fmt.Sprintf("bind_port=%d\n", port) + configString += fmt.Sprintf("bind_ip=%s\n", host) for i := 0; i < len(settings); i += 2 { - fmt.Fprintf(conf, "%s=%s\n", settings[i], settings[i+1]) + configString += fmt.Sprintf("%s=%s\n", settings[i], settings[i+1]) } - conf.WriteString("[app:object-server]\n") - conf.WriteString("[object-replicator]\n") - conf.WriteString("[object-auditor]\n") - defer conf.Close() - defer os.RemoveAll(conf.Name()) - - _, _, server, _, _ := objectserver.GetServer(conf.Name(), &flag.FlagSet{}) - ts.Config.Handler = server.GetHandler() - replicator, _ := objectserver.NewReplicator(conf.Name(), &flag.FlagSet{}) - auditor, _ := objectserver.NewAuditor(conf.Name(), &flag.FlagSet{}) + configString += "[app:object-server]\n[object-replicator]\n[object-auditor]\n" + conf, _ := hummingbird.StringConfig(configString) + _, _, server, _, _ := objectserver.GetServer(conf, &flag.FlagSet{}) + ts.Config.Handler = server.GetHandler(conf) + replicator, _ := objectserver.NewReplicator(conf, &flag.FlagSet{}) + auditor, _ := objectserver.NewAuditor(conf, &flag.FlagSet{}) replicator.(*objectserver.Replicator).Ring = env.ring env.ring.(*FakeRing).devices = append(env.ring.(*FakeRing).devices, &hummingbird.Device{ Id: i, Device: "sda", Ip: host, Port: port, Region: 0, ReplicationIp: host, ReplicationPort: port, Weight: 1, Zone: i, diff --git a/go/proxyserver/main.go b/go/proxyserver/main.go index 15a1b0e9ca..e33e363022 100644 --- a/go/proxyserver/main.go +++ b/go/proxyserver/main.go @@ -16,7 +16,6 @@ package proxyserver import ( - "errors" "flag" "fmt" "net/http" @@ -68,7 +67,7 @@ func (server *ProxyServer) LogRequest(next http.Handler) http.Handler { return http.HandlerFunc(fn) } -func (server *ProxyServer) GetHandler() http.Handler { +func (server *ProxyServer) GetHandler(config hummingbird.Config) http.Handler { router := hummingbird.NewRouter() router.Get("/healthcheck", http.HandlerFunc(server.HealthcheckHandler)) @@ -108,21 +107,17 @@ func (server *ProxyServer) GetHandler() http.Handler { ).Then(router) } -func GetServer(conf string, flags *flag.FlagSet) (string, int, hummingbird.Server, hummingbird.SysLogLike, error) { +func GetServer(serverconf hummingbird.Config, flags *flag.FlagSet) (string, int, hummingbird.Server, hummingbird.SysLogLike, error) { var err error server := &ProxyServer{} server.C, err = client.NewProxyDirectClient() if err != nil { return "", 0, nil, nil, err } - server.mc, err = hummingbird.NewMemcacheRingFromIniFile(hummingbird.IniFile{}) + server.mc, err = hummingbird.NewMemcacheRingFromConfig(serverconf) if err != nil { return "", 0, nil, nil, err } - serverconf, err := hummingbird.LoadIniFile(conf) - if err != nil { - return "", 0, nil, nil, errors.New(fmt.Sprintf("Unable to load %s", conf)) - } bindIP := serverconf.GetDefault("DEFAULT", "bind_ip", "0.0.0.0") bindPort := serverconf.GetInt("DEFAULT", "bind_port", 8080)