// Copyright (c) 2015 Rackspace // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or // implied. // See the License for the specific language governing permissions and // limitations under the License. package objectserver import ( "crypto/md5" "encoding/hex" "fmt" "io" "math/big" "net/http" "os" "path/filepath" "strings" "time" "github.com/openstack/swift/go/hummingbird" ) /*This hash is used to represent a zero byte async file that is created for an expiring object*/ const zeroByteHash = "d41d8cd98f00b204e9800998ecf8427e" const deleteAtAccount = ".expiring_objects" const waitForContainerUpdate = time.Second / 4 func headerToMap(headers http.Header) map[string]string { ret := make(map[string]string) for key, value := range headers { if len(value) > 0 { ret[key] = headers.Get(key) } } return ret } func splitHeader(header string) []string { if header == "" { return []string{} } return strings.Split(header, ",") } func (server *ObjectServer) hashPath(account, container, obj string) string { h := md5.New() io.WriteString(h, server.hashPathPrefix+"/"+account+"/"+container+"/"+obj+server.hashPathSuffix) return hex.EncodeToString(h.Sum(nil)) } func (server *ObjectServer) expirerContainer(deleteAt time.Time, account, container, obj string) string { i := new(big.Int) fmt.Sscanf(server.hashPath(account, container, obj), "%x", i) shardInt := i.Mod(i, big.NewInt(100)).Int64() timestamp := (deleteAt.Unix()/server.expiringDivisor)*server.expiringDivisor - shardInt if timestamp < 0 { timestamp = 0 } else if timestamp > 9999999999 { timestamp = 9999999999 } return fmt.Sprintf("%010d", timestamp) } func (server *ObjectServer) sendContainerUpdate(host, device, method, partition, account, container, obj string, headers http.Header) bool { obj_url := fmt.Sprintf("http://%s/%s/%s/%s/%s/%s", host, device, partition, hummingbird.Urlencode(account), hummingbird.Urlencode(container), hummingbird.Urlencode(obj)) if req, err := http.NewRequest(method, obj_url, nil); err == nil { req.Header = headers if resp, err := server.updateClient.Do(req); err == nil { resp.Body.Close() if resp.StatusCode/100 == 2 { return true } } } return false } func (server *ObjectServer) saveAsync(method, account, container, obj, localDevice string, headers http.Header) { hash := server.hashPath(account, container, obj) asyncFile := filepath.Join(server.driveRoot, localDevice, "async_pending", hash[29:32], hash+"-"+headers.Get("X-Timestamp")) tempDir := TempDirPath(server.driveRoot, localDevice) data := map[string]interface{}{ "op": method, "account": account, "container": container, "obj": obj, "headers": headerToMap(headers), } if os.MkdirAll(filepath.Dir(asyncFile), 0755) == nil { writer, err := NewAtomicFileWriter(tempDir, filepath.Dir(asyncFile)) if err == nil { defer writer.Abandon() writer.Write(hummingbird.PickleDumps(data)) writer.Save(asyncFile) } } } func (server *ObjectServer) updateContainer(metadata map[string]string, request *http.Request, vars map[string]string, logger hummingbird.LoggingContext) { partition := request.Header.Get("X-Container-Partition") hosts := splitHeader(request.Header.Get("X-Container-Host")) devices := splitHeader(request.Header.Get("X-Container-Device")) if partition == "" || len(hosts) == 0 || len(devices) == 0 { return } requestHeaders := http.Header{ "Referer": {hummingbird.GetDefault(request.Header, "Referer", "-")}, "User-Agent": {hummingbird.GetDefault(request.Header, "User-Agent", "-")}, "X-Trans-Id": {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")}, "X-Timestamp": {request.Header.Get("X-Timestamp")}, } if request.Method != "DELETE" { requestHeaders.Add("X-Content-Type", metadata["Content-Type"]) requestHeaders.Add("X-Size", metadata["Content-Length"]) requestHeaders.Add("X-Etag", metadata["ETag"]) } failures := 0 for index := range hosts { if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, vars["account"], vars["container"], vars["obj"], requestHeaders) { logger.LogError("ERROR container update failed with %s/%s (saving for async update later)", hosts[index], devices[index]) failures++ } } if failures > 0 { server.saveAsync(request.Method, vars["account"], vars["container"], vars["obj"], vars["device"], requestHeaders) } } func (server *ObjectServer) updateDeleteAt(request *http.Request, deleteAtStr string, vars map[string]string, logger hummingbird.LoggingContext) { deleteAt, err := hummingbird.ParseDate(deleteAtStr) if err != nil { return } container := hummingbird.GetDefault(request.Header, "X-Delete-At-Container", "") if container == "" { container = server.expirerContainer(deleteAt, vars["account"], vars["container"], vars["obj"]) } obj := fmt.Sprintf("%010d-%s/%s/%s", deleteAt.Unix(), vars["account"], vars["container"], vars["obj"]) partition := hummingbird.GetDefault(request.Header, "X-Delete-At-Partition", "") hosts := splitHeader(request.Header.Get("X-Delete-At-Host")) devices := splitHeader(request.Header.Get("X-Delete-At-Device")) requestHeaders := http.Header{ "Referer": {hummingbird.GetDefault(request.Header, "Referer", "-")}, "User-Agent": {hummingbird.GetDefault(request.Header, "User-Agent", "-")}, "X-Trans-Id": {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")}, "X-Timestamp": {request.Header.Get("X-Timestamp")}, } if request.Method != "DELETE" { requestHeaders.Add("X-Content-Type", "text/plain") requestHeaders.Add("X-Size", "0") requestHeaders.Add("X-Etag", zeroByteHash) } failures := 0 for index := range hosts { if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, deleteAtAccount, container, obj, requestHeaders) { logger.LogError("ERROR container update failed with %s/%s (saving for async update later)", hosts[index], devices[index]) failures++ } } if failures > 0 || len(hosts) == 0 { server.saveAsync(request.Method, deleteAtAccount, container, obj, vars["device"], requestHeaders) } } func (server *ObjectServer) containerUpdates(request *http.Request, metadata map[string]string, deleteAt string, vars map[string]string, logger hummingbird.LoggingContext) { defer logger.LogPanics("PANIC WHILE UPDATING CONTAINER LISTINGS") if deleteAt != "" { go server.updateDeleteAt(request, deleteAt, vars, logger) } firstDone := make(chan struct{}, 1) go func() { server.updateContainer(metadata, request, vars, logger) firstDone <- struct{}{} }() go func() { time.Sleep(waitForContainerUpdate) firstDone <- struct{}{} }() <-firstDone }