diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94a2dd1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4947287 --- /dev/null +++ b/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e251af8 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +Golang Client +============= +stackforge/golang-client is yet another implementation of [OpenStack] +(http://www.openstack.org/) API client in [Go language](http://golang.org). +The code follows OpenStack licensing and borrows its infrastructure for code +hosting. It currently implements [Identity Service v2] +(http://docs.openstack.org/api/openstack-identity-service/2.0/content/) +and [Object Storage v1] +(http://docs.openstack.org/api/openstack-object-storage/1.0/content/). +Some API calls are not implemented initially, but the intention is to expand +the lib over time (where pragmatic). + +Code maturity is considered experimental. + +Installation +------------ +Use `go get`. Or alternatively, download or clone the repository. + +The lib was developed and tested on go 1.0.3 and 1.1.1, but maintenance has moved +to 1.1.1 only. No external dependencies, so far. + +Usage +----- +The `*_integration_test.go` files show usage examples for using the lib to connect +to live OpenStack service. The documentation follows golang documentation +convention: `go doc`. Here is a short example code snippet: + + auth, err := identity.AuthUserNameTenantId(identityHost, + userName, password, tenantId) + ... + httpHdr, err := objectstorage.GetAccountMeta(objectstorageHost, + auth.Access.Token.Id) + +Testing +------- +There are two types of test files. The `*_test.go` are standard +golang unit test files. The `*_integration_test.go` are +test files that require an active OpenStack service account before +you can properly test. If you do not have an account, +then running `go test` on the `*_integration_test.go` files will fail. + +If you already have an account, please read +`identity/identitytest/setupUser.go` on how to set up the JSON data file so +you can authenticate to the OpenStack service. If you do not have an account, +please change the file extension to something that golang compiler will +ignore to avoid fails. + +The tests were written against the [OpenStack API specifications] +(http://docs.openstack.org/api/api-specs.html). +The integration test were successful against the following: + +- [HP Cloud](http://docs.hpcloud.com/api/) + +If you use another provider and successfully completed the tests, please email +the maintainer(s) so your service can be mentioned here. Alternatively, if you +are a service provider and can arrange a free (temporary) account, a quick test +can be arranged. + +License +------- +Apache v2. + +Contributing +------------ +The code repository borrows OpenStack StackForge infrastructure. +Please use the [recommended workflow] +(https://wiki.openstack.org/wiki/GerritWorkflow). If you are not a member yet, +please consider joining as an [OpenStack contributor] +(https://wiki.openstack.org/wiki/HowToContribute). If you have questions or +comments, you can email the maintainer(s). + +Maintainer +---------- +Slamet Hendry (slamet dot hendry at gmail dot com) + +Coding Style +------------ +The source code is automatically formatted to follow `go fmt` by the [IDE] +(https://code.google.com/p/liteide/). And where pragmatic, the source code +follows this general [coding style] +(http://slamet.neocities.org/coding-style.html). \ No newline at end of file diff --git a/identity/auth.go b/identity/auth.go new file mode 100644 index 0000000..0aecd3a --- /dev/null +++ b/identity/auth.go @@ -0,0 +1,141 @@ +//Package identity provides functions for client-side access to OpenStack +//IdentityService. +package identity + +import ( + "encoding/json" + "errors" + "fmt" + "golang-client/misc" + "io/ioutil" + "strings" + "time" +) + +type Auth struct { + Access Access +} + +type Access struct { + Token Token + User User + ServiceCatalog []Service +} + +type Token struct { + Id string + Expires time.Time + Tenant Tenant +} + +type Tenant struct { + Id string + Name string +} + +type User struct { + Id string + Name string + Roles []Role + Roles_links []string +} + +type Role struct { + Id string + Name string + TenantId string +} + +type Service struct { + Name string + Type string + Endpoints []Endpoint + Endpoints_links []string +} + +type Endpoint struct { + TenantId string + PublicURL string + InternalURL string + Region string + VersionId string + VersionInfo string + VersionList string +} + +func AuthKey(url, accessKey, secretKey string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "apiAccessKeyCredentials":{"accessKey":"%s","secretKey":"%s"}} + }`, + accessKey, secretKey)) + return auth(&url, &jsonStr) +} + +func AuthKeyTenantId(url, accessKey, secretKey, tenantId string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "apiAccessKeyCredentials":{"accessKey":"%s","secretKey":"%s"},"tenantId":"%s"} + }`, + accessKey, secretKey, tenantId)) + return auth(&url, &jsonStr) +} + +func AuthUserName(url, username, password string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "passwordCredentials":{"username":"%s","password":"%s"}} + }`, + username, password)) + return auth(&url, &jsonStr) +} + +func AuthUserNameTenantName(url, username, password, tenantName string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "passwordCredentials":{"username":"%s","password":"%s"},"tenantName":"%s"} + }`, + username, password, tenantName)) + return auth(&url, &jsonStr) +} + +func AuthUserNameTenantId(url, username, password, tenantId string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "passwordCredentials":{"username":"%s","password":"%s"},"tenantId":"%s"} + }`, + username, password, tenantId)) + return auth(&url, &jsonStr) +} + +func AuthTenantNameTokenId(url, tenantName, tokenId string) (Auth, error) { + jsonStr := (fmt.Sprintf(`{"auth":{ + "tenantName":"%s","token":{"id":"%s"}} + }`, + tenantName, tokenId)) + return auth(&url, &jsonStr) +} + +func auth(url, jsonStr *string) (Auth, error) { + var s []byte = []byte(*jsonStr) + resp, err := misc.CallAPI("POST", *url, &s, + "Accept-Encoding", "gzip,deflate", + "Accept", "application/json", + "Content-Type", "application/json", + "Content-Length", string(len(*jsonStr))) + if err != nil { + return Auth{}, err + } + if err = misc.CheckHttpResponseStatusCode(resp); err != nil { + return Auth{}, err + } + var contentType string = strings.ToLower(resp.Header.Get("Content-Type")) + if strings.Contains(contentType, "json") != true { + return Auth{}, errors.New("err: header Content-Type is not JSON") + } + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return Auth{}, err + } + var auth = Auth{} + if err = json.Unmarshal(body, &auth); err != nil { + return Auth{}, err + } + return auth, nil +} diff --git a/identity/auth_integration_test.go b/identity/auth_integration_test.go new file mode 100644 index 0000000..4ff0fa8 --- /dev/null +++ b/identity/auth_integration_test.go @@ -0,0 +1,101 @@ +//PRE-REQUISITE: Must have valid IdentityService account, either internally +//hosted or with one of the OpenStack providers. See identitytest/ for the +//JSON specification. +//The JSON file ought to be in .hgignore / .gitignore for security reason. +package identity_test + +import ( + "golang-client/identity" + "golang-client/identity/identitytest" + "testing" + "time" +) + +var account = identitytest.SetupUser("identitytest/user.json") + +func TestAuthKey(t *testing.T) { + //Not in OpenStack api doc, but in HPCloud api doc. + auth, err := identity.AuthKey(account.Host, + account.AccessKey, + account.SecretKey) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} + +func TestAuthKeyTenantId(t *testing.T) { + //Not in OpenStack nor HPCloud api doc, but in HPCloud curl example. + auth, err := identity.AuthKeyTenantId(account.Host, + account.AccessKey, + account.SecretKey, + account.TenantId) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} + +func TestAuthUserName(t *testing.T) { + //Not in OpenStack api doc, but in HPCloud api doc. + auth, err := identity.AuthUserName(account.Host, + account.UserName, + account.Password) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} +func TestAuthUserNameTenantName(t *testing.T) { + //In OpenStack api doc, but not in HPCloud api doc, but tested valid in HPCloud. + auth, err := identity.AuthUserNameTenantName(account.Host, + account.UserName, + account.Password, + account.TenantName) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} + +func TestAuthUserNameTenantId(t *testing.T) { + //Not in OpenStack api doc, but in HPCloud api doc. + auth, err := identity.AuthUserNameTenantId(account.Host, + account.UserName, + account.Password, + account.TenantId) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} + +func TestAuthTenantNameTokenId(t *testing.T) { + //Not in OpenStack api doc, but in HPCloud api doc. + auth, err := identity.AuthUserNameTenantId(account.Host, + account.UserName, + account.Password, + account.TenantId) + if err != nil { + t.Error(err) + } + auth, err = identity.AuthTenantNameTokenId(account.Host, + account.TenantName, + auth.Access.Token.Id) + if err != nil { + t.Error(err) + } + if !auth.Access.Token.Expires.After(time.Now()) { + t.Error("expiry is wrong") + } +} diff --git a/identity/identitytest/setupUser.go b/identity/identitytest/setupUser.go new file mode 100644 index 0000000..5f6f8a5 --- /dev/null +++ b/identity/identitytest/setupUser.go @@ -0,0 +1,33 @@ +package identitytest + +import ( + "encoding/json" + "io/ioutil" +) + +//SetupUser() is used to retrieve externally stored testing credentials. +//The testing credentials are stored outside +//the source code so they do not get checked in, assuming the user.json is +//in .gitignore / .hgignore. "user.json" should contain the following where +//... is the actual value from the test user account credentials. +//{ +// "TenantId":"...", +// "TenantName": "...", +// "AccessKey": "...", +// "SecretKey": "...", +// "UserName": "...", +// "Password": "...", +// "Host": "https://.../v2.0/tokens" +//} +func SetupUser(jsonFile string) (acct struct { + TenantId, TenantName, AccessKey, SecretKey, UserName, Password, Host string +},) { + usrJson, err := ioutil.ReadFile(jsonFile) + if err != nil { + panic("ReadFile json failed") + } + if err = json.Unmarshal(usrJson, &acct); err != nil { + panic("Unmarshal json failed") + } + return acct +} diff --git a/misc/util.go b/misc/util.go new file mode 100644 index 0000000..3df9e4e --- /dev/null +++ b/misc/util.go @@ -0,0 +1,96 @@ +package misc + +import ( + "bytes" + "errors" + "io" + "net/http" +) + +//CallAPI sends an HTTP request using "method" to "url". +//For uploading / sending file, caller needs to set the "content". Otherwise, +//set it to zero length []byte. If Header fields need to be set, then set it in +// "h". "h" needs to be even numbered, i.e. pairs of field name and the field +//content. +// +//fileContent, err := ioutil.ReadFile("fileName.ext"); +// +//resp, err := CallAPI("PUT", "http://domain/hello/", &fileContent, +//"Name", "world") +// +//is similar to: curl -X PUT -H "Name: world" -T fileName.ext +//http://domain/hello/ +func CallAPI(method, url string, content *[]byte, h ...string) (*http.Response, error) { + if len(h)%2 == 1 { //odd # + return nil, errors.New("syntax err: # header != # of values") + } + //I think the above err check is unnecessary and wastes cpu cycle, since + //len(h) is not determined at run time. If the coder puts in odd # of args, + //the integration testing should catch it. + //But hey, things happen, so I decided to add it anyway, although you can + //comment it out, if you are confident in your test suites. + var req *http.Request + var err error + req, err = http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + for i := 0; i < len(h)-1; i = i + 2 { + req.Header.Set(h[i], h[i+1]) + } + req.ContentLength = int64(len(*content)) + if req.ContentLength > 0 { + req.Body = readCloser{bytes.NewReader(*content)} + //req.Body = *(new(io.ReadCloser)) //these 3 lines do not work but I am + //req.Body.Read(content) //keeping them here in case I wonder why + //req.Body.Close() //I did not implement it this way :) + } + return (new(http.Client)).Do(req) +} + +type readCloser struct { + io.Reader +} + +func (readCloser) Close() error { + //cannot put this func inside CallAPI; golang disallow nested func + return nil +} + +//CheckStatusCode compares http response header StatusCode against expected +//statuses. Primary function is to ensure StatusCode is in the 20x (return nil). +//Ok: 200. Created: 201. Accepted: 202. No Content: 204. +//Otherwise return error message. +func CheckHttpResponseStatusCode(resp *http.Response) error { + switch resp.StatusCode { + case 200, 201, 202, 204: + return nil + case 400: + return errors.New("Error: response == 400 bad request") + case 401: + return errors.New("Error: response == 401 unauthorised") + case 403: + return errors.New("Error: response == 403 forbidden") + case 404: + return errors.New("Error: response == 404 not found") + case 405: + return errors.New("Error: response == 405 method not allowed") + case 409: + return errors.New("Error: response == 409 conflict") + case 413: + return errors.New("Error: response == 413 over limit") + case 415: + return errors.New("Error: response == 415 bad media type") + case 422: + return errors.New("Error: response == 422 unprocessable") + case 429: + return errors.New("Error: response == 429 too many request") + case 500: + return errors.New("Error: response == 500 instance fault / server err") + case 501: + return errors.New("Error: response == 501 not implemented") + case 503: + return errors.New("Error: response == 503 service unavailable") + } + return errors.New("Error: unexpected response status code") +} diff --git a/misc/util_test.go b/misc/util_test.go new file mode 100644 index 0000000..7c3c4df --- /dev/null +++ b/misc/util_test.go @@ -0,0 +1,96 @@ +package misc_test + +import ( + "bytes" + "errors" + misc "golang-client/misc" + "io/ioutil" + "net/http" + "net/http/httptest" + "strconv" + "testing" +) + +func TestCallAPI(t *testing.T) { + tokn := "eaaafd18-0fed-4b3a-81b4-663c99ec1cbb" + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Auth-Token") != tokn { + t.Error(errors.New("Token failed")) + } + w.WriteHeader(200) //ok + })) + zeroByte := &([]byte{}) + if _, err := misc.CallAPI("HEAD", apiServer.URL, zeroByte, "X-Auth-Token", tokn); err != nil { + t.Error(err) + } + if _, err := misc.CallAPI("DELETE", apiServer.URL, zeroByte, "X-Auth-Token", tokn); err != nil { + t.Error(err) + } + if _, err := misc.CallAPI("POST", apiServer.URL, zeroByte, "X-Auth-Token", tokn); err != nil { + t.Error(err) + } +} + +func TestCallAPIGetContent(t *testing.T) { + tokn := "eaaafd18-0fed-4b3a-81b4-663c99ec1cbb" + fContent, err := ioutil.ReadFile("./util.go") + if err != nil { + t.Error(err) + } + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + if r.Header.Get("X-Auth-Token") != tokn { + t.Error(errors.New("Token failed")) + } + w.Header().Set("Content-Length", r.Header.Get("Content-Length")) + w.Write(body) + })) + var resp *http.Response + if resp, err = misc.CallAPI("GET", apiServer.URL, &fContent, "X-Auth-Token", tokn, + "Etag", "md5hash-blahblah"); err != nil { + t.Error(err) + } + if strconv.Itoa(len(fContent)) != resp.Header.Get("Content-Length") { + t.Error(errors.New("Failed: Content-Length comparison")) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Error(err) + } + if !bytes.Equal(fContent, body) { + t.Error(errors.New("Failed: Content body comparison")) + } +} + +func TestCallAPIPutContent(t *testing.T) { + tokn := "eaaafd18-0fed-4b3a-81b4-663c99ec1cbb" + fContent, err := ioutil.ReadFile("./util.go") + if err != nil { + t.Error(err) + } + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Auth-Token") != tokn { + t.Error(errors.New("Token failed")) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + if strconv.Itoa(len(fContent)) != r.Header.Get("Content-Length") { + t.Error(errors.New("Failed: Content-Length comparison")) + } + if !bytes.Equal(fContent, body) { + t.Error(errors.New("Failed: Content body comparison")) + } + w.WriteHeader(200) + })) + if _, err = misc.CallAPI("PUT", apiServer.URL, &fContent, "X-Auth-Token", tokn); err != nil { + t.Error(err) + } +} diff --git a/objectstorage/objectstorage.go b/objectstorage/objectstorage.go new file mode 100644 index 0000000..76521d3 --- /dev/null +++ b/objectstorage/objectstorage.go @@ -0,0 +1,176 @@ +package objectstorage + +import ( + "golang-client/misc" + "io/ioutil" + "net/http" + "net/url" + "strconv" +) + +var zeroByte = &([]byte{}) //pointer to empty []byte + +//ListContainers calls the OpenStack list containers API using +//previously obtained token. +//"limit" and "marker" corresponds to the API's "limit" and "marker". +//"url" can be regular storage or cdn-enabled storage URL. +//It returns []byte which then needs to be unmarshalled to decode the JSON. +func ListContainers(limit int64, marker, url, token string) ([]byte, error) { + return ListObjects(limit, marker, "", "", "", url, token) +} + +//GetAccountMeta calls the OpenStack retrieve account metadata API using +//previously obtained token. +func GetAccountMeta(url, token string) (http.Header, error) { + return GetObjectMeta(url, token) +} + +//DeleteContainer calls the OpenStack delete container API using +//previously obtained token. +func DeleteContainer(url, token string) error { + return DeleteObject(url, token) +} + +//GetContainerMeta calls the OpenStack retrieve object metadata API +//using previously obtained token. +//url can be regular storage or CDN-enabled storage URL. +func GetContainerMeta(url, token string) (http.Header, error) { + return GetObjectMeta(url, token) +} + +//SetContainerMeta calls the OpenStack API to create / update meta data +//for container using previously obtained token. +//url can be regular storage or CDN-enabled storage URL. +func SetContainerMeta(url string, token string, s ...string) (err error) { + return SetObjectMeta(url, token, s...) +} + +//PutContainer calls the OpenStack API to create / update +//container using previously obtained token. +func PutContainer(url, token string, s ...string) error { + return PutObject(zeroByte, url, token, s...) +} + +//ListObjects calls the OpenStack list object API using previously +//obtained token. "Limit", "marker", "prefix", "path", "delim" corresponds +//to the API's "limit", "marker", "prefix", "path", and "delimiter". +func ListObjects(limit int64, + marker, prefix, path, delim, conUrl, token string) ([]byte, error) { + var query string = "?format=json" + if limit > 0 { + query += "&limit=" + strconv.FormatInt(limit, 10) + } + if marker != "" { + query += "&marker=" + url.QueryEscape(marker) + } + if prefix != "" { + query += "&prefix=" + url.QueryEscape(prefix) + } + if path != "" { + query += "&path=" + url.QueryEscape(path) + } + if delim != "" { + query += "&delimiter=" + url.QueryEscape(delim) + } + resp, err := misc.CallAPI("GET", conUrl+query, zeroByte, + "X-Auth-Token", token) + if err != nil { + return nil, err + } + if err = misc.CheckHttpResponseStatusCode(resp); err != nil { + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return []byte{}, err + } + return body, nil +} + +//PutObject calls the OpenStack create object API using previously +//obtained token. +//url can be regular storage or CDN-enabled storage URL. +func PutObject(fContent *[]byte, url, token string, s ...string) (err error) { + s = append(s, "X-Auth-Token") + s = append(s, token) + resp, err := misc.CallAPI("PUT", url, fContent, s...) + if err != nil { + return err + } + return misc.CheckHttpResponseStatusCode(resp) +} + +//CopyObject calls the OpenStack copy object API using previously obtained +//token. Note from API doc: "The destination container must exist before +//attempting the copy." +func CopyObject(srcUrl, destUrl, token string) (err error) { + resp, err := misc.CallAPI("COPY", srcUrl, zeroByte, + "X-Auth-Token", token, + "Destination", destUrl) + if err != nil { + return err + } + return misc.CheckHttpResponseStatusCode(resp) +} + +//DeleteObject calls the OpenStack delete object API using +//previously obtained token. +// +//Note from API doc: "A DELETE to a versioned object removes the current version +//of the object and replaces it with the next-most current version, moving it +//from the non-current container to the current." .. "If you want to completely +//remove an object and you have five total versions of it, you must DELETE it +//five times." +func DeleteObject(url, token string) (err error) { + resp, err := misc.CallAPI("DELETE", url, zeroByte, "X-Auth-Token", token) + if err != nil { + return err + } + return misc.CheckHttpResponseStatusCode(resp) +} + +//SetObjectMeta calls the OpenStack API to create/update meta data for +//object using previously obtained token. +func SetObjectMeta(url string, token string, s ...string) (err error) { + s = append(s, "X-Auth-Token") + s = append(s, token) + resp, err := misc.CallAPI("POST", url, zeroByte, s...) + if err != nil { + return err + } + return misc.CheckHttpResponseStatusCode(resp) +} + +//GetObjectMeta calls the OpenStack retrieve object metadata API using +//previously obtained token. +func GetObjectMeta(url, token string) (http.Header, error) { + resp, err := misc.CallAPI("HEAD", url, zeroByte, "X-Auth-Token", token) + if err != nil { + return nil, err + } + return resp.Header, misc.CheckHttpResponseStatusCode(resp) +} + +//GetObject calls the OpenStack retrieve object API using previously +//obtained token. It returns http.Header, object / file content downloaded +//from the server, and err. +// +//Since this implementation of GetObject retrieves header info, it +//effectively executes GetObjectMeta also in addition to getting the +//object content. +func GetObject(url, token string) (http.Header, []byte, error) { + resp, err := misc.CallAPI("GET", url, zeroByte, "X-Auth-Token", token) + if err != nil { + return nil, nil, err + } + if err = misc.CheckHttpResponseStatusCode(resp); err != nil { + return nil, nil, err + } + var body []byte + if body, err = ioutil.ReadAll(resp.Body); err != nil { + return nil, nil, err + } + resp.Body.Close() + return resp.Header, body, nil +} diff --git a/objectstorage/objectstorage_integration_test.go b/objectstorage/objectstorage_integration_test.go new file mode 100644 index 0000000..3832d14 --- /dev/null +++ b/objectstorage/objectstorage_integration_test.go @@ -0,0 +1,159 @@ +package objectstorage_test + +import ( + "bytes" + "encoding/json" + "golang-client/identity" + "golang-client/identity/identitytest" + "golang-client/objectstorage" + "io/ioutil" + "testing" +) + +//PRE-REQUISITE: Must have valid ObjectStorage account, either internally +//hosted or with one of the OpenStack providers. Identity is assumed to +//use IdentityService mechanism, instead of legacy Swift mechanism. +func TestEndToEnd(t *testing.T) { + //user.json holds the user account info needed to authenticate + account := identitytest.SetupUser("../identity/identitytest/user.json") + auth, err := identity.AuthUserNameTenantId(account.Host, + account.UserName, + account.Password, + account.TenantId) + if err != nil { + t.Fatal(err) + } + + url := "" + for _, svc := range auth.Access.ServiceCatalog { + if svc.Type == "object-store" { + url = svc.Endpoints[0].PublicURL + "/" + break + } + } + if url == "" { + t.Fatal("object-store url not found during authentication") + } + + hdr, err := objectstorage.GetAccountMeta(url, auth.Access.Token.Id) + if err != nil { + t.Error("\nGetAccountMeta error\n", err) + } + + container := "testContainer1" + if err = objectstorage.PutContainer(url+container, auth.Access.Token.Id, + "X-Log-Retention", "true"); err != nil { + t.Fatal("\nPutContainer\n", err) + } + + containersJson, err := objectstorage.ListContainers(0, "", + url, auth.Access.Token.Id) + if err != nil { + t.Fatal(err) + } + + type containerType struct { + Name string + Bytes, Count int + } + containersList := []containerType{} + + if err = json.Unmarshal(containersJson, &containersList); err != nil { + t.Error(err) + } + + found := false + for i := 0; i < len(containersList); i++ { + if containersList[i].Name == container { + found = true + } + } + if !found { + t.Fatal("created container is missing from downloaded containersList") + } + + if err = objectstorage.SetContainerMeta(url+container, auth.Access.Token.Id, + "X-Container-Meta-fubar", "false"); err != nil { + t.Error(err) + } + hdr, err = objectstorage.GetContainerMeta(url+container, auth.Access.Token.Id) + if err != nil { + t.Error("\nGetContainerMeta error\n", err) + } + if hdr.Get("X-Container-Meta-fubar") != "false" { + t.Error("container meta does not match") + } + + var fContent []byte + srcFile := "objectstorage_integration_test.go" + fContent, err = ioutil.ReadFile(srcFile) + if err != nil { + t.Fatal(err) + } + + object := container + "/" + srcFile + if err = objectstorage.PutObject(&fContent, url+object, auth.Access.Token.Id, + "X-Object-Meta-fubar", "false"); err != nil { + t.Fatal(err) + } + objectsJson, err := objectstorage.ListObjects(0, "", "", "", "", + url+container, auth.Access.Token.Id) + + type objectType struct { + Name, Hash, Content_type, Last_modified string + Bytes int + } + objectsList := []objectType{} + + if err = json.Unmarshal(objectsJson, &objectsList); err != nil { + t.Error(err) + } + found = false + for i := 0; i < len(objectsList); i++ { + if objectsList[i].Name == srcFile { + found = true + } + } + if !found { + t.Fatal("created object is missing from the objectsList") + } + + if err = objectstorage.SetObjectMeta(url+object, auth.Access.Token.Id, + "X-Object-Meta-fubar", "true"); err != nil { + t.Error("\nSetObjectMeta error\n", err) + } + hdr, err = objectstorage.GetObjectMeta(url+object, auth.Access.Token.Id) + if err != nil { + t.Error("\nGetObjectMeta error\n", err) + } + if hdr.Get("X-Object-Meta-fubar") != "true" { + t.Error("\nSetObjectMeta error\n", "object meta does not match") + } + + _, body, err := objectstorage.GetObject(url+object, auth.Access.Token.Id) + if err != nil { + t.Error("\nGetObject error\n", err) + } + if !bytes.Equal(fContent, body) { + t.Error("\nGetObject error\n", "byte comparison of uploaded != downloaded") + } + + if err = objectstorage.CopyObject(url+object, "/"+object+".dup", + auth.Access.Token.Id); err != nil { + t.Fatal("\nCopyObject error\n", err) + } + + if err = objectstorage.DeleteObject(url+object, + auth.Access.Token.Id); err != nil { + t.Fatal("\nDeleteObject error\n", err) + } + if err = objectstorage.DeleteObject(url+object+".dup", + auth.Access.Token.Id); err != nil { + t.Fatal("\nDeleteObject error\n", err) + } + + if err = objectstorage.DeleteContainer(url+container, + auth.Access.Token.Id); err != nil { + t.Error("\nDeleteContainer error\n", err) + } +} diff --git a/objectstorage/objectstorage_test.go b/objectstorage/objectstorage_test.go new file mode 100644 index 0000000..e4ac30e --- /dev/null +++ b/objectstorage/objectstorage_test.go @@ -0,0 +1,316 @@ +package objectstorage_test + +import ( + "errors" + "golang-client/objectstorage" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "strconv" + "testing" +) + +var znHome = "./" +var objFile = "objectstorage_test.go" +var srcFile = znHome + objFile +var tokn = "eaaafd18-0fed-4b3a-81b4-663c99ec1cbb" +var containerName = "John's container" +var containerPrefix = "/" + containerName +var objPrefix = containerPrefix + "/" + objFile + +func TestGetAccountMeta(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("X-Account-Container-Count", "7") + w.Header().Set("X-Account-Object-Count", "413") + w.Header().Set("X-Account-Bytes-Used", "987654321000") + w.WriteHeader(204) + return + } + t.Error(errors.New("Failed: r.Method == HEAD")) + })) + defer apiServer.Close() + meta, err := objectstorage.GetAccountMeta(apiServer.URL, tokn) + if err != nil { + t.Error(err) + } + if meta.Get("X-Account-Container-Count") != "7" || + meta.Get("X-Account-Object-Count") != "413" || + meta.Get("X-Account-Bytes-Used") != "987654321000" { + t.Error("Failed: meta not matching") + } +} + +func TestListContainers(t *testing.T) { + var containerList = `[ + {"name":"container 1", + "count":2, "bytes":78}, + {"name":"container 2", + "count":1, + "bytes":17}]` + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + w.Write([]byte(containerList)) + return + } + t.Error(errors.New("Failed: r.Method == GET")) + })) + defer apiServer.Close() + myList, err := objectstorage.ListContainers(0, "", apiServer.URL, tokn) + if err != nil { + t.Error(err) + } + if string(myList) != containerList { + t.Error(errors.New("Failed: input != output")) + } +} + +func TestListObjects(t *testing.T) { + var objList = `[ + {"name":"test obj 1", + "hash":"4281c348eaf83e70ddce0e07221c3d28", + "bytes":14, + "content_type":"application\/octet-stream", + "last_modified":"2009-02-03T05:26:32.612278"}, + {"name":"test obj 2", + "hash":"b039efe731ad111bc1b0ef221c3849d0", + "bytes":64, + "content_type":"application\/octet-stream", + "last_modified":"2009-02-03T05:26:32.612278"} + ]` + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + w.Write([]byte(objList)) + return + } + t.Error(errors.New("Failed: r.Method == GET")) + })) + defer apiServer.Close() + myList, err := objectstorage.ListObjects( + 0, "", "", "", "", apiServer.URL+containerPrefix, tokn) + if err != nil { + t.Error(err) + } + if string(myList) != objList { + t.Error(errors.New("Failed: input != output")) + } +} + +func TestDeleteContainer(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "DELETE" { + w.WriteHeader(204) + return + } + t.Error(errors.New("Failed: r.Method == DELETE")) + })) + defer apiServer.Close() + if err := objectstorage.DeleteContainer(apiServer.URL+containerPrefix, + tokn); err != nil { + t.Error(err) + } +} + +func TestGetContainerMeta(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("X-Container-Object-Count", "7") + w.Header().Set("X-Container-Bytes-Used", "413") + w.Header().Set("X-Container-Meta-InspectedBy", "Jack Wolf") + w.WriteHeader(204) + return + } + t.Error(errors.New("Failed: r.Method == HEAD")) + })) + defer apiServer.Close() + meta, err := objectstorage.GetContainerMeta(apiServer.URL+containerPrefix, tokn) + if err != nil { + t.Error(err) + } + if meta.Get("X-Container-Object-Count") != "7" || + meta.Get("X-Container-Bytes-Used") != "413" || + meta.Get("X-Container-Meta-InspectedBy") != "Jack Wolf" { + t.Error("Failed: meta not matching") + } +} + +func TestSetContainerMeta(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" && r.Header.Get("X-Container-Meta-Fruit") == "Apple" { + w.WriteHeader(204) + return + } + t.Error(errors.New( + "Failed: r.Method == POST && X-Container-Meta-Fruit == Apple")) + })) + defer apiServer.Close() + if err := objectstorage.SetContainerMeta( + apiServer.URL+containerPrefix, tokn, + "X-Container-Meta-Fruit", "Apple"); err != nil { + t.Error(err) + } +} + +func TestPutContainer(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" { + w.WriteHeader(201) + return + } + t.Error(errors.New("Failed: r.Method == PUT")) + })) + defer apiServer.Close() + if err := objectstorage.PutContainer(apiServer.URL+containerPrefix, + tokn, "X-TTL", "259200", "X-Log-Retention", "true"); err != nil { + t.Error(err) + } +} + +func TestPutObject(t *testing.T) { + var fContent []byte + f, err := os.Open(srcFile) + defer f.Close() + if err != nil { + t.Error(err) + } + fContent, err = ioutil.ReadAll(f) + if err != nil { + t.Error(err) + } + f.Close() + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + rBody, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + if r.Method == "PUT" && len(fContent) == len(rBody) { + w.WriteHeader(201) + return + } + t.Error(errors.New("Failed: Not 201")) + })) + defer apiServer.Close() + if err = objectstorage.PutObject(&fContent, apiServer.URL+objPrefix, + tokn); err != nil { + t.Error(err) + } +} + +func TestCopyObject(t *testing.T) { + destUrl := "/destContainer/dest/Obj" + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "COPY" && r.Header.Get("Destination") == destUrl { + w.WriteHeader(200) + return + } + t.Error(errors.New( + "Failed: r.Method == COPY && r.Header.Get(Destination) == destUrl")) + })) + defer apiServer.Close() + if err := objectstorage.CopyObject(apiServer.URL+objPrefix, destUrl, + tokn); err != nil { + t.Error(err) + } +} + +func TestGetObjectMeta(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("X-Object-Meta-Fruit", "Apple") + w.Header().Set("X-Object-Meta-Veggie", "Carrot") + w.WriteHeader(200) + return + } + t.Error(errors.New( + "Failed: r.Method == HEAD && r.Header.Get(X-Auth-Token) == tokn")) + })) + defer apiServer.Close() + meta, err := objectstorage.GetObjectMeta(apiServer.URL+objPrefix, tokn) + if err != nil { + t.Error(err) + } + if meta.Get("X-Object-Meta-Fruit") != "Apple" || + meta.Get("X-Object-Meta-Veggie") != "Carrot" { + t.Error("Failed: meta not matching") + } +} + +func TestSetObjectMeta(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, + r *http.Request) { + if r.Method == "POST" && r.Header.Get("X-Object-Meta-Fruit") == "Apple" { + w.WriteHeader(202) + return + } + t.Error(errors.New("Failed: r.Method == POST && X-Object-Meta-Fruit == Apple")) + })) + defer apiServer.Close() + if err := objectstorage.SetObjectMeta(apiServer.URL+objPrefix, + tokn, "X-Object-Meta-Fruit", "Apple"); err != nil { + t.Error(err) + } +} + +func TestGetObject(t *testing.T) { + var unCompressedLen int + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + fContent, err := ioutil.ReadFile(srcFile) + if err != nil { + t.Error(err) + } + unCompressedLen = len(fContent) + w.Header().Set("Content-Length", strconv.Itoa(unCompressedLen)) + w.Header().Set("X-Object-ModTime", "93000299") + w.Header().Set("X-Object-Mode", "rwxrwxrwx") + w.Write(fContent) + return + } + t.Error(errors.New("Failed: r.Method == GET")) + })) + defer apiServer.Close() + hdr, body, err := objectstorage.GetObject(apiServer.URL+objPrefix, tokn) + if err != nil { + t.Error(err) + } + if unCompressedLen != len(body) { + t.Error(errors.New("GET: incorrect uncompressed len")) + } + if hdr.Get("X-Object-ModTime") != "93000299" || + hdr.Get("Content-Length") != strconv.Itoa(len(body)) || + hdr.Get("X-Object-Mode") != "rwxrwxrwx" { + // + t.Error(errors.New("GET: incorrect hdr")) + } +} + +func TestDeleteObject(t *testing.T) { + var apiServer = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "DELETE" { + w.WriteHeader(204) + return + } + t.Error(errors.New("Failed: r.Method == DELETE")) + })) + defer apiServer.Close() + if err := objectstorage.DeleteObject(apiServer.URL+objPrefix, tokn); err != nil { + t.Error(err) + } +}