Browse Source

adjustment of the story

Change-Id: Ibf81b7d37bd724b78bd2cfd9c4e88f1c37578268
Nate Johnston 10 months ago
parent
commit
afe7bbf0ac
4 changed files with 23 additions and 281 deletions
  1. 3
    3
      Makefile
  2. 12
    19
      STORY.md
  3. 8
    8
      events_json_fixtures_test.go
  4. 0
    251
      qualys/assets_test.go

+ 3
- 3
Makefile View File

@@ -7,12 +7,12 @@ SOURCE?=./...
7 7
 
8 8
 env:
9 9
 	@echo "Running build"
10
-	$(HOME)/go/bin/vgo build
10
+#	$(HOME)/go/bin/vgo build
11 11
 
12 12
 test: 
13 13
 	@echo "Running tests"
14
-	$(HOME)/go/bin/vgo test $(SOURCE) -cover
14
+#	$(HOME)/go/bin/vgo test $(SOURCE) -cover
15 15
 
16 16
 fmt: 
17 17
 	@echo "Running fmt"
18
-	go fmt $(SOURCE)
18
+#	go fmt $(SOURCE)

+ 12
- 19
STORY.md View File

@@ -102,28 +102,21 @@ compared against the intended messages.  This facilitates good testing.
102 102
 
103 103
 ## Fate of the Project
104 104
 
105
-The OSEL project was implemented and installed into production.  There were two
106
-problems with it.
107
-
108
-The first to become visible is that there was no exponential backoff for the
109
-AMQP connection to the OpenStack control plane's RabbitMQ.  When that RabbitMQ
110
-had issues - which was surprisingly often - OSEL would hanner away, trying to
111
-connect to it.  That would not be too much of an issue; despite what was
112
-effectively an infinite loop, CPU usage was not extreme.  The real problem was
113
-that connection failures were logged - and logs could become several gigabytes
114
-in a matter of hours.  This was mitigated by the OpenStack operations team
115
-rotating the logs hourly, and alerting if an hour's worth of logs exceeded a
116
-set size.  It was my intention to use one of the many [exponential backoff
105
+The OSEL project was implemented and installed into production.  There was a
106
+problem with it.
107
+
108
+There was no exponential backoff for the AMQP connection to the OpenStack
109
+control plane's RabbitMQ.  When that RabbitMQ had issues - which was
110
+surprisingly often - OSEL would hanner away, trying to connect to it.  That
111
+would not be too much of an issue; despite what was effectively an infinite
112
+loop, CPU usage was not extreme.  The real problem was that connection failures
113
+were logged - and logs could become several gigabytes in a matter of hours.
114
+This was mitigated by the OpenStack operations team rotating the logs hourly,
115
+and alerting if an hour's worth of logs exceeded a set size.  It was my
116
+intention to use one of the many [exponential backoff
117 117
 modules](https://github.com/cenkalti/backoff) available out there to make this
118 118
 more graceful.
119 119
 
120
-The second - and fatal - issue is that S3 RiskFabric was not configured to
121
-ingest from Qualys scans more than once a day.  Since Qualys was already
122
-scanning the CIDR block that corresponded to our OpenStack instances once a
123
-day, we were essentially just adding noise to the system.  The frequency of the
124
-S3-Qualys imports could not be easily altered, and as a result the project was
125
-shelved. 
126
-
127 120
 ## Remaining Work
128 121
 
129 122
 If OSEL were ever to be un-shelved, here are a few of the things that I wish I

+ 8
- 8
events_json_fixtures_test.go View File

@@ -17,7 +17,7 @@ const (
17 17
     "payload": {
18 18
       "port": {
19 19
         "status": "DOWN",
20
-        "binding:host_id": "oscomp-ch2-a06",
20
+        "binding:host_id": "controller",
21 21
         "name": "",
22 22
         "allowed_address_pairs": [
23 23
 
@@ -61,7 +61,7 @@ const (
61 61
     "_context_project_id": "0b65cf220eab4a3cbd68681d188d7dc7",
62 62
     "_context_timestamp": "2016-10-03 18:40:34.477012",
63 63
     "_context_user_name": "neutron",
64
-    "publisher_id": "network.osctrl-ch2-a03",
64
+    "publisher_id": "network.controller03",
65 65
     "message_id": "71047538-531f-4aca-be09-a31bec441d16"
66 66
   }
67 67
 
@@ -103,7 +103,7 @@ const (
103 103
      "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
104 104
      "_context_timestamp":"2016-10-03 17:50:59.925462",
105 105
      "_context_user_name":"admin",
106
-     "publisher_id":"network.osctrl-ch2-a03",
106
+     "publisher_id":"network.controller03",
107 107
      "message_id":"6c93e24f-0892-494b-8e68-46252ceb9611"
108 108
   }
109 109
   `
@@ -144,7 +144,7 @@ const (
144 144
      "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
145 145
      "_context_timestamp":"2016-10-03 18:05:35.769947",
146 146
      "_context_user_name":"admin",
147
-     "publisher_id":"network.osctrl-ch2-a03",
147
+     "publisher_id":"network.controller03",
148 148
      "message_id":"f67b70d5-a782-4c5e-a274-a7ff197b73ec"
149 149
   }
150 150
   `
@@ -184,7 +184,7 @@ const (
184 184
      "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
185 185
      "_context_timestamp":"2016-10-03 17:32:25.665588",
186 186
      "_context_user_name":"admin",
187
-     "publisher_id":"network.osctrl-ch2-a03",
187
+     "publisher_id":"network.controller03",
188 188
      "message_id":"4df01871-8bdb-4b85-bb34-cbff59ee6034"
189 189
   }
190 190
   `
@@ -224,7 +224,7 @@ const (
224 224
      "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
225 225
      "_context_timestamp":"2016-10-03 17:36:58.712962",
226 226
      "_context_user_name":"admin",
227
-     "publisher_id":"network.osctrl-ch2-a03",
227
+     "publisher_id":"network.controller03",
228 228
      "message_id":"e2d7c089-8194-4523-8f84-ae22db497f60"
229 229
   }
230 230
   `
@@ -264,7 +264,7 @@ const (
264 264
      "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
265 265
      "_context_timestamp":"2016-10-03 18:09:11.876789",
266 266
      "_context_user_name":"admin",
267
-     "publisher_id":"network.osctrl-ch2-a03",
267
+     "publisher_id":"network.controller03",
268 268
      "message_id":"afb043b6-fa56-470b-b17e-984fb4cb6505"
269 269
   }
270 270
   `
@@ -292,7 +292,7 @@ const (
292 292
     "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
293 293
     "_context_timestamp": "2016-10-03 18:14:32.962116",
294 294
     "_context_user_name": "admin",
295
-    "publisher_id": "network.osctrl-ch2-a03",
295
+    "publisher_id": "network.controller03",
296 296
     "message_id": "9bc5106c-a08b-4cda-9311-20bc16bc3008"
297 297
   }
298 298
   `

+ 0
- 251
qualys/assets_test.go View File

@@ -1,251 +0,0 @@
1
-package qualys
2
-
3
-import (
4
-	"fmt"
5
-	"net/http"
6
-	"reflect"
7
-	"testing"
8
-)
9
-
10
-func TestListAssetGroups(t *testing.T) {
11
-
12
-	cases := []struct {
13
-		name     string
14
-		response string
15
-		expected []AssetGroup
16
-		opts     *ListAssetGroupOptions
17
-		isErr    bool
18
-	}{
19
-		{
20
-			name:     "ListAssetGroups - single item, without list options",
21
-			response: assetGroupsXMLSingleGroup,
22
-			expected: []AssetGroup{
23
-				{
24
-					ID:    "1759735",
25
-					Title: "AG - Elastic Cloud Dynamic Perimeter",
26
-					IPs: AssetGroupIPs{
27
-						IPs:      []string{"10.1.1.1", "10.10.10.11"},
28
-						IPRanges: nil,
29
-					},
30
-				},
31
-			},
32
-			opts: nil,
33
-		},
34
-		{
35
-			name:     "ListAssetGroups - single item, with list options",
36
-			response: assetGroupsXMLSingleGroup,
37
-			expected: []AssetGroup{
38
-				{
39
-					ID:    "1759735",
40
-					Title: "AG - Elastic Cloud Dynamic Perimeter",
41
-					IPs: AssetGroupIPs{
42
-						IPs:      []string{"10.1.1.1", "10.10.10.11"},
43
-						IPRanges: nil,
44
-					},
45
-				},
46
-			},
47
-			opts: &ListAssetGroupOptions{Ids: []string{}},
48
-		},
49
-		{
50
-			name:     "ListAssetGroups - multi item",
51
-			response: assetGroupsXMLMultiGroups,
52
-			expected: []AssetGroup{
53
-				{ID: "1759734", Title: "AG - New"},
54
-				{ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter",
55
-					IPs: AssetGroupIPs{
56
-						IPs:      []string{"10.10.10.14"},
57
-						IPRanges: []string{"10.10.10.3-10.10.10.6"},
58
-					},
59
-				},
60
-			},
61
-			opts: &ListAssetGroupOptions{Ids: []string{"1", "2"}},
62
-		},
63
-	}
64
-
65
-	for _, c := range cases {
66
-		setup()
67
-		defer teardown()
68
-		mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
69
-			testMethod(t, r, "GET")
70
-			fmt.Fprint(w, c.response)
71
-		})
72
-
73
-		assetGroups, _, err := client.Assets.ListAssetGroups(c.opts)
74
-		if err != nil {
75
-			t.Errorf("Assets.ListAssetGroups returned error: %v", err)
76
-		}
77
-
78
-		if !reflect.DeepEqual(assetGroups, c.expected) {
79
-			t.Errorf("Assets.ListAssetGroups case: %s returned %+v, expected %+v", c.name, assetGroups, c.expected)
80
-		}
81
-	}
82
-}
83
-
84
-func TestGetAssetGroupByID(t *testing.T) {
85
-	setup()
86
-	defer teardown()
87
-
88
-	mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
89
-		testMethod(t, r, "GET")
90
-		fmt.Fprint(w, assetGroupsXMLSingleGroup)
91
-	})
92
-
93
-	groupID := "1759735"
94
-
95
-	assetGroup, _, err := client.Assets.GetAssetGroupByID(groupID)
96
-	if err != nil {
97
-		t.Errorf("Assets.GetAssetGroupByID(%s) returned error: %v", groupID, err)
98
-	}
99
-
100
-	expected := &AssetGroup{
101
-		ID:    "1759735",
102
-		Title: "AG - Elastic Cloud Dynamic Perimeter",
103
-		IPs: AssetGroupIPs{
104
-			IPs:      []string{"10.1.1.1", "10.10.10.11"},
105
-			IPRanges: nil,
106
-		},
107
-	}
108
-	if !reflect.DeepEqual(assetGroup, expected) {
109
-		t.Errorf("Assets.GetAssetGroupByID(%s) returned %+v, expected %+v", groupID, assetGroup, expected)
110
-	}
111
-}
112
-
113
-func TestAddIPsToGroup(t *testing.T) {
114
-	setup()
115
-	defer teardown()
116
-
117
-	groupID := "1759735"
118
-	ip := "10.10.10.10"
119
-
120
-	mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
121
-		testMethod(t, r, "POST")
122
-		if r.FormValue("add_ips") != ip {
123
-			t.Errorf("Request form data did not include the correct IP")
124
-		}
125
-		if r.FormValue("id") != groupID {
126
-			t.Errorf("Request form data did not include the correct asset group ID")
127
-		}
128
-		fmt.Fprint(w, assetGroupsAddIPsResponse)
129
-	})
130
-	opts := &AddIPsToGroupOptions{
131
-		GroupID: groupID,
132
-		IPs:     []string{ip},
133
-	}
134
-
135
-	_, err := client.Assets.AddIPsToGroup(opts)
136
-	if err != nil {
137
-		t.Errorf("Assets.AddIPsToGroup returned error: %v", err)
138
-	}
139
-}
140
-
141
-func TestAssetGroupContainsIP(t *testing.T) {
142
-	cases := []struct {
143
-		name     string
144
-		ip       string
145
-		group    *AssetGroup
146
-		expected bool
147
-	}{
148
-		{
149
-			name:     "AssetGroup.ContainsIP - nil",
150
-			ip:       "10.1.1.1",
151
-			group:    &AssetGroup{ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter"},
152
-			expected: false,
153
-		},
154
-		{
155
-			name: "AssetGroup.ContainsIP - empty",
156
-			ip:   "10.1.1.1",
157
-			group: &AssetGroup{
158
-				ID:    "1759735",
159
-				Title: "AG - Elastic Cloud Dynamic Perimeter",
160
-				IPs:   AssetGroupIPs{}},
161
-			expected: false,
162
-		},
163
-		{
164
-			name: "AssetGroup.ContainsIP - single item list",
165
-			ip:   "10.1.1.1",
166
-			group: &AssetGroup{
167
-				ID:    "1759735",
168
-				Title: "AG - Elastic Cloud Dynamic Perimeter",
169
-				IPs: AssetGroupIPs{
170
-					IPs:      []string{"10.1.1.1"},
171
-					IPRanges: []string{},
172
-				},
173
-			},
174
-			expected: true,
175
-		},
176
-		{
177
-			name: "AssetGroup.ContainsIP - multi item list",
178
-			ip:   "10.1.1.1",
179
-			group: &AssetGroup{
180
-				ID:    "1759735",
181
-				Title: "AG - Elastic Cloud Dynamic Perimeter",
182
-				IPs: AssetGroupIPs{
183
-					IPs:      []string{"10.1.1.1"},
184
-					IPRanges: []string{"10.10.1.1-10.10.10.10"},
185
-				},
186
-			},
187
-			expected: true,
188
-		},
189
-	}
190
-	for _, c := range cases {
191
-		contains := c.group.ContainsIP(c.ip)
192
-		if contains != c.expected {
193
-			t.Errorf("%s - AssetGroup.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected)
194
-		}
195
-	}
196
-}
197
-
198
-func TestAssetGroupIPsContainsIP(t *testing.T) {
199
-	group := AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.10.3-10.10.10.6"}}
200
-
201
-	cases := []struct {
202
-		name     string
203
-		ip       string
204
-		group    AssetGroupIPs
205
-		expected bool
206
-	}{
207
-		{
208
-			name:     "AssetGroupIPs.ContainsIP - IP value match",
209
-			ip:       "10.0.1.1",
210
-			group:    group,
211
-			expected: true,
212
-		},
213
-		{
214
-			name:     "AssetGroupIPs.ContainsIP - IP value no match",
215
-			ip:       "192.0.1.1",
216
-			group:    group,
217
-			expected: false,
218
-		},
219
-		{
220
-			name:     "AssetGroupIPs.ContainsIP - IP Range value match",
221
-			ip:       "10.10.10.4",
222
-			group:    group,
223
-			expected: true,
224
-		},
225
-		{
226
-			name:     "AssetGroupIPs.ContainsIP - IP Range value no match",
227
-			ip:       "10.10.10.1",
228
-			group:    group,
229
-			expected: false,
230
-		},
231
-		{
232
-			name:     "AssetGroupIPs.ContainsIP - IP Range value match",
233
-			ip:       "10.10.0.4",
234
-			group:    AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.0.0-10.10.10.6"}},
235
-			expected: true,
236
-		},
237
-		{
238
-			name:     "AssetGroupIPs.ContainsIP - IP Range value no match",
239
-			ip:       "10.10.0.4",
240
-			group:    AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.1.3-10.10.10.6"}},
241
-			expected: false,
242
-		},
243
-	}
244
-
245
-	for _, c := range cases {
246
-		contains := c.group.ContainsIP(c.ip)
247
-		if contains != c.expected {
248
-			t.Errorf("%s - AssetGroupIPs.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected)
249
-		}
250
-	}
251
-}

Loading…
Cancel
Save