Browse Source

Terminal state transitions in transactions

Multiple spots were not using DB transactions when processing the terminal
state transitions (error, abort, finish, timeout). The pattern looked like
this:

    node_info.fsm_event(istate.Events.error)
    # more code
    node_info.finished(error='Oops!')

which led to brief periodes of state inconsistency of NodeInfo records in
the DB.

This patch refactors the NodeInfo.finished() method to require a terminal state
transition to perform as part of the NodeInfo state update:

   NodeInfo().finished(istate.Events.finish)
   NodeInfo().finished(istate.Events.abort, 'Canceled by operator')

This patch also introduces a new state: aborting to allow the inspector to
try call power-off the node before marking the introspection aborted.

There's a new DB migration since the new state implies a schema change too
(Enum).

Closes-Bug: #1721233
Closes-Bug: #1721230
Closes-Bug: #1723384

Change-Id: I0bb051d1956a996ed006d55a5ca2d670d9455047
changes/28/510928/10
dparalen 5 years ago
parent
commit
7e72ceffd1
  1. 289
      doc/source/images/states.svg
  2. 42
      ironic_inspector/introspect.py
  3. 16
      ironic_inspector/introspection_state.py
  4. 43
      ironic_inspector/migrations/versions/18440d0834af_introducing_the_aborting_state.py
  5. 21
      ironic_inspector/node_cache.py
  6. 19
      ironic_inspector/process.py
  7. 42
      ironic_inspector/test/unit/test_introspect.py
  8. 6
      ironic_inspector/test/unit/test_node_cache.py
  9. 53
      ironic_inspector/test/unit/test_process.py
  10. 12
      releasenotes/notes/db-status-consistency-enhancements-f97fbaccfc81a60b.yaml

289
doc/source/images/states.svg

@ -4,227 +4,240 @@
<!-- Generated by graphviz version 2.40.1 (20161225.0304)
-->
<!-- Title: Ironic Inspector states Pages: 1 -->
<svg width="842pt" height="383pt"
viewBox="0.00 0.00 841.68 383.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 379)">
<svg width="851pt" height="382pt"
viewBox="0.00 0.00 851.12 382.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 378)">
<title>Ironic Inspector states</title>
<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-379 837.6827,-379 837.6827,4 -4,4"/>
<!-- enrolling -->
<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-378 847.1163,-378 847.1163,4 -4,4"/>
<!-- aborting -->
<g id="node1" class="node">
<title>enrolling</title>
<ellipse fill="none" stroke="#000000" cx="32.7967" cy="-223" rx="32.5946" ry="18"/>
<text text-anchor="middle" x="32.7967" y="-219.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">enrolling</text>
<title>aborting</title>
<ellipse fill="none" stroke="#000000" cx="32.7967" cy="-161" rx="30.9953" ry="18"/>
<text text-anchor="middle" x="32.7967" y="-157.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">aborting</text>
</g>
<!-- error -->
<g id="node2" class="node">
<title>error</title>
<ellipse fill="none" stroke="#000000" cx="159.1451" cy="-190" rx="27" ry="18"/>
<text text-anchor="middle" x="159.1451" y="-186.7" font-family="Times,serif" font-size="11.00" fill="#ff0000">error</text>
<ellipse fill="none" stroke="#000000" cx="168.5787" cy="-189" rx="27" ry="18"/>
<text text-anchor="middle" x="168.5787" y="-185.7" font-family="Times,serif" font-size="11.00" fill="#ff0000">error</text>
</g>
<!-- enrolling&#45;&gt;error -->
<!-- aborting&#45;&gt;error -->
<g id="edge1" class="edge">
<title>enrolling&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M62.4198,-215.263C80.9292,-210.4286 104.8518,-204.1805 124.2748,-199.1075"/>
<polygon fill="#000000" stroke="#000000" points="125.1781,-202.4891 133.969,-196.5756 123.4091,-195.7163 125.1781,-202.4891"/>
<text text-anchor="middle" x="98.8693" y="-211" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- enrolling&#45;&gt;error -->
<g id="edge3" class="edge">
<title>enrolling&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M50.793,-207.6437C59.9733,-200.8122 71.6953,-193.5255 83.5934,-190 95.8748,-186.3609 109.7805,-185.4983 122.284,-185.8233"/>
<polygon fill="#000000" stroke="#000000" points="122.2248,-189.3248 132.3907,-186.3408 122.5828,-182.3339 122.2248,-189.3248"/>
<text text-anchor="middle" x="98.8693" y="-192" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- processing -->
<g id="node3" class="node">
<title>processing</title>
<ellipse fill="none" stroke="#000000" cx="796.5702" cy="-294" rx="37.2253" ry="18"/>
<text text-anchor="middle" x="796.5702" y="-290.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">processing</text>
<title>aborting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M52.0855,-175.1999C56.7664,-179.0856 61.5754,-183.4801 65.5934,-188 75.4225,-199.0568 70.6085,-208.9109 83.5934,-216 101.8225,-225.9522 124.2849,-218.0586 141.5376,-208.2902"/>
<polygon fill="#000000" stroke="#000000" points="143.8354,-210.9866 150.5265,-202.7719 140.1731,-205.021 143.8354,-210.9866"/>
<text text-anchor="middle" x="103.5861" y="-222" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort_end</text>
</g>
<!-- enrolling&#45;&gt;processing -->
<!-- aborting&#45;&gt;error -->
<g id="edge2" class="edge">
<title>enrolling&#45;&gt;processing</title>
<path fill="none" stroke="#000000" d="M38.9232,-240.6973C53.3543,-278.7629 93.3124,-365 159.1451,-365 159.1451,-365 159.1451,-365 664.6261,-365 705.9926,-365 746.88,-337.6031 771.9214,-316.8465"/>
<polygon fill="#000000" stroke="#000000" points="774.4666,-319.2748 779.784,-310.111 769.9125,-313.9587 774.4666,-319.2748"/>
<text text-anchor="middle" x="423.3931" y="-367" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
<title>aborting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M55.9956,-173.0111C64.4643,-176.8635 74.2575,-180.7193 83.5934,-183 98.8555,-186.7284 116.0834,-188.3958 131.0182,-189.0706"/>
<polygon fill="#000000" stroke="#000000" points="131.3207,-192.5819 141.427,-189.4018 131.5434,-185.5854 131.3207,-192.5819"/>
<text text-anchor="middle" x="103.5861" y="-190" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- error&#45;&gt;error -->
<g id="edge4" class="edge">
<g id="edge6" class="edge">
<title>error&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M153.7522,-207.7817C152.8358,-217.3149 154.6334,-226 159.1451,-226 161.8945,-226 163.636,-222.7749 164.3696,-218.0981"/>
<polygon fill="#000000" stroke="#000000" points="167.8742,-217.8376 164.5381,-207.7817 160.8752,-217.7232 167.8742,-217.8376"/>
<text text-anchor="middle" x="159.1451" y="-228" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort</text>
<path fill="none" stroke="#000000" d="M163.1858,-206.7817C162.2694,-216.3149 164.067,-225 168.5787,-225 171.3281,-225 173.0695,-221.7749 173.8032,-217.0981"/>
<polygon fill="#000000" stroke="#000000" points="177.3078,-216.8376 173.9716,-206.7817 170.3088,-216.7232 177.3078,-216.8376"/>
<text text-anchor="middle" x="168.5787" y="-227" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort</text>
</g>
<!-- error&#45;&gt;error -->
<g id="edge5" class="edge">
<g id="edge7" class="edge">
<title>error&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M150.3671,-207.1418C145.2035,-224.585 148.1295,-244 159.1451,-244 168.0953,-244 171.7051,-231.183 169.9745,-217.0206"/>
<polygon fill="#000000" stroke="#000000" points="173.3833,-216.2213 167.9232,-207.1418 166.5295,-217.6445 173.3833,-216.2213"/>
<text text-anchor="middle" x="159.1451" y="-246" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
<path fill="none" stroke="#000000" d="M159.8007,-206.1418C154.6371,-223.585 157.5631,-243 168.5787,-243 177.5289,-243 181.1387,-230.183 179.4081,-216.0206"/>
<polygon fill="#000000" stroke="#000000" points="182.8169,-215.2213 177.3568,-206.1418 175.9631,-216.6445 182.8169,-215.2213"/>
<text text-anchor="middle" x="168.5787" y="-245" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- reapplying -->
<g id="node4" class="node">
<g id="node5" class="node">
<title>reapplying</title>
<ellipse fill="none" stroke="#000000" cx="289.8062" cy="-207" rx="37.219" ry="18"/>
<text text-anchor="middle" x="289.8062" y="-203.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">reapplying</text>
<ellipse fill="none" stroke="#000000" cx="299.2398" cy="-206" rx="37.219" ry="18"/>
<text text-anchor="middle" x="299.2398" y="-202.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">reapplying</text>
</g>
<!-- error&#45;&gt;reapplying -->
<g id="edge6" class="edge">
<g id="edge8" class="edge">
<title>error&#45;&gt;reapplying</title>
<path fill="none" stroke="#000000" d="M170.8266,-206.6885C178.8931,-216.6767 190.5869,-228.473 204.1451,-234 222.0093,-241.2823 242.6135,-235.2198 259.137,-227.0102"/>
<polygon fill="#000000" stroke="#000000" points="261.2238,-229.8599 268.3427,-222.0131 257.8843,-223.7078 261.2238,-229.8599"/>
<text text-anchor="middle" x="219.421" y="-239" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
<path fill="none" stroke="#000000" d="M180.2602,-205.6885C188.3266,-215.6767 200.0205,-227.473 213.5787,-233 231.4429,-240.2823 252.047,-234.2198 268.5706,-226.0102"/>
<polygon fill="#000000" stroke="#000000" points="270.6574,-228.8599 277.7763,-221.0131 267.3179,-222.7078 270.6574,-228.8599"/>
<text text-anchor="middle" x="228.8546" y="-238" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
</g>
<!-- starting -->
<g id="node5" class="node">
<g id="node6" class="node">
<title>starting</title>
<ellipse fill="none" stroke="#000000" cx="548.712" cy="-106" rx="28.6835" ry="18"/>
<text text-anchor="middle" x="548.712" y="-102.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">starting</text>
<ellipse fill="none" stroke="#000000" cx="558.1456" cy="-134" rx="28.6835" ry="18"/>
<text text-anchor="middle" x="558.1456" y="-130.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">starting</text>
</g>
<!-- error&#45;&gt;starting -->
<g id="edge7" class="edge">
<g id="edge9" class="edge">
<title>error&#45;&gt;starting</title>
<path fill="none" stroke="#000000" d="M185.7778,-186.305C191.8127,-185.5078 198.1957,-184.6962 204.1451,-184 220.6565,-182.0679 486.8945,-161.2162 501.8707,-154 512.9535,-148.6598 522.867,-139.5514 530.6885,-130.6948"/>
<polygon fill="#000000" stroke="#000000" points="533.5841,-132.6853 537.257,-122.7474 528.1884,-128.2258 533.5841,-132.6853"/>
<text text-anchor="middle" x="360.1914" y="-174" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
<path fill="none" stroke="#000000" d="M195.1811,-184.4987C201.2175,-183.5785 207.6089,-182.6838 213.5787,-182 279.4894,-174.4508 447.7374,-178.9859 511.3043,-160 517.5036,-158.1484 523.7925,-155.3833 529.6645,-152.3379"/>
<polygon fill="#000000" stroke="#000000" points="531.4549,-155.3469 538.476,-147.4125 528.0394,-149.2367 531.4549,-155.3469"/>
<text text-anchor="middle" x="369.625" y="-177" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
</g>
<!-- enrolling -->
<g id="node3" class="node">
<title>enrolling</title>
<ellipse fill="none" stroke="#000000" cx="32.7967" cy="-215" rx="32.5946" ry="18"/>
<text text-anchor="middle" x="32.7967" y="-211.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">enrolling</text>
</g>
<!-- enrolling&#45;&gt;error -->
<g id="edge3" class="edge">
<title>enrolling&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M51.1993,-199.8824C55.8972,-196.0142 60.9359,-191.8577 65.5934,-188 73.6133,-181.3573 73.8017,-176.5451 83.5934,-173 100.6616,-166.8205 120.5641,-170.0469 136.8166,-175.1731"/>
<polygon fill="#000000" stroke="#000000" points="135.8115,-178.5296 146.4064,-178.5432 138.1324,-171.9256 135.8115,-178.5296"/>
<text text-anchor="middle" x="103.5861" y="-175" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- enrolling&#45;&gt;error -->
<g id="edge5" class="edge">
<title>enrolling&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M52.9994,-200.4234C57.4523,-196.6728 61.9286,-192.4341 65.5934,-188 76.2367,-175.1224 69.2274,-163.5281 83.5934,-155 98.8749,-145.9284 106.7294,-149.3503 123.5787,-155 131.5424,-157.6703 139.2218,-162.3686 145.9131,-167.4359"/>
<polygon fill="#000000" stroke="#000000" points="143.8604,-170.2793 153.8075,-173.9271 148.3063,-164.8724 143.8604,-170.2793"/>
<text text-anchor="middle" x="103.5861" y="-157" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- processing -->
<g id="node4" class="node">
<title>processing</title>
<ellipse fill="none" stroke="#000000" cx="806.0038" cy="-280" rx="37.2253" ry="18"/>
<text text-anchor="middle" x="806.0038" y="-276.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">processing</text>
</g>
<!-- enrolling&#45;&gt;processing -->
<g id="edge4" class="edge">
<title>enrolling&#45;&gt;processing</title>
<path fill="none" stroke="#000000" d="M39.1648,-232.6774C54.6464,-272.1459 98.2329,-364 168.5787,-364 168.5787,-364 168.5787,-364 674.0597,-364 718.8487,-364 760.6945,-329.1311 784.7975,-304.3245"/>
<polygon fill="#000000" stroke="#000000" points="787.3872,-306.6797 791.6917,-296.9987 782.2895,-301.8824 787.3872,-306.6797"/>
<text text-anchor="middle" x="432.8267" y="-366" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
</g>
<!-- processing&#45;&gt;error -->
<g id="edge11" class="edge">
<g id="edge13" class="edge">
<title>processing&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M764.8941,-303.4514C738.3628,-310.531 699.3647,-319 664.6261,-319 289.8062,-319 289.8062,-319 289.8062,-319 250.2686,-319 233.8818,-321.0567 204.1451,-295 181.4063,-275.0751 169.7917,-241.6479 164.1159,-217.864"/>
<polygon fill="#000000" stroke="#000000" points="167.5185,-217.0393 161.9695,-208.0138 160.6789,-218.5297 167.5185,-217.0393"/>
<text text-anchor="middle" x="486.5948" y="-321" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
<path fill="none" stroke="#000000" d="M778.8595,-292.6534C752.6321,-303.7077 711.4854,-318 674.0597,-318 299.2398,-318 299.2398,-318 299.2398,-318 259.7022,-318 243.3154,-320.0567 213.5787,-294 190.8399,-274.0751 179.2253,-240.6479 173.5495,-216.864"/>
<polygon fill="#000000" stroke="#000000" points="176.952,-216.0393 171.4031,-207.0138 170.1125,-217.5297 176.952,-216.0393"/>
<text text-anchor="middle" x="496.0284" y="-320" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- processing&#45;&gt;error -->
<g id="edge13" class="edge">
<g id="edge15" class="edge">
<title>processing&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M760.2154,-290.0289C722.0298,-286.2059 660.2037,-281 606.6691,-281 289.8062,-281 289.8062,-281 289.8062,-281 249.4674,-281 236.5959,-274.9619 204.1451,-251 191.6594,-241.7805 181.1271,-228.1091 173.3984,-216.0446"/>
<polygon fill="#000000" stroke="#000000" points="176.2299,-213.9642 168.0487,-207.2323 170.2462,-217.5968 176.2299,-213.9642"/>
<text text-anchor="middle" x="486.5948" y="-283" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
<path fill="none" stroke="#000000" d="M768.7865,-280C710.5917,-280 594.5102,-280 496.0284,-280 299.2398,-280 299.2398,-280 299.2398,-280 258.901,-280 246.0295,-273.9619 213.5787,-250 201.093,-240.7805 190.5607,-227.1091 182.832,-215.0446"/>
<polygon fill="#000000" stroke="#000000" points="185.6635,-212.9642 177.4823,-206.2323 179.6798,-216.5968 185.6635,-212.9642"/>
<text text-anchor="middle" x="496.0284" y="-282" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- finished -->
<g id="node6" class="node">
<g id="node7" class="node">
<title>finished</title>
<ellipse fill="none" stroke="#000000" cx="423.3931" cy="-207" rx="29.8518" ry="18"/>
<text text-anchor="middle" x="423.3931" y="-203.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">finished</text>
<ellipse fill="none" stroke="#000000" cx="432.8267" cy="-206" rx="29.8518" ry="18"/>
<text text-anchor="middle" x="432.8267" y="-202.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">finished</text>
</g>
<!-- processing&#45;&gt;finished -->
<g id="edge12" class="edge">
<g id="edge14" class="edge">
<title>processing&#45;&gt;finished</title>
<path fill="none" stroke="#000000" d="M762.8931,-286.1487C693.4478,-269.9587 534.7324,-232.9569 461.5916,-215.9053"/>
<polygon fill="#000000" stroke="#000000" points="461.9756,-212.4011 451.4421,-213.5391 460.3862,-219.2183 461.9756,-212.4011"/>
<text text-anchor="middle" x="606.6691" y="-253" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
<path fill="none" stroke="#000000" d="M771.6489,-273.1875C702.0548,-259.3872 544.8371,-228.2113 471.6566,-213.6999"/>
<polygon fill="#000000" stroke="#000000" points="471.9795,-210.1958 461.4897,-211.6838 470.6179,-217.0621 471.9795,-210.1958"/>
<text text-anchor="middle" x="616.1027" y="-245" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
</g>
<!-- reapplying&#45;&gt;error -->
<g id="edge14" class="edge">
<g id="edge16" class="edge">
<title>reapplying&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M252.5474,-205.7684C237.4566,-204.9118 219.906,-203.46 204.1451,-201 200.8871,-200.4915 197.5185,-199.8637 194.1582,-199.1679"/>
<polygon fill="#000000" stroke="#000000" points="194.7462,-195.7127 184.2212,-196.9277 193.2066,-202.5413 194.7462,-195.7127"/>
<text text-anchor="middle" x="219.421" y="-206" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
<path fill="none" stroke="#000000" d="M261.981,-204.7684C246.8902,-203.9118 229.3396,-202.46 213.5787,-200 210.3207,-199.4915 206.9521,-198.8637 203.5918,-198.1679"/>
<polygon fill="#000000" stroke="#000000" points="204.1798,-194.7127 193.6548,-195.9277 202.6402,-201.5413 204.1798,-194.7127"/>
<text text-anchor="middle" x="228.8546" y="-205" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- reapplying&#45;&gt;error -->
<g id="edge17" class="edge">
<g id="edge19" class="edge">
<title>reapplying&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M260.1535,-196.0759C252.015,-193.5688 243.1124,-191.2685 234.6969,-190 222.3581,-188.1401 208.7564,-187.6625 196.5344,-187.7968"/>
<polygon fill="#000000" stroke="#000000" points="196.1254,-184.3057 186.2154,-188.0528 196.2992,-191.3036 196.1254,-184.3057"/>
<text text-anchor="middle" x="219.421" y="-192" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
<path fill="none" stroke="#000000" d="M269.5871,-195.0759C261.4486,-192.5688 252.546,-190.2685 244.1305,-189 231.7917,-187.1401 218.19,-186.6625 205.968,-186.7968"/>
<polygon fill="#000000" stroke="#000000" points="205.559,-183.3057 195.649,-187.0528 205.7328,-190.3036 205.559,-183.3057"/>
<text text-anchor="middle" x="228.8546" y="-191" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- reapplying&#45;&gt;reapplying -->
<g id="edge16" class="edge">
<g id="edge18" class="edge">
<title>reapplying&#45;&gt;reapplying</title>
<path fill="none" stroke="#000000" d="M277.4022,-224.0373C274.8708,-233.8579 279.0054,-243 289.8062,-243 296.5567,-243 300.7033,-239.4289 302.2458,-234.3529"/>
<polygon fill="#000000" stroke="#000000" points="305.7448,-234.0251 302.2103,-224.0373 298.7449,-234.0494 305.7448,-234.0251"/>
<text text-anchor="middle" x="289.8062" y="-245" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
<path fill="none" stroke="#000000" d="M286.8358,-223.0373C284.3044,-232.8579 288.439,-242 299.2398,-242 305.9903,-242 310.1369,-238.4289 311.6794,-233.3529"/>
<polygon fill="#000000" stroke="#000000" points="315.1784,-233.0251 311.6438,-223.0373 308.1785,-233.0494 315.1784,-233.0251"/>
<text text-anchor="middle" x="299.2398" y="-244" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
</g>
<!-- reapplying&#45;&gt;finished -->
<g id="edge15" class="edge">
<g id="edge17" class="edge">
<title>reapplying&#45;&gt;finished</title>
<path fill="none" stroke="#000000" d="M325.6382,-201.6062C340.9792,-199.936 359.097,-198.8108 375.4673,-200 378.3764,-200.2113 381.3771,-200.4939 384.3911,-200.8245"/>
<polygon fill="#000000" stroke="#000000" points="384.2187,-204.3302 394.5761,-202.1002 385.0886,-197.3845 384.2187,-204.3302"/>
<text text-anchor="middle" x="360.1914" y="-202" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
<path fill="none" stroke="#000000" d="M336.4512,-206C353.8759,-206 374.6665,-206 392.4694,-206"/>
<polygon fill="#000000" stroke="#000000" points="392.5582,-209.5001 402.5581,-206 392.5581,-202.5001 392.5582,-209.5001"/>
<text text-anchor="middle" x="369.625" y="-208" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
</g>
<!-- starting&#45;&gt;error -->
<g id="edge18" class="edge">
<g id="edge20" class="edge">
<title>starting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M520.9066,-111.2751C462.18,-122.5338 321.2756,-150.1669 204.1451,-178 200.9199,-178.7664 197.5735,-179.5951 194.2274,-180.4465"/>
<polygon fill="#000000" stroke="#000000" points="193.1069,-177.1215 184.312,-183.0292 194.8713,-183.8955 193.1069,-177.1215"/>
<text text-anchor="middle" x="360.1914" y="-149" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
<path fill="none" stroke="#000000" d="M529.2657,-135.6441C469.6892,-139.4273 329.0596,-150.383 213.5787,-175 209.9228,-175.7793 206.1385,-176.7264 202.3881,-177.7565"/>
<polygon fill="#000000" stroke="#000000" points="201.2581,-174.4405 192.6514,-180.6191 203.2326,-181.1563 201.2581,-174.4405"/>
<text text-anchor="middle" x="369.625" y="-153" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
</g>
<!-- starting&#45;&gt;error -->
<g id="edge21" class="edge">
<g id="edge22" class="edge">
<title>starting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M519.8505,-106.9285C466.0809,-109.3456 347.44,-117.9303 252.6969,-148 231.0771,-154.8617 207.8053,-165.2964 189.8809,-174.048"/>
<polygon fill="#000000" stroke="#000000" points="187.9134,-171.1174 180.5174,-178.7035 191.0299,-177.3854 187.9134,-171.1174"/>
<text text-anchor="middle" x="360.1914" y="-128" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- starting&#45;&gt;starting -->
<g id="edge19" class="edge">
<title>starting&#45;&gt;starting</title>
<path fill="none" stroke="#000000" d="M538.7888,-123.0373C536.7637,-132.8579 540.0714,-142 548.712,-142 554.1124,-142 557.4296,-138.4289 558.6637,-133.3529"/>
<polygon fill="#000000" stroke="#000000" points="562.1629,-133.0276 558.6352,-123.0373 555.163,-133.047 562.1629,-133.0276"/>
<text text-anchor="middle" x="548.712" y="-144" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
<path fill="none" stroke="#000000" d="M529.1622,-133.2847C523.2537,-133.166 517.0791,-133.0616 511.3043,-133 497.7265,-132.8551 494.331,-132.9463 480.7525,-133 424.5719,-133.2222 410.3036,-128.9589 354.3492,-134 312.9186,-137.7326 302.5786,-140.285 262.1305,-150 240.1701,-155.2745 234.4263,-156.3133 213.5787,-165 208.4706,-167.1285 203.1744,-169.6481 198.0866,-172.2412"/>
<polygon fill="#000000" stroke="#000000" points="196.4433,-169.1508 189.2389,-176.9191 199.7153,-175.3391 196.4433,-169.1508"/>
<text text-anchor="middle" x="369.625" y="-136" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- waiting -->
<g id="node7" class="node">
<g id="node8" class="node">
<title>waiting</title>
<ellipse fill="none" stroke="#000000" cx="664.6261" cy="-58" rx="28.6835" ry="18"/>
<text text-anchor="middle" x="664.6261" y="-54.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">waiting</text>
<ellipse fill="none" stroke="#000000" cx="674.0597" cy="-104" rx="28.6835" ry="18"/>
<text text-anchor="middle" x="674.0597" y="-100.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">waiting</text>
</g>
<!-- starting&#45;&gt;waiting -->
<g id="edge20" class="edge">
<g id="edge21" class="edge">
<title>starting&#45;&gt;waiting</title>
<path fill="none" stroke="#000000" d="M573.0114,-95.9376C589.8579,-88.9615 612.5106,-79.581 631.0755,-71.8933"/>
<polygon fill="#000000" stroke="#000000" points="632.4586,-75.1088 640.3587,-68.0491 629.7804,-68.6414 632.4586,-75.1088"/>
<text text-anchor="middle" x="606.6691" y="-89" font-family="Times,serif" font-size="10.00" fill="#000000">wait</text>
<path fill="none" stroke="#000000" d="M585.03,-127.042C600.5376,-123.0284 620.2591,-117.9243 637.1971,-113.5405"/>
<polygon fill="#000000" stroke="#000000" points="638.5171,-116.8143 647.3211,-110.9203 636.7631,-110.0376 638.5171,-116.8143"/>
<text text-anchor="middle" x="616.1027" y="-124" font-family="Times,serif" font-size="10.00" fill="#000000">wait</text>
</g>
<!-- finished&#45;&gt;reapplying -->
<g id="edge9" class="edge">
<g id="edge11" class="edge">
<title>finished&#45;&gt;reapplying</title>
<path fill="none" stroke="#000000" d="M393.7612,-209.8611C387.7142,-210.3357 381.3902,-210.7533 375.4673,-211 362.9165,-211.5229 349.3336,-211.2665 336.7624,-210.6879"/>
<polygon fill="#000000" stroke="#000000" points="336.6587,-207.1773 326.4853,-210.136 336.2832,-214.1673 336.6587,-207.1773"/>
<text text-anchor="middle" x="360.1914" y="-213" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
<path fill="none" stroke="#000000" d="M405.3864,-213.4152C398.7314,-214.9027 391.6021,-216.2408 384.9009,-217 371.1291,-218.5602 356.1342,-217.5595 342.5951,-215.6438"/>
<polygon fill="#000000" stroke="#000000" points="342.9994,-212.1639 332.5698,-214.0278 341.8854,-219.0747 342.9994,-212.1639"/>
<text text-anchor="middle" x="369.625" y="-219" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
</g>
<!-- finished&#45;&gt;starting -->
<g id="edge10" class="edge">
<g id="edge12" class="edge">
<title>finished&#45;&gt;starting</title>
<path fill="none" stroke="#000000" d="M441.3483,-192.5292C462.6588,-175.3541 498.3641,-146.5776 522.6991,-126.965"/>
<polygon fill="#000000" stroke="#000000" points="525.2004,-129.4443 530.7902,-120.444 520.8078,-123.994 525.2004,-129.4443"/>
<text text-anchor="middle" x="486.5948" y="-168" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
<path fill="none" stroke="#000000" d="M454.6556,-193.4586C474.8717,-181.8437 505.1314,-164.4585 527.7155,-151.4831"/>
<polygon fill="#000000" stroke="#000000" points="529.6397,-154.4142 536.5669,-146.3977 526.1525,-148.3447 529.6397,-154.4142"/>
<text text-anchor="middle" x="496.0284" y="-178" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
</g>
<!-- finished&#45;&gt;finished -->
<g id="edge8" class="edge">
<g id="edge10" class="edge">
<title>finished&#45;&gt;finished</title>
<path fill="none" stroke="#000000" d="M412.4067,-224.0373C410.1646,-233.8579 413.8267,-243 423.3931,-243 429.3721,-243 433.0448,-239.4289 434.4111,-234.3529"/>
<polygon fill="#000000" stroke="#000000" points="437.9102,-234.0265 434.3795,-224.0373 430.9102,-234.048 437.9102,-234.0265"/>
<text text-anchor="middle" x="423.3931" y="-245" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
<path fill="none" stroke="#000000" d="M421.8403,-223.0373C419.5982,-232.8579 423.2603,-242 432.8267,-242 438.8057,-242 442.4784,-238.4289 443.8447,-233.3529"/>
<polygon fill="#000000" stroke="#000000" points="447.3438,-233.0265 443.8131,-223.0373 440.3438,-233.048 447.3438,-233.0265"/>
<text text-anchor="middle" x="432.8267" y="-244" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
</g>
<!-- waiting&#45;&gt;error -->
<g id="edge22" class="edge">
<title>waiting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M635.8244,-59.3578C567.489,-63.3 390.9294,-77.7349 252.6969,-126 229.7823,-134.0008 224.248,-137.3998 204.1451,-151 196.6367,-156.0797 188.9922,-162.2552 182.186,-168.1799"/>
<polygon fill="#000000" stroke="#000000" points="179.7533,-165.6601 174.6393,-174.939 184.4235,-170.8745 179.7533,-165.6601"/>
<text text-anchor="middle" x="423.3931" y="-91" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort</text>
<!-- waiting&#45;&gt;aborting -->
<g id="edge23" class="edge">
<title>waiting&#45;&gt;aborting</title>
<path fill="none" stroke="#000000" d="M645.777,-99.8122C633.1209,-97.5459 618.1234,-94.3127 604.9869,-90 498.5822,-55.0672 481.6173,0 369.625,0 168.5787,0 168.5787,0 168.5787,0 99.5714,0 58.515,-87.5008 41.7023,-133.4889"/>
<polygon fill="#000000" stroke="#000000" points="38.3188,-132.5602 38.3021,-143.155 44.9222,-134.8831 38.3188,-132.5602"/>
<text text-anchor="middle" x="369.625" y="-2" font-family="Times,serif" font-size="10.00" fill="#000000">abort</text>
</g>
<!-- waiting&#45;&gt;error -->
<g id="edge25" class="edge">
<g id="edge26" class="edge">
<title>waiting&#45;&gt;error</title>
<path fill="none" stroke="#000000" d="M647.5018,-43.0636C625.9836,-25.8786 587.1915,0 548.712,0 289.8062,0 289.8062,0 289.8062,0 212.0914,0 177.1061,-109.2549 164.7343,-162.0677"/>
<polygon fill="#000000" stroke="#000000" points="161.252,-161.6033 162.4928,-172.1253 168.0844,-163.1262 161.252,-161.6033"/>
<text text-anchor="middle" x="423.3931" y="-2" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
<path fill="none" stroke="#000000" d="M656.9354,-89.0636C635.4172,-71.8786 596.6251,-46 558.1456,-46 299.2398,-46 299.2398,-46 299.2398,-46 236.9733,-46 196.4266,-120.8054 178.7539,-162.2077"/>
<polygon fill="#000000" stroke="#000000" points="175.5131,-160.8858 174.9451,-171.4654 181.9867,-163.5492 175.5131,-160.8858"/>
<text text-anchor="middle" x="432.8267" y="-48" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
</g>
<!-- waiting&#45;&gt;processing -->
<g id="edge23" class="edge">
<g id="edge24" class="edge">
<title>waiting&#45;&gt;processing</title>
<path fill="none" stroke="#000000" d="M674.3283,-75.3536C697.2037,-116.2695 754.6279,-218.9806 781.8345,-267.6433"/>
<polygon fill="#000000" stroke="#000000" points="778.9188,-269.6004 786.8538,-276.6209 785.0287,-266.1844 778.9188,-269.6004"/>
<text text-anchor="middle" x="726.4625" y="-192" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
<path fill="none" stroke="#000000" d="M686.4727,-120.5577C709.6349,-151.4538 759.571,-218.0633 787.0016,-254.653"/>
<polygon fill="#000000" stroke="#000000" points="784.3406,-256.9386 793.1395,-262.8404 789.9415,-252.7397 784.3406,-256.9386"/>
<text text-anchor="middle" x="735.8961" y="-204" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
</g>
<!-- waiting&#45;&gt;starting -->
<g id="edge24" class="edge">
<g id="edge25" class="edge">
<title>waiting&#45;&gt;starting</title>
<path fill="none" stroke="#000000" d="M635.7298,-56.4637C622.955,-56.7905 608.0343,-58.5932 595.5533,-64 585.985,-68.1451 577.0389,-75.0396 569.5816,-82.0564"/>
<polygon fill="#000000" stroke="#000000" points="566.6659,-80.0265 562.1149,-89.5942 571.639,-84.9528 566.6659,-80.0265"/>
<text text-anchor="middle" x="606.6691" y="-66" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
<path fill="none" stroke="#000000" d="M645.9643,-99.7816C633.2083,-98.8275 618.079,-99.0601 604.9869,-103 597.5398,-105.2411 590.1868,-109.0944 583.5919,-113.3378"/>
<polygon fill="#000000" stroke="#000000" points="581.3225,-110.6538 575.1313,-119.2515 585.3328,-116.3912 581.3225,-110.6538"/>
<text text-anchor="middle" x="616.1027" y="-105" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 17 KiB

42
ironic_inspector/introspect.py

@ -56,22 +56,12 @@ def introspect(node_id, token=None):
bmc_address=bmc_address,
ironic=ironic)
def _handle_exceptions(fut):
try:
fut.result()
except utils.Error as exc:
# Logging has already happened in Error.__init__
node_info.finished(error=str(exc))
except Exception as exc:
msg = _('Unexpected exception in background introspection thread')
LOG.exception(msg, node_info=node_info)
node_info.finished(error=msg)
future = utils.executor().submit(_background_introspect, ironic, node_info)
future.add_done_callback(_handle_exceptions)
def _background_introspect(ironic, node_info):
utils.executor().submit(_background_introspect, node_info, ironic)
@node_cache.release_lock
@node_cache.fsm_transition(istate.Events.wait)
def _background_introspect(node_info, ironic):
global _LAST_INTROSPECTION_TIME
LOG.debug('Attempting to acquire lock on last introspection time')
@ -85,13 +75,9 @@ def _background_introspect(ironic, node_info):
_LAST_INTROSPECTION_TIME = time.time()
node_info.acquire_lock()
try:
_background_introspect_locked(node_info, ironic)
finally:
node_info.release_lock()
_background_introspect_locked(node_info, ironic)
@node_cache.fsm_transition(istate.Events.wait)
def _background_introspect_locked(node_info, ironic):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
@ -151,17 +137,10 @@ def abort(node_id, token=None):
@node_cache.release_lock
@node_cache.fsm_transition(istate.Events.abort, reentrant=False)
@node_cache.fsm_event_before(istate.Events.abort)
def _abort(node_info, ironic):
# runs in background
if node_info.finished_at is not None:
# introspection already finished; nothing to do
LOG.info('Cannot abort introspection as it is already '
'finished', node_info=node_info)
node_info.release_lock()
return
# finish the introspection
LOG.debug('Forcing power-off', node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
@ -169,7 +148,8 @@ def _abort(node_info, ironic):
LOG.warning('Failed to power off node: %s', exc,
node_info=node_info)
node_info.finished(error=_('Canceled by operator'))
node_info.finished(istate.Events.abort_end,
error=_('Canceled by operator'))
# block this node from PXE Booting the introspection image
try:

16
ironic_inspector/introspection_state.py

@ -18,6 +18,8 @@ from automaton import machines
class States(object):
"""States of an introspection."""
# received a request to abort the introspection
aborting = 'aborting'
# received introspection data from a nonexistent node
# active - the inspector performs an operation on the node
enrolling = 'enrolling'
@ -44,7 +46,7 @@ class States(object):
def all(cls):
"""Return a list of all states."""
return [cls.starting, cls.waiting, cls.processing, cls.finished,
cls.error, cls.reapplying, cls.enrolling]
cls.error, cls.reapplying, cls.enrolling, cls.aborting]
class Events(object):
@ -52,6 +54,9 @@ class Events(object):
# cancel a waiting node introspection
# API, user
abort = 'abort'
# finish the abort request
# internal
abort_end = 'abort_end'
# mark an introspection failed
# internal
error = 'error'
@ -82,6 +87,13 @@ class Events(object):
# Error transition is allowed in any state.
State_space = [
{
'name': States.aborting,
'next_states': {
Events.abort_end: States.error,
Events.timeout: States.error,
}
},
{
'name': States.enrolling,
'next_states': {
@ -135,7 +147,7 @@ State_space = [
{
'name': States.waiting,
'next_states': {
Events.abort: States.error,
Events.abort: States.aborting,
Events.process: States.processing,
Events.start: States.starting,
Events.timeout: States.error,

43
ironic_inspector/migrations/versions/18440d0834af_introducing_the_aborting_state.py

@ -0,0 +1,43 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Introducing the aborting state
Revision ID: 18440d0834af
Revises: 882b2d84cb1b
Create Date: 2017-12-11 15:40:13.905554
"""
# revision identifiers, used by Alembic.
revision = '18440d0834af'
down_revision = '882b2d84cb1b'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy import sql
from ironic_inspector import introspection_state as istate
old_state = sa.Enum(*(set(istate.States.all()) - {istate.States.aborting}),
name='node_state')
new_state = sa.Enum(*istate.States.all(), name='node_state')
Node = sql.table('nodes', sql.column('state', old_state))
def upgrade():
with op.batch_alter_table('nodes') as batch_op:
batch_op.alter_column('state', existing_type=old_state,
type_=new_state)

21
ironic_inspector/node_cache.py

@ -211,8 +211,8 @@ class NodeInfo(object):
"""Update node_info.state based on a fsm.process_event(event) call.
An AutomatonException triggers an error event.
If strict, node_info.finished(error=str(exc)) is called with the
AutomatonException instance and a EventError raised.
If strict, node_info.finished(istate.Events.error, error=str(exc))
is called with the AutomatonException instance and a EventError raised.
:param event: an event to process by the fsm
:strict: whether to fail the introspection upon an invalid event
@ -229,8 +229,7 @@ class NodeInfo(object):
if strict:
LOG.error(msg, node_info=self)
# assuming an error event is always possible
fsm.process_event(istate.Events.error)
self.finished(error=str(exc))
self.finished(istate.Events.error, error=str(exc))
else:
LOG.warning(msg, node_info=self)
raise utils.NodeStateInvalidEvent(str(exc), node_info=self)
@ -273,19 +272,21 @@ class NodeInfo(object):
db.Option(uuid=self.uuid, name=name, value=encoded).save(
session)
def finished(self, error=None):
"""Record status for this node.
def finished(self, event, error=None):
"""Record status for this node and process a terminal transition.
Also deletes look up attributes from the cache.
:param event: the event to process
:param error: error message
"""
self.release_lock()
self.release_lock()
self.finished_at = timeutils.utcnow()
self.error = error
with db.ensure_transaction() as session:
self.fsm_event(event)
self._commit(finished_at=self.finished_at, error=self.error)
db.model_query(db.Attribute, session=session).filter_by(
node_uuid=self.uuid).delete()
@ -553,7 +554,7 @@ def triggers_fsm_error_transition(errors=(Exception,),
'func': reflection.get_callable_name(func)},
node_info=node_info)
# an error event should be possible from all states
node_info.fsm_event(istate.Events.error)
node_info.finished(istate.Events.error, error=str(exc))
return ret
return inner
return outer
@ -899,8 +900,8 @@ def clean_up():
'while introspection in "%s" state',
node_info.state,
node_info=node_info)
node_info.fsm_event(istate.Events.timeout)
node_info.finished(error='Introspection timeout')
node_info.finished(
istate.Events.timeout, error='Introspection timeout')
finally:
node_info.release_lock()

19
ironic_inspector/process.py

@ -205,7 +205,7 @@ def process(introspection_data):
msg = _('The following failures happened during running '
'pre-processing hooks:\n%s') % '\n'.join(failures)
if node_info is not None:
node_info.finished(error='\n'.join(failures))
node_info.finished(istate.Events.error, error='\n'.join(failures))
_store_logs(introspection_data, node_info)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
@ -228,13 +228,13 @@ def process(introspection_data):
node = node_info.node()
except ir_utils.NotFound as exc:
with excutils.save_and_reraise_exception():
node_info.finished(error=str(exc))
node_info.finished(istate.Events.error, error=str(exc))
_store_logs(introspection_data, node_info)
try:
result = _process_node(node_info, node, introspection_data)
except utils.Error as exc:
node_info.finished(error=str(exc))
node_info.finished(istate.Events.error, error=str(exc))
with excutils.save_and_reraise_exception():
_store_logs(introspection_data, node_info)
except Exception as exc:
@ -242,7 +242,7 @@ def process(introspection_data):
msg = _('Unexpected exception %(exc_class)s during processing: '
'%(error)s') % {'exc_class': exc.__class__.__name__,
'error': exc}
node_info.finished(error=msg)
node_info.finished(istate.Events.error, error=msg)
_store_logs(introspection_data, node_info)
raise utils.Error(msg, node_info=node_info, data=introspection_data,
code=500)
@ -282,7 +282,7 @@ def _process_node(node_info, node, introspection_data):
return resp
@node_cache.fsm_transition(istate.Events.finish)
@node_cache.triggers_fsm_error_transition()
def _finish(node_info, ironic, introspection_data, power_off=True):
if power_off:
LOG.debug('Forcing power off of node %s', node_info.uuid)
@ -299,13 +299,12 @@ def _finish(node_info, ironic, introspection_data, power_off=True):
'its power management configuration: '
'%(exc)s') % {'node': node_info.uuid, 'exc':
exc})
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info,
data=introspection_data)
LOG.info('Node powered-off', node_info=node_info,
data=introspection_data)
node_info.finished()
node_info.finished(istate.Events.finish)
LOG.info('Introspection finished successfully',
node_info=node_info, data=introspection_data)
@ -348,7 +347,7 @@ def _reapply(node_info):
msg = (_('Unexpected exception %(exc_class)s while fetching '
'unprocessed introspection data from Swift: %(error)s') %
{'exc_class': exc.__class__.__name__, 'error': exc})
node_info.finished(error=msg)
node_info.finished(istate.Events.error, error=msg)
return
try:
@ -357,14 +356,12 @@ def _reapply(node_info):
msg = _('Encountered an exception while getting the Ironic client: '
'%s') % exc
LOG.error(msg, node_info=node_info, data=introspection_data)
node_info.fsm_event(istate.Events.error)
node_info.finished(error=msg)
node_info.finished(istate.Events.error, error=msg)
return
try:
_reapply_with_data(node_info, introspection_data)
except Exception as exc:
node_info.finished(error=str(exc))
return
_finish(node_info, ironic, introspection_data,

42
ironic_inspector/test/unit/test_introspect.py

@ -21,6 +21,7 @@ from oslo_config import cfg
from ironic_inspector.common import ironic as ir_utils
from ironic_inspector import introspect
from ironic_inspector import introspection_state as istate
from ironic_inspector import node_cache
from ironic_inspector.pxe_filter import base as pxe_filter
from ironic_inspector.test import base as test_base
@ -135,7 +136,7 @@ class TestIntrospect(BaseTest):
cli.node.set_power_state.assert_called_once_with(self.uuid,
'reboot')
start_mock.return_value.finished.assert_called_once_with(
error=mock.ANY)
introspect.istate.Events.error, error=mock.ANY)
self.node_info.acquire_lock.assert_called_once_with()
self.node_info.release_lock.assert_called_once_with()
@ -153,7 +154,7 @@ class TestIntrospect(BaseTest):
ironic=cli)
self.assertFalse(cli.node.set_boot_device.called)
start_mock.return_value.finished.assert_called_once_with(
error=mock.ANY)
introspect.istate.Events.error, error=mock.ANY)
self.node_info.acquire_lock.assert_called_once_with()
self.node_info.release_lock.assert_called_once_with()
@ -186,7 +187,8 @@ class TestIntrospect(BaseTest):
introspect.introspect(self.uuid)
self.node_info.ports.assert_called_once_with()
self.node_info.finished.assert_called_once_with(error=mock.ANY)
self.node_info.finished.assert_called_once_with(
introspect.istate.Events.error, error=mock.ANY)
self.assertEqual(0, self.sync_filter_mock.call_count)
self.assertEqual(0, cli.node.set_power_state.call_count)
self.node_info.acquire_lock.assert_called_once_with()
@ -311,6 +313,10 @@ class TestAbort(BaseTest):
super(TestAbort, self).setUp()
self.node_info.started_at = None
self.node_info.finished_at = None
# NOTE(milan): node_info.finished() is a mock; no fsm_event call, then
self.fsm_calls = [
mock.call(istate.Events.abort, strict=False),
]
def test_ok(self, client_mock, get_mock):
cli = self._prepare(client_mock)
@ -326,8 +332,9 @@ class TestAbort(BaseTest):
self.node_info.acquire_lock.assert_called_once_with(blocking=False)
self.sync_filter_mock.assert_called_once_with(cli)
cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
self.node_info.finished.assert_called_once_with(error='Canceled '
'by operator')
self.node_info.finished.assert_called_once_with(
introspect.istate.Events.abort_end, error='Canceled by operator')
self.node_info.fsm_event.assert_has_calls(self.fsm_calls)
def test_node_not_found(self, client_mock, get_mock):
cli = self._prepare(client_mock)
@ -340,6 +347,7 @@ class TestAbort(BaseTest):
self.assertEqual(0, self.sync_filter_mock.call_count)
self.assertEqual(0, cli.node.set_power_state.call_count)
self.assertEqual(0, self.node_info.finished.call_count)
self.assertEqual(0, self.node_info.fsm_event.call_count)
def test_node_locked(self, client_mock, get_mock):
cli = self._prepare(client_mock)
@ -353,19 +361,7 @@ class TestAbort(BaseTest):
self.assertEqual(0, self.sync_filter_mock.call_count)
self.assertEqual(0, cli.node.set_power_state.call_count)
self.assertEqual(0, self.node_info.finshed.call_count)
def test_introspection_already_finished(self, client_mock, get_mock):
cli = self._prepare(client_mock)
get_mock.return_value = self.node_info
self.node_info.acquire_lock.return_value = True
self.node_info.started_at = time.time()
self.node_info.finished_at = time.time()
introspect.abort(self.uuid)
self.assertEqual(0, self.sync_filter_mock.call_count)
self.assertEqual(0, cli.node.set_power_state.call_count)
self.assertEqual(0, self.node_info.finshed.call_count)
self.assertEqual(0, self.node_info.fsm_event.call_count)
def test_firewall_update_exception(self, client_mock, get_mock):
cli = self._prepare(client_mock)
@ -382,8 +378,9 @@ class TestAbort(BaseTest):
self.node_info.acquire_lock.assert_called_once_with(blocking=False)
self.sync_filter_mock.assert_called_once_with(cli)
cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
self.node_info.finished.assert_called_once_with(error='Canceled '
'by operator')
self.node_info.finished.assert_called_once_with(
introspect.istate.Events.abort_end, error='Canceled by operator')
self.node_info.fsm_event.assert_has_calls(self.fsm_calls)
def test_node_power_off_exception(self, client_mock, get_mock):
cli = self._prepare(client_mock)
@ -400,5 +397,6 @@ class TestAbort(BaseTest):
self.node_info.acquire_lock.assert_called_once_with(blocking=False)
self.sync_filter_mock.assert_called_once_with(cli)
cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
self.node_info.finished.assert_called_once_with(error='Canceled '
'by operator')
self.node_info.finished.assert_called_once_with(
introspect.istate.Events.abort_end, error='Canceled by operator')
self.node_info.fsm_event.assert_has_calls(self.fsm_calls)

6
ironic_inspector/test/unit/test_node_cache.py

@ -497,7 +497,7 @@ class TestNodeInfoFinished(test_base.NodeTest):
session)
def test_success(self):
self.node_info.finished()
self.node_info.finished(istate.Events.finish)
session = db.get_writer_session()
with session.begin():
@ -511,7 +511,7 @@ class TestNodeInfoFinished(test_base.NodeTest):
session=session).all())
def test_error(self):
self.node_info.finished(error='boom')
self.node_info.finished(istate.Events.error, error='boom')
self.assertEqual((datetime.datetime(1, 1, 1), 'boom'),
tuple(db.model_query(db.Node.finished_at,
@ -521,7 +521,7 @@ class TestNodeInfoFinished(test_base.NodeTest):
def test_release_lock(self):
self.node_info.acquire_lock()
self.node_info.finished()
self.node_info.finished(istate.Events.finish)
self.assertFalse(self.node_info._locked)

53
ironic_inspector/test/unit/test_process.py

@ -140,7 +140,8 @@ class TestProcess(BaseProcessTest):
process.process, self.data)
self.cli.node.get.assert_called_once_with(self.uuid)
self.assertFalse(self.process_mock.called)
self.node_info.finished.assert_called_once_with(error=mock.ANY)
self.node_info.finished.assert_called_once_with(
istate.Events.error, error=mock.ANY)
def test_already_finished(self):
self.node_info.finished_at = timeutils.utcnow()
@ -155,7 +156,8 @@ class TestProcess(BaseProcessTest):
self.assertRaisesRegex(utils.Error, 'boom',
process.process, self.data)
self.node_info.finished.assert_called_once_with(error='boom')
self.node_info.finished.assert_called_once_with(
istate.Events.error, error='boom')
def test_unexpected_exception(self):
self.process_mock.side_effect = RuntimeError('boom')
@ -166,6 +168,7 @@ class TestProcess(BaseProcessTest):
self.assertEqual(500, ctx.exception.http_code)
self.node_info.finished.assert_called_once_with(
istate.Events.error,
error='Unexpected exception RuntimeError during processing: boom')
def test_hook_unexpected_exceptions(self):
@ -179,7 +182,7 @@ class TestProcess(BaseProcessTest):
process.process, self.data)
self.node_info.finished.assert_called_once_with(
error=mock.ANY)
istate.Events.error, error=mock.ANY)
error_message = self.node_info.finished.call_args[1]['error']
self.assertIn('RuntimeError', error_message)
self.assertIn('boom', error_message)
@ -422,7 +425,7 @@ class TestProcessNode(BaseTest):
self.assertFalse(self.cli.node.validate.called)
post_hook_mock.assert_called_once_with(self.data, self.node_info)
finished_mock.assert_called_once_with(mock.ANY)
finished_mock.assert_called_once_with(mock.ANY, istate.Events.finish)
def test_port_failed(self):
self.cli.port.create.side_effect = (
@ -445,9 +448,9 @@ class TestProcessNode(BaseTest):
self.cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
finished_mock.assert_called_once_with(
mock.ANY,
mock.ANY, istate.Events.error,
error='Failed to power off node %s, check its power '
'management configuration: boom' % self.uuid
'management configuration: boom' % self.uuid
)
@mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
@ -460,7 +463,8 @@ class TestProcessNode(BaseTest):
self.assertTrue(post_hook_mock.called)
self.assertTrue(self.cli.node.set_power_state.called)
finished_mock.assert_called_once_with(self.node_info)
finished_mock.assert_called_once_with(
self.node_info, istate.Events.finish)
@mock.patch.object(node_cache.NodeInfo, 'finished', autospec=True)
def test_no_power_off(self, finished_mock):
@ -468,7 +472,8 @@ class TestProcessNode(BaseTest):
process._process_node(self.node_info, self.node, self.data)
self.assertFalse(self.cli.node.set_power_state.called)
finished_mock.assert_called_once_with(self.node_info)
finished_mock.assert_called_once_with(
self.node_info, istate.Events.finish)
@mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
def test_store_data(self, swift_mock):
@ -524,6 +529,7 @@ class TestReapply(BaseTest):
pop_mock.return_value = node_cache.NodeInfo(
uuid=self.node.uuid,
started_at=self.started_at)
pop_mock.return_value.finished = mock.Mock()
pop_mock.return_value.acquire_lock = mock.Mock()
return func(self, pop_mock, *args, **kw)
@ -604,8 +610,7 @@ class TestReapplyNode(BaseTest):
return wrapper
@prepare_mocks
def test_ok(self, finished_mock, swift_mock, apply_mock,
post_hook_mock):
def test_ok(self, finished_mock, swift_mock, apply_mock, post_hook_mock):
swift_name = 'inspector_data-%s' % self.uuid
swift_mock.get_object.return_value = json.dumps(self.data)
@ -623,7 +628,8 @@ class TestReapplyNode(BaseTest):
# assert no power operations were performed
self.assertFalse(self.cli.node.set_power_state.called)
finished_mock.assert_called_once_with(self.node_info)
finished_mock.assert_called_once_with(
self.node_info, istate.Events.finish)
# asserting validate_interfaces was called
self.assertEqual(self.pxe_interfaces, swifted_data['interfaces'])
@ -639,9 +645,8 @@ class TestReapplyNode(BaseTest):
)
@prepare_mocks
def test_get_incomming_data_exception(self, finished_mock,
swift_mock, apply_mock,
post_hook_mock):
def test_get_incomming_data_exception(self, finished_mock, swift_mock,
apply_mock, post_hook_mock):
exc = Exception('Oops')
expected_error = ('Unexpected exception Exception while fetching '
'unprocessed introspection data from Swift: Oops')
@ -652,12 +657,12 @@ class TestReapplyNode(BaseTest):
self.assertFalse(swift_mock.create_object.called)
self.assertFalse(apply_mock.called)
self.assertFalse(post_hook_mock.called)
finished_mock.assert_called_once_with(self.node_info,
expected_error)
finished_mock.assert_called_once_with(
self.node_info, istate.Events.error, error=expected_error)
@prepare_mocks
def test_prehook_failure(self, finished_mock, swift_mock,
apply_mock, post_hook_mock):
def test_prehook_failure(self, finished_mock, swift_mock, apply_mock,
post_hook_mock):
CONF.set_override('processing_hooks', 'example',
'processing')
plugins_base._HOOKS_MGR = None
@ -676,23 +681,23 @@ class TestReapplyNode(BaseTest):
'preprocessing in hook example: %(error)s' %
{'exc_class': type(exc).__name__, 'error':
exc})
finished_mock.assert_called_once_with(self.node_info,
error=exc_failure)
finished_mock.assert_called_once_with(
self.node_info, istate.Events.error, error=exc_failure)
# assert _reapply ended having detected the failure
self.assertFalse(swift_mock.create_object.called)
self.assertFalse(apply_mock.called)
self.assertFalse(post_hook_mock.called)
@prepare_mocks
def test_generic_exception_creating_ports(self, finished_mock,
swift_mock, apply_mock,
post_hook_mock):
def test_generic_exception_creating_ports(self, finished_mock, swift_mock,
apply_mock, post_hook_mock):
swift_mock.get_object.return_value = json.dumps(self.data)
exc = Exception('Oops')
self.cli.port.create.side_effect = exc
self.call()
finished_mock.assert_called_once_with(self.node_info, error=str(exc))
finished_mock.assert_called_once_with(
self.node_info, istate.Events.error, error=str(exc))
self.assertFalse(swift_mock.create_object.called)
self.assertFalse(apply_mock.called)
self.assertFalse(post_hook_mock.called)

12
releasenotes/notes/db-status-consistency-enhancements-f97fbaccfc81a60b.yaml

@ -0,0 +1,12 @@
---
upgrade:
- |
A new state ``aborting`` was introduced to distinguish between the node
introspection abort precondition (being able to perform the state
transition from the ``waiting`` state) from the activities necessary to
abort an ongoing node introspection (power-off, set finished timestamp
etc.)
fixes:
- |
The ``node_info.finished(<transition>, error=<error>)`` now updates node
state together with other status attributes in a single DB transaction.
Loading…
Cancel
Save