openstack/nova
Code Issues Proposed changes

Merge "live migration: Avoid volume rollback mismatches"

Browse Source
This commit is contained in:
Zuul 2025-05-16 04:49:25 +00:00 committed by Gerrit Code Review
commit cda0d82570
5 changed files with 407 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

350
doc/source/_static/images/live-migration.svg
View File

@ -1,13 +1,33 @@
<?xml version='1.0' encoding='UTF-8'?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<svg viewBox="0 0 1072 959" xmlns="http://www.w3.org/2000/svg" xmlns:inkspace="http://www.inkscape.org/namespaces/inkscape" xmlns:xlink="http://www.w3.org/1999/xlink">
<defs id="defs_block">
<filter height="1.504" id="filter_blur" inkspace:collect="always" width="1.1575" x="-0.07875" y="-0.252">
<feGaussianBlur id="feGaussianBlur3780" inkspace:collect="always" stdDeviation="4.2" />
</filter>
</defs>
<title>blockdiag</title>
<desc>seqdiag {
<?xml version="1.0" encoding="UTF-8"?>
<svg width="1072" height="1250" version="1.1" viewBox="0 0 1072 1250" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<defs id="defs_block">
<filter id="filter_blur" x="-.11021" y="-.70533" width="1.2204" height="2.4107">
<feGaussianBlur stdDeviation="4.2"/>
</filter>
<filter id="filter112" x="-.030556" y="-.51333" width="1.0849" height="2.4267" color-interpolation-filters="sRGB">
<feFlood flood-color="rgb(0,0,0)" flood-opacity=".69804" in="SourceGraphic" result="flood"/>
<feGaussianBlur in="SourceGraphic" result="blur" stdDeviation="3.000000"/>
<feOffset dx="6.000000" dy="6.000000" in="blur" result="offset"/>
<feComposite in="flood" in2="offset" operator="in" result="comp1"/>
<feComposite in="SourceGraphic" in2="comp1" result="comp2"/>
</filter>
<filter id="filter114" x="-.045833" y="-.51333" width="1.1274" height="2.4267" color-interpolation-filters="sRGB">
<feFlood flood-color="rgb(0,0,0)" flood-opacity=".69804" in="SourceGraphic" result="flood"/>
<feGaussianBlur in="SourceGraphic" result="blur" stdDeviation="3.000000"/>
<feOffset dx="6.000000" dy="6.000000" in="blur" result="offset"/>
<feComposite in="flood" in2="offset" operator="in" result="comp1"/>
<feComposite in="SourceGraphic" in2="comp1" result="comp2"/>
</filter>
<filter id="filter134" x="-.035648" y="-.51333" width="1.0991" height="2.4267" color-interpolation-filters="sRGB">
<feFlood flood-color="rgb(0,0,0)" flood-opacity=".69804" in="SourceGraphic" result="flood"/>
<feGaussianBlur in="SourceGraphic" result="blur" stdDeviation="3.000000"/>
<feOffset dx="6.000000" dy="6.000000" in="blur" result="offset"/>
<feComposite in="flood" in2="offset" operator="in" result="comp1"/>
<feComposite in="SourceGraphic" in2="comp1" result="comp2"/>
</filter>
</defs>
<title>blockdiag</title>
<desc>seqdiag {
Conductor; Source; Destination;
edge_length = 300;
span_height = 15;
@ -36,114 +56,204 @@
Source &lt;-- Destination;
Source -&gt;&gt; Destination [label = "cast", note = "rollback_live_migration_at_destination"];
}</desc>
<polygon fill="rgb(0,0,0)" points="747,84 967,84 975,92 975,99 747,99 747,84" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="217,124 407,124 415,132 415,139 217,139 217,124" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="447,289 535,289 543,297 543,304 447,304 447,289" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="747,344 955,344 963,352 963,359 747,359 747,344" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="169,494 407,494 415,502 415,509 169,509 169,494" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="747,564 1051,564 1059,572 1059,579 747,579 747,564" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="247,714 407,714 415,722 415,729 247,729 247,714" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="747,784 913,784 921,792 921,799 747,799 747,784" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<polygon fill="rgb(0,0,0)" points="747,894 991,894 999,902 999,909 747,909 747,894" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" />
<rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="67" y="21" />
<rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="367" y="21" />
<rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="667" y="21" />
<path d="M 128 55 L 128 959" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="8 4" />
<path d="M 428 55 L 428 959" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="8 4" />
<path d="M 728 55 L 728 959" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="8 4" />
<rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="64" y="15" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="54" x="128.0" y="41">Conductor</text>
<rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="364" y="15" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="36" x="428.0" y="41">Source</text>
<rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="664" y="15" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="66" x="728.0" y="41">Destination</text>
<path d="M 136 85 L 720 85" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="712,81 720,85 712,89" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="744,78 964,78 972,86 972,93 744,93 744,78" stroke="rgb(0,0,0)" />
<path d="M 964 78 L 964 86" fill="none" stroke="rgb(0,0,0)" />
<path d="M 964 86 L 972 86" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="204" x="854.0" y="91">check_can_live_migrate_destination</text>
<path d="M 436 125 L 720 125" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="444,121 436,125 444,129" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="214,118 404,118 412,126 412,133 214,133 214,118" stroke="rgb(0,0,0)" />
<path d="M 404 118 L 404 126" fill="none" stroke="rgb(0,0,0)" />
<path d="M 404 126 L 412 126" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="174" x="309.0" y="131">check_can_live_migrate_source</text>
<path d="M 436 165 L 720 165" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="4" />
<polygon fill="rgb(0,0,0)" points="712,161 720,165 712,169" stroke="rgb(0,0,0)" />
<path d="M 136 220 L 720 220" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="4" />
<polygon fill="rgb(0,0,0)" points="144,216 136,220 144,224" stroke="rgb(0,0,0)" />
<path d="M 136 290 L 420 290" fill="none" stroke="rgb(0,0,0)" />
<path d="M 412 286 L 420 290" fill="none" stroke="rgb(0,0,0)" />
<path d="M 420 290 L 412 294" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="444,283 532,283 540,291 540,298 444,298 444,283" stroke="rgb(0,0,0)" />
<path d="M 532 283 L 532 291" fill="none" stroke="rgb(0,0,0)" />
<path d="M 532 291 L 540 291" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="72" x="488.0" y="296">live_migrate</text>
<path d="M 436 345 L 720 345" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="712,341 720,345 712,349" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="744,338 952,338 960,346 960,353 744,353 744,338" stroke="rgb(0,0,0)" />
<path d="M 952 338 L 952 346" fill="none" stroke="rgb(0,0,0)" />
<path d="M 952 346 L 960 346" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="192" x="848.0" y="351">pre_live_migration (set up dest)</text>
<path d="M 436 385 L 720 385" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="4" />
<polygon fill="rgb(0,0,0)" points="444,381 436,385 444,389" stroke="rgb(0,0,0)" />
<path d="M 436 495 L 578 495" fill="none" stroke="rgb(0,0,0)" />
<path d="M 578 495 L 578 511" fill="none" stroke="rgb(0,0,0)" />
<path d="M 578 511 L 436 511" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="444,507 436,511 444,515" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="166,488 404,488 412,496 412,503 166,503 166,488" stroke="rgb(0,0,0)" />
<path d="M 404 488 L 404 496" fill="none" stroke="rgb(0,0,0)" />
<path d="M 404 496 L 412 496" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="222" x="285.0" y="501">post_live_migration (clean up source)</text>
<path d="M 436 565 L 720 565" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="712,561 720,565 712,569" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="744,558 1048,558 1056,566 1056,573 744,573 744,558" stroke="rgb(0,0,0)" />
<path d="M 1048 558 L 1048 566" fill="none" stroke="rgb(0,0,0)" />
<path d="M 1048 566 L 1056 566" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="288" x="896.0" y="571">post_live_migration_at_destination (finish dest)</text>
<path d="M 436 605 L 720 605" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="4" />
<polygon fill="rgb(0,0,0)" points="444,601 436,605 444,609" stroke="rgb(0,0,0)" />
<path d="M 436 715 L 578 715" fill="none" stroke="rgb(0,0,0)" />
<path d="M 578 715 L 578 731" fill="none" stroke="rgb(0,0,0)" />
<path d="M 578 731 L 436 731" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="444,727 436,731 444,735" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="244,708 404,708 412,716 412,723 244,723 244,708" stroke="rgb(0,0,0)" />
<path d="M 404 708 L 404 716" fill="none" stroke="rgb(0,0,0)" />
<path d="M 404 716 L 412 716" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="144" x="324.0" y="721">_rollback_live_migration</text>
<path d="M 436 785 L 720 785" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(0,0,0)" points="712,781 720,785 712,789" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="744,778 910,778 918,786 918,793 744,793 744,778" stroke="rgb(0,0,0)" />
<path d="M 910 778 L 910 786" fill="none" stroke="rgb(0,0,0)" />
<path d="M 910 786 L 918 786" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="150" x="827.0" y="791">remove_volume_connections</text>
<path d="M 436 825 L 720 825" fill="none" stroke="rgb(0,0,0)" stroke-dasharray="4" />
<polygon fill="rgb(0,0,0)" points="444,821 436,825 444,829" stroke="rgb(0,0,0)" />
<path d="M 436 895 L 720 895" fill="none" stroke="rgb(0,0,0)" />
<path d="M 712 891 L 720 895" fill="none" stroke="rgb(0,0,0)" />
<path d="M 720 895 L 712 899" fill="none" stroke="rgb(0,0,0)" />
<polygon fill="rgb(255,255,255)" points="744,888 988,888 996,896 996,903 744,903 744,888" stroke="rgb(0,0,0)" />
<path d="M 988 888 L 988 896" fill="none" stroke="rgb(0,0,0)" />
<path d="M 988 896 L 996 896" fill="none" stroke="rgb(0,0,0)" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="228" x="866.0" y="901">rollback_live_migration_at_destination</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="156.0" y="81">call</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="704.0" y="121">call</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="156.0" y="286">cast</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="456.0" y="341">call</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="456.0" y="561">call</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="456.0" y="781">call</text>
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="24" x="456.0" y="891">cast</text>
<path d="M 40 457 L 325 457" fill="none" stroke="rgb(0,0,0)" />
<path d="M 40 461 L 325 461" fill="none" stroke="rgb(0,0,0)" />
<path d="M 531 457 L 816 457" fill="none" stroke="rgb(0,0,0)" />
<path d="M 531 461 L 816 461" fill="none" stroke="rgb(0,0,0)" />
<rect fill="rgb(208,208,208)" height="19" stroke="rgb(0,0,0)" width="206" x="325" y="450" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="186" x="428.0" y="465">driver.live_migration (success)</text>
<path d="M 40 677 L 325 677" fill="none" stroke="rgb(0,0,0)" />
<path d="M 40 681 L 325 681" fill="none" stroke="rgb(0,0,0)" />
<path d="M 531 677 L 816 677" fill="none" stroke="rgb(0,0,0)" />
<path d="M 531 681 L 816 681" fill="none" stroke="rgb(0,0,0)" />
<rect fill="rgb(208,208,208)" height="19" stroke="rgb(0,0,0)" width="206" x="325" y="670" />
<text fill="rgb(0,0,0)" font-family="sans-serif" font-size="11" font-style="normal" font-weight="normal" text-anchor="middle" textLength="186" x="428.0" y="685">driver.live_migration (failure)</text>
<g stroke="#000">
<polygon points="747 99 747 84 967 84 975 92 975 99" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="217 139 217 124 407 124 415 132 415 139" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="447 304 447 289 535 289 543 297 543 304" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="747 359 747 344 955 344 963 352 963 359" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="169 509 169 494 407 494 415 502 415 509" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="747 579 747 564 1051 564 1059 572 1059 579" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="247 729 247 714 407 714 415 722 415 729" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="747 799 747 784 913 784 921 792 921 799" filter="url(#filter_blur)" opacity=".7"/>
<polygon points="747 909 747 894 991 894 999 902 999 909" filter="url(#filter_blur)" opacity=".7"/>
<rect x="67" y="21" width="128" height="40" filter="url(#filter_blur)" opacity=".7"/>
<rect x="367" y="21" width="128" height="40" filter="url(#filter_blur)" opacity=".7"/>
<rect x="667" y="21" width="128" height="40" filter="url(#filter_blur)" opacity=".7"/>
</g>
<g fill="none" stroke="#000">
<path d="m128 55v1195.3" stroke-dasharray="9.19926, 4.59963" stroke-width="1.1499"/>
<path d="m428 55v1196.4" stroke-dasharray="9.20317, 4.60158" stroke-width="1.1504"/>
<path d="m728 55v1195.3" stroke-dasharray="9.19927, 4.59963" stroke-width="1.1499"/>
</g>
<rect x="64" y="15" width="128" height="40" fill="#fff" stroke="#000"/>
<text x="128" y="41" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="54">Conductor</text>
<rect x="364" y="15" width="128" height="40" fill="#fff" stroke="#000"/>
<text x="428" y="41" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="36">Source</text>
<rect x="664" y="15" width="128" height="40" fill="#fff" stroke="#000"/>
<text x="728" y="41" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="66">Destination</text>
<g stroke="#000">
<path d="m136 85h584" fill="none"/>
<polygon points="712 81 720 85 712 89"/>
<polygon points="744 93 744 78 964 78 972 86 972 93" fill="#fff"/>
<path d="m964 78v8" fill="none"/>
<path d="m964 86h8" fill="none"/>
</g>
<text x="854" y="91" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="204">check_can_live_migrate_destination</text>
<g stroke="#000">
<path d="m436 125h284" fill="none"/>
<polygon points="444 121 436 125 444 129"/>
<polygon points="214 133 214 118 404 118 412 126 412 133" fill="#fff"/>
<path d="m404 118v8" fill="none"/>
<path d="m404 126h8" fill="none"/>
</g>
<text x="309" y="131" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="174">check_can_live_migrate_source</text>
<g stroke="#000">
<path d="m436 165h284" fill="none" stroke-dasharray="4"/>
<polygon points="712 161 720 165 712 169"/>
<path d="m136 220h584" fill="none" stroke-dasharray="4"/>
<polygon points="144 216 136 220 144 224"/>
</g>
<g fill="none" stroke="#000">
<path d="m136 290h284"/>
<path d="m412 286 8 4"/>
<path d="m420 290-8 4"/>
</g>
<g stroke="#000">
<polygon points="444 298 444 283 532 283 540 291 540 298" fill="#fff"/>
<path d="m532 283v8" fill="none"/>
<path d="m532 291h8" fill="none"/>
</g>
<text x="488" y="296" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="72">live_migrate</text>
<g stroke="#000">
<path d="m436 345h284" fill="none"/>
<polygon points="712 341 720 345 712 349"/>
<polygon points="744 353 744 338 952 338 960 346 960 353" fill="#fff"/>
<path d="m952 338v8" fill="none"/>
<path d="m952 346h8" fill="none"/>
</g>
<text x="848" y="351" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="192">pre_live_migration (set up dest)</text>
<path d="m436 385h284" fill="none" stroke="#000" stroke-dasharray="4"/>
<polygon points="444 381 436 385 444 389" stroke="#000"/>
<g fill="none" stroke="#000">
<path d="m436 495h142"/>
<path d="m578 495v16"/>
<path d="m578 511h-142"/>
</g>
<g stroke="#000">
<polygon points="444 507 436 511 444 515"/>
<polygon points="166 503 166 488 404 488 412 496 412 503" fill="#fff"/>
<path d="m404 488v8" fill="none"/>
<path d="m404 496h8" fill="none"/>
</g>
<text x="285" y="501" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="222">post_live_migration (clean up source)</text>
<g stroke="#000">
<path d="m436 565h284" fill="none"/>
<polygon points="712 561 720 565 712 569"/>
<polygon points="1056 573 744 573 744 558 1048 558 1056 566" fill="#fff"/>
<path d="m1048 558v8" fill="none"/>
<path d="m1048 566h8" fill="none"/>
</g>
<text x="896" y="571" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="288">post_live_migration_at_destination (finish dest)</text>
<path d="m436 605h284" fill="none" stroke="#000" stroke-dasharray="4"/>
<polygon points="444 601 436 605 444 609" stroke="#000"/>
<g fill="none" stroke="#000">
<path d="m436 715h142"/>
<path d="m578 715v16"/>
<path d="m578 731h-142"/>
</g>
<g stroke="#000">
<polygon points="444 727 436 731 444 735"/>
<polygon points="244 723 244 708 404 708 412 716 412 723" fill="#fff"/>
<path d="m404 708v8" fill="none"/>
<path d="m404 716h8" fill="none"/>
</g>
<text x="324" y="721" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="144">_rollback_live_migration</text>
<g stroke="#000">
<path d="m436 785h284" fill="none"/>
<polygon points="712 781 720 785 712 789"/>
<polygon points="744 793 744 778 910 778 918 786 918 793" fill="#fff"/>
<path d="m910 778v8" fill="none"/>
<path d="m910 786h8" fill="none"/>
</g>
<text x="827" y="791" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="150">remove_volume_connections</text>
<path d="m436 825h284" fill="none" stroke="#000" stroke-dasharray="4"/>
<polygon points="444 821 436 825 444 829" stroke="#000"/>
<g fill="none" stroke="#000">
<path d="m436 895h284"/>
<path d="m712 891 8 4"/>
<path d="m720 895-8 4"/>
</g>
<g stroke="#000">
<polygon points="744 903 744 888 988 888 996 896 996 903" fill="#fff"/>
<path d="m988 888v8" fill="none"/>
<path d="m988 896h8" fill="none"/>
</g>
<g fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle">
<text x="866" y="901" textLength="228">rollback_live_migration_at_destination</text>
<text x="156" y="81" textLength="24">call</text>
<text x="704" y="121" textLength="24">call</text>
<text x="156" y="286" textLength="24">cast</text>
<text x="456" y="341" textLength="24">call</text>
<text x="456" y="561" textLength="24">call</text>
<text x="456" y="781" textLength="24">call</text>
<text x="456" y="891" textLength="24">cast</text>
</g>
<g fill="none" stroke="#000">
<path d="m435.53 1202.9h284"/>
<path d="m711.53 1198.9 8 4"/>
<path d="m719.53 1202.9-8 4"/>
</g>
<g stroke="#000">
<polygon transform="translate(-.4697 307.88)" points="996 903 744 903 744 888 988 888 996 896" fill="#fff" filter="url(#filter112)"/>
<path d="m987.53 1195.9v8" fill="none"/>
<path d="m987.53 1203.9h8" fill="none"/>
</g>
<text transform="translate(-.4697 307.88)" x="866" y="901" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="228">rollback_live_migration_at_destination</text>
<text transform="translate(-.4697 307.88)" x="456" y="891" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="24">cast</text>
<g fill="none" stroke="#000">
<path d="m40 457h285"/>
<path d="m40 461h285"/>
<path d="m531 457h285"/>
<path d="m531 461h285"/>
</g>
<rect x="325" y="450" width="206" height="19" fill="#d0d0d0" stroke="#000"/>
<text x="428" y="465" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="186">driver.live_migration (success)</text>
<g fill="none" stroke="#000">
<path d="m40 677h285"/>
<path d="m40 681h285"/>
<path d="m531 677h285"/>
<path d="m531 681h285"/>
</g>
<rect x="325" y="670" width="206" height="19" fill="#d0d0d0" stroke="#000"/>
<text x="428" y="685" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="186">driver.live_migration (failure)</text>
<metadata>
<rdf:RDF>
<cc:Work rdf:about="">
<dc:title>blockdiag</dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g stroke="#000">
<path d="m36.758 987.32h607.82" fill="none" stroke-width="1.4604"/>
<path d="m36.758 991.32h607.82" fill="none" stroke-width="1.4604"/>
<rect x="644.52" y="979.25" width="164.48" height="19.104" fill="#d0d0d0" stroke-width=".89601"/>
</g>
<text transform="matrix(1.2901 0 0 1.4925 232.46 -61.454)" x="428" y="685" fill="#000000" font-family="sans-serif" font-size="7.9272px" text-anchor="middle" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" textLength="186"><tspan x="428" y="685"/></text>
<g stroke="#000">
<path d="m435.19 1022.8h284" fill="none"/>
<polygon transform="translate(-.80621 677.83)" points="720 345 712 349 712 341"/>
<path d="m435.19 1114.8h284" fill="none" stroke-dasharray="4"/>
<polygon transform="translate(-.80621 729.83)" points="444 381 436 385 444 389"/>
</g>
<text transform="translate(-.80621 677.83)" x="456" y="341" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="24">call</text>
<text x="650.71216" y="992.81818" fill="#000000" font-family="sans-serif" font-size="11px" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" xml:space="preserve"><tspan x="650.71216" y="992.81818">pre_live_migration (failure)</tspan></text>
<g stroke="#000">
<polygon transform="translate(7.6667 445.48)" points="412 723 244 723 244 708 404 708 412 716" fill="#fff" filter="url(#filter114)"/>
<path d="m411.67 1153.5v8" fill="none"/>
<path d="m411.67 1161.5h8" fill="none"/>
</g>
<text transform="translate(7.6667 445.48)" x="324" y="721" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="144">_rollback_live_migration</text>
<text x="546.15149" y="1108.5454" fill="#000000" font-family="sans-serif" font-size="11px" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" xml:space="preserve"><tspan x="546.15149" y="1108.5454">Exception</tspan></text>
<g stroke="#000">
<polygon transform="translate(-.93939 677.7)" points="952 338 960 346 960 353 744 353 744 338" fill="#fff" filter="url(#filter134)"/>
<path d="m951.06 1015.7v8" fill="none"/>
<path d="m951.06 1023.7h8" fill="none"/>
</g>
<text transform="translate(-.93939 677.7)" x="848" y="351" fill="#000000" font-family="sans-serif" font-size="11px" text-anchor="middle" textLength="192">pre_live_migration (set up dest)</text>
<rect x="769.35" y="1042.8" width="163.86" height="55.028" fill="#d0d0d0" stroke="#000"/>
<g fill="#000000" font-family="sans-serif" font-size="11px">
<text x="775.7337" y="1058.0815" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" xml:space="preserve"><tspan x="775.7337" y="1058.0815">except Exception:</tspan></text>
<text x="793.64282" y="1073.3087" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" xml:space="preserve"><tspan x="793.64282" y="1073.3087">disconnect volumes</tspan></text>
<text x="793.64282" y="1089.551" style="font-variant-caps:normal;font-variant-east-asian:normal;font-variant-ligatures:normal;font-variant-numeric:normal" xml:space="preserve"><tspan x="793.64282" y="1089.551">delete new attachments</tspan></text>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 15 KiB

97
nova/compute/manager.py
View File

@ -9316,43 +9316,56 @@ class ComputeManager(manager.Manager):
bdm.attachment_id = attach_ref['id']
bdm.save()
# Retrieve connection_info for the destination. Note that it is not
# saved back to the database yet.
block_device_info = self._get_instance_block_device_info(
context, instance, refresh_conn_info=True,
bdms=bdms)
# The driver pre_live_migration will plug vifs on the host
migrate_data = self.driver.pre_live_migration(context,
instance,
block_device_info,
network_info,
disk,
migrate_data)
LOG.debug('driver pre_live_migration data is %s', migrate_data)
# driver.pre_live_migration is what plugs vifs on the destination
# host so now we can set the wait_for_vif_plugged flag in the
# migrate_data object which the source compute will use to
# determine if it should wait for a 'network-vif-plugged' event
# from neutron before starting the actual guest transfer in the
# hypervisor
using_multiple_port_bindings = (
'vifs' in migrate_data and migrate_data.vifs)
migrate_data.wait_for_vif_plugged = (
CONF.compute.live_migration_wait_for_vif_plug and
using_multiple_port_bindings
)
# The driver pre_live_migration will plug vifs and connect volumes
# on the host
try:
migrate_data = self.driver.pre_live_migration(
context, instance, block_device_info, network_info, disk,
migrate_data)
# NOTE(tr3buchet): setup networks on destination host
self.network_api.setup_networks_on_host(context, instance,
self.host)
LOG.debug('driver pre_live_migration data is %s', migrate_data)
# driver.pre_live_migration is what plugs vifs on the
# destination host so now we can set the wait_for_vif_plugged
# flag in the migrate_data object which the source compute will
# use to determine if it should wait for a
# 'network-vif-plugged' event from neutron before starting the
# actual guest transfer in the hypervisor
using_multiple_port_bindings = (
'vifs' in migrate_data and migrate_data.vifs)
migrate_data.wait_for_vif_plugged = (
CONF.compute.live_migration_wait_for_vif_plug and
using_multiple_port_bindings
)
# NOTE(lyarwood): The above call to driver.pre_live_migration
# can result in the virt drivers attempting to stash additional
# metadata into the connection_info of the underlying bdm.
# Ensure this is saved to the database by calling .save() against
# the driver BDMs we passed down via block_device_info.
for driver_bdm in block_device_info['block_device_mapping']:
driver_bdm.save()
# NOTE(tr3buchet): setup networks on destination host
self.network_api.setup_networks_on_host(context, instance,
self.host)
# NOTE(lyarwood): The above call to driver.pre_live_migration
# can result in the virt drivers attempting to stash additional
# metadata into the connection_info of the underlying bdm.
# Ensure this is saved to the database by calling .save()
# against the driver BDMs we passed down via block_device_info.
for driver_bdm in block_device_info['block_device_mapping']:
driver_bdm.save()
except Exception:
# NOTE(melwitt): Try to disconnect any volumes which may have
# been connected during driver pre_live_migration(). By the
# time this error is received by the source host, BDM records
# in the database will refer only to the source host. Detach
# volumes while we still have connection_info about the
# destination host.
for driver_bdm in block_device_info['block_device_mapping']:
driver_bdm.driver_detach(
context, instance, self.volume_api, self.driver)
# Re-raise to perform any remaining rollback actions.
raise
except Exception:
# If we raise, migrate_data with the updated attachment ids
# will not be returned to the source host for rollback.
@ -10321,8 +10334,15 @@ class ComputeManager(manager.Manager):
# TODO(lyarwood): Turn the following into a lookup method within
# BlockDeviceMappingList.
vol_bdms = [bdm for bdm in bdms if bdm.is_volume]
self._remove_remote_volume_connections(context, dest, vol_bdms,
instance)
if not pre_live_migration:
# This will do both a driver detach and a Cinder attachment delete.
# If we are in here due to a pre_live_migration failure, BDMs have
# already been rolled back to contain info for the source, so don't
# try to remove volume connections on the destination.
# See ComputeManager.pre_live_migration() for details.
self._remove_remote_volume_connections(
context, dest, vol_bdms, instance)
self._rollback_volume_bdms(context, vol_bdms, source_bdms, instance)
self._notify_about_instance_usage(context, instance,
@ -10469,6 +10489,17 @@ class ComputeManager(manager.Manager):
# from remote volumes if necessary
block_device_info = self._get_instance_block_device_info(context,
instance)
# NOTE(melwitt): By the time we get here, the instance BDMs have
# already been rolled back to contain info for the source during
# _rollback_live_migration().
# The code above predates the addition of rollback of the instance
# BDM records to point at the source. It also predates the addition
# of a driver detach call to remove_volume_connection().
# Set the list for Cinder volumes to empty to avoid attempting to
# disconnect volumes during driver.cleanup() on the destination.
bdi_without_volumes = copy.deepcopy(block_device_info)
bdi_without_volumes['block_device_mapping'] = []
# free any instance PCI claims done on destination during
# check_can_live_migrate_destination()
self.rt.free_pci_device_claims_for_instance(context, instance)
@ -10478,7 +10509,7 @@ class ComputeManager(manager.Manager):
# specific resources like vpmem
with instance.mutated_migration_context():
self.driver.rollback_live_migration_at_destination(
context, instance, network_info, block_device_info,
context, instance, network_info, bdi_without_volumes,
destroy_disks=destroy_disks, migrate_data=migrate_data)
self._notify_about_instance_usage(

129
nova/tests/functional/regressions/test_bug_1899835.py
View File

@ -13,6 +13,7 @@
from unittest import mock
import fixtures
from oslo_serialization import jsonutils
from nova import context
from nova import objects
@ -20,8 +21,6 @@ from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.functional.libvirt import base
from oslo_serialization import jsonutils
class TestVolumeDisconnectDuringPreLiveMigrationRollback(base.ServersTestBase):
"""Regression test for bug #1899835
@ -75,13 +74,6 @@ class TestVolumeDisconnectDuringPreLiveMigrationRollback(base.ServersTestBase):
mock_src_connector.assert_called_once()
mock_src_connect.assert_called_once()
# Fetch the connection_info from the src
ctxt = context.get_admin_context()
bdm = objects.BlockDeviceMapping.get_by_volume_id(
ctxt, nova_fixtures.CinderFixture.IMAGE_BACKED_VOL,
instance_uuid=server['id'])
src_connection_info = jsonutils.loads(bdm.connection_info)
with test.nested(
mock.patch.object(
self.computes['dest'].driver, 'get_volume_connector'),
@ -106,19 +98,106 @@ class TestVolumeDisconnectDuringPreLiveMigrationRollback(base.ServersTestBase):
# Assert that connect_volume hasn't been called on the dest
mock_dest_connect.assert_not_called()
# FIXME(lyarwood): This is bug #1899835, disconnect_volume shouldn't be
# called on the destination host without connect_volume first being
# called and especially using with the connection_info from the source
self.assertEqual(2, mock_dest_disconnect.call_count)
# First call is from ComputeManager._remove_volume_connection() called
# eventually from ComputeManager._rollback_live_migration() on the
# source.
call1 = mock.call(
mock.ANY, src_connection_info, mock.ANY, encryption=mock.ANY)
# Second call is from LibvirtDriver.destroy() =>
# LibvirtDriver.cleanup() on the destination as part of
# ComputeManager.rollback_live_migration_at_destination().
call2 = mock.call(
mock.ANY, src_connection_info, mock.ANY, destroy_secrets=True,
force=True)
mock_dest_disconnect.assert_has_calls([call1, call2])
# Assert that disconnect_volume was not called on the destination host
self.assertEqual(0, mock_dest_disconnect.call_count)
def test_pre_live_migration_failed_after_new_attachment_created(self):
"""Test a scenario where live migration fails during pre_live_migration
where the BDM record attachment_id got updated with a new Cinder
attachment but the BDM record connection_info was not yet updated.
In other words, BlockDeviceMapping.attachment_id is pointing at the
destination but BlockDeviceMapping.connection_info is still pointing
at the source.
The _disconnect_volume() should run on the destination and with
destination connection_info despite the BDM database record still
containing source connection_info.
"""
server = {
'name': 'test',
'imageRef': '',
'flavorRef': 1,
'networks': 'none',
'host': 'src',
'block_device_mapping_v2': [{
'source_type': 'volume',
'destination_type': 'volume',
'boot_index': 0,
'uuid': nova_fixtures.CinderFixture.IMAGE_BACKED_VOL
}]
}
with test.nested(
mock.patch.object(
self.computes['src'].driver, 'get_volume_connector'),
mock.patch.object(
self.computes['src'].driver, '_connect_volume'),
) as (
mock_src_connector, mock_src_connect
):
server = self.api.post_server({'server': server})
server = self._wait_for_state_change(server, 'ACTIVE')
# Assert that we called the src connector and connect mocks
mock_src_connector.assert_called_once()
mock_src_connect.assert_called_once()
# Fetch the connection_info from the src
ctxt = context.get_admin_context()
bdm = objects.BlockDeviceMapping.get_by_volume_id(
ctxt, nova_fixtures.CinderFixture.IMAGE_BACKED_VOL,
instance_uuid=server['id'])
src_connection_info = jsonutils.loads(bdm.connection_info)
with test.nested(
mock.patch.object(
self.computes['dest'].driver, 'get_volume_connector'),
mock.patch.object(
self.computes['dest'].driver, '_connect_volume'),
mock.patch.object(
self.computes['dest'].driver, '_disconnect_volume'),
mock.patch.object(
self.computes['dest'].manager.network_api,
'setup_networks_on_host',
# Fail setup networks during pre_live_migration but have it
# succeed during rollback
side_effect=[test.TestingException, None], autospec=False),
) as (
mock_dest_connector, mock_dest_connect, mock_dest_disconnect,
mock_setup_networks
):
# Attempt to live migrate and ensure it is marked as error
self._live_migrate(server, 'failed')
# Assert that we called the dest connector and setup networks mocks
mock_dest_connector.assert_called_once()
mock_setup_networks.assert_called()
# Assert that connect_volume has been called on the dest
mock_dest_connect.assert_called()
# Assert that disconnect_volume was called on the destination host
# and with connection_info for the destination
mock_dest_disconnect.assert_called_once()
# def _disconnect_volume(self, context, connection_info, instance, ...)
dest_connection_info = mock_dest_disconnect.call_args.args[1]
volumes_attached = server['os-extended-volumes:volumes_attached']
self.assertEqual(1, len(volumes_attached))
# The volume_id in the destination connection_info should match the
# volume attached to the server
self.assertEqual(
volumes_attached[0]['id'],
dest_connection_info['data']['volume_id'])
# The attachment_id should not be the same as the source (new
# attachment_id)
self.assertNotEqual(
src_connection_info['data']['attachment_id'],
dest_connection_info['data']['attachment_id'])
# The connection_info in the database should still be for the source
bdm = objects.BlockDeviceMapping.get_by_volume_id(
ctxt, nova_fixtures.CinderFixture.IMAGE_BACKED_VOL,
instance_uuid=server['id'])
current_connection_info = jsonutils.loads(bdm.connection_info)
self.assertEqual(src_connection_info, current_connection_info)

3
nova/tests/unit/compute/test_compute.py
View File

@ -6826,9 +6826,6 @@ class ComputeTestCase(BaseTestCase,
final_bdm.save.assert_called()
mock_get_bdms.assert_called_with(c, instance.uuid)
mock_remove.assert_has_calls([
mock.call(c, instance, uuids.volume_id_1, dest_host),
mock.call(c, instance, uuids.volume_id_2, dest_host)])
mock_rollback.assert_called_once_with(c, instance, dest_host,
destroy_disks=True,
migrate_data=test.MatchType(

9
releasenotes/notes/pre-live-migration-rollback-7de399d9b3979f79.yaml Normal file
View File

@ -0,0 +1,9 @@
fixes:
- |
When live migration fails during pre_live_migration on the destination,
during rollback Cinder volumes will now be disconnected from the
destination locally instead of remotely over RPC from the source. This
should ensure that only connection_info for the destination will be used to
disconnect volumes from the destination.
See `bug #1899835 <https://bugs.launchpad.net/nova/+bug/1899835>`_ for more
details.