Skip to content

Commit 76b68d2

Browse files
committed
add patch to wait for spot instance init before runs
1 parent 2813e2d commit 76b68d2

1 file changed

Lines changed: 79 additions & 0 deletions

File tree

.github/scripts/firesim-manager-entrypoint.sh

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,85 @@ print(
147147
)
148148
PY
149149

150+
python - <<'PY'
151+
from pathlib import Path
152+
153+
path = Path("deploy/buildtools/buildfarm.py")
154+
text = path.read_text()
155+
156+
if "import subprocess\n" not in text:
157+
text = text.replace("import os\n", "import os\nimport subprocess\nimport time\n", 1)
158+
159+
old = """ wait_on_instance_launches([build_host.launched_instance_object])
160+
build_host.ip_address = build_host.launched_instance_object.private_ip_address
161+
"""
162+
new = """ wait_on_instance_launches([build_host.launched_instance_object])
163+
build_host.launched_instance_object.reload()
164+
build_host.ip_address = build_host.launched_instance_object.private_ip_address
165+
166+
ready_timeout_seconds = int(os.environ.get("FIRESIM_BUILD_HOST_SSH_READY_TIMEOUT_SECONDS", "600"))
167+
deadline = time.time() + ready_timeout_seconds
168+
ssh_target = f"ubuntu@{build_host.ip_address}"
169+
readiness_cmd = (
170+
"cloud-init status --wait >/dev/null 2>&1 || true; "
171+
f"/bin/bash -l -c 'mkdir -p {build_host.dest_build_dir}/platforms/f2/'"
172+
)
173+
ssh_cmd = [
174+
"ssh",
175+
"-i",
176+
os.path.expanduser("~/firesim.pem"),
177+
"-o",
178+
"BatchMode=yes",
179+
"-o",
180+
"StrictHostKeyChecking=no",
181+
"-o",
182+
"UserKnownHostsFile=/dev/null",
183+
"-o",
184+
"ConnectTimeout=10",
185+
ssh_target,
186+
readiness_cmd,
187+
]
188+
189+
last_stdout = ""
190+
last_stderr = ""
191+
last_returncode = None
192+
while time.time() < deadline:
193+
completed = subprocess.run(
194+
ssh_cmd,
195+
stdout=subprocess.PIPE,
196+
stderr=subprocess.PIPE,
197+
text=True,
198+
)
199+
if completed.returncode == 0:
200+
rootLogger.info(f"Build host {build_host.ip_address} is SSH-ready.")
201+
return
202+
203+
last_stdout = completed.stdout
204+
last_stderr = completed.stderr
205+
last_returncode = completed.returncode
206+
rootLogger.info(
207+
f"Waiting for build host {build_host.ip_address} SSH readiness; "
208+
f"rc={completed.returncode}"
209+
)
210+
time.sleep(10)
211+
212+
rootLogger.critical(
213+
f"Timed out after {ready_timeout_seconds}s waiting for build host "
214+
f"{build_host.ip_address} SSH readiness. Last rc={last_returncode}. "
215+
f"stdout={last_stdout!r} stderr={last_stderr!r}"
216+
)
217+
raise RuntimeError(f"Build host {build_host.ip_address} did not become SSH-ready")
218+
"""
219+
220+
if old in text:
221+
text = text.replace(old, new, 1)
222+
elif "FIRESIM_BUILD_HOST_SSH_READY_TIMEOUT_SECONDS" not in text:
223+
raise SystemExit("Could not find expected FireSim build-host wait block")
224+
225+
path.write_text(text)
226+
print("Patched FireSim build-farm wait to require SSH/login-shell readiness.")
227+
PY
228+
150229
had_nounset=0
151230
case "$-" in
152231
*u*)

0 commit comments

Comments
 (0)