Skip to content

Commit 1392647

Browse files
authored
Merge branch 'vmware:master' into fix/interactive-iso-install-nonetype
2 parents e2aa994 + f614b59 commit 1392647

37 files changed

Lines changed: 927 additions & 411 deletions

.github/workflows/photon-os-installer.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,13 @@ jobs:
4242
pip install -r requirements.txt # Ensure this file lists the necessary dependencies
4343
4444
- name: install pytest
45-
run: pip install pytest
45+
run: pip install pytest pexpect
4646

4747
- name: Run Pytest
4848
run: |
4949
pytest -x tests/poi-container-test.py
5050
51+
- name: Run interactive installer dialog tests
52+
run: |
53+
pytest -x tests/test_iso_dialogs.py
54+

.github/workflows/poi-vcf.yml

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,29 @@ jobs:
2626
rm -r "$VENV_DIR"
2727
exit $STATUS
2828
29+
dialog-tests:
30+
runs-on: self-hosted
31+
steps:
32+
- name: Checkout code
33+
uses: actions/checkout@v3
34+
35+
- name: Run interactive installer dialog tests
36+
run: |
37+
VENV_DIR="$(mktemp -d /var/tmp/venv.XXXXXX)"
38+
# --without-pip + a --python-targeted install avoids relying on
39+
# ensurepip's bundled wheels, which aren't available for every
40+
# python3 in this runner fleet.
41+
python3 -m venv --without-pip "$VENV_DIR"
42+
# requirements.txt is needed too: the dialog code under test
43+
# (photon_installer/commandutils.py, installer.py) imports
44+
# requests/yaml/OpenSSL/jc transitively, even though the tests
45+
# themselves never touch the network/disk/packages.
46+
python3 -m pip --python "$VENV_DIR/bin/python3" install -r requirements.txt pytest pexpect --index-url https://packages.vcfd.broadcom.net/artifactory/api/pypi/pypi/simple/
47+
"$VENV_DIR/bin/pytest" -x tests/test_iso_dialogs.py
48+
STATUS=$?
49+
rm -r "$VENV_DIR"
50+
exit $STATUS
51+
2952
build-container:
3053
runs-on: [ self-hosted, "docker:root" ]
3154
steps:
@@ -251,6 +274,7 @@ jobs:
251274
fail-fast: false
252275
matrix:
253276
name: [minimal]
277+
boot_mode: [efi, bios]
254278
steps:
255279
- name: Checkout code
256280
uses: actions/checkout@master
@@ -267,12 +291,13 @@ jobs:
267291
- name: build ISO
268292
env:
269293
NAME: ${{ matrix.name }}
294+
BOOT_MODE: ${{ matrix.boot_mode }}
270295
run: |
271296
POI_IMAGE_NAME=${POI_IMAGE_NAME_BASE}:${GITHUB_SHA::7}
272297
VM_NAME_BASE=poi-iso-boot-${GITHUB_SHA::7}
273-
ISO_OVF=photon5-iso-boot-${GITHUB_SHA::7}.ovf
298+
ISO_OVF=photon5-iso-boot-${GITHUB_SHA::7}-${BOOT_MODE}.ovf
274299
KS_FILE=${NAME}_ks.yaml
275-
VM_NAME=${VM_NAME_BASE}-${NAME}
300+
VM_NAME=${VM_NAME_BASE}-${NAME}-${BOOT_MODE}
276301
# used by pytest
277302
export VC_IP=${{ vars.VC_IP }}
278303
export VC_USER=${{ vars.VC_USER }}
@@ -287,12 +312,55 @@ jobs:
287312
# create OVF that includes that ISO, and a blank hard disk
288313
# base64 encode the ks file, must have no new lines (-w0)
289314
KSDATA64=$(base64 -w0 < ${KS_FILE})
290-
docker run --rm -v$(pwd):/workdir ${POI_IMAGE_NAME} ova-compose -i iso_ova.yaml -o ${ISO_OVF} -m --param ksdata64=${KSDATA64}
315+
docker run --rm -v$(pwd):/workdir ${POI_IMAGE_NAME} ova-compose -i iso_ova.yaml -o ${ISO_OVF} -m --param ksdata64=${KSDATA64} --param boot_mode=${BOOT_MODE}
291316
sudo chown -R $(id -u -n):$(id -g -n) .
292317
popd
293318
294319
(cd poi-harness && echo "poi-harness sha is $(git rev-parse --short HEAD)")
295-
pytest ./poi-harness/ci/pytest/ -rs --deploy --ovf examples/iso/${ISO_OVF} --name ${VM_NAME} --ks_config examples/iso/${KS_FILE} --ova_config examples/iso/iso_ova.yaml --param ksdata64=${KSDATA64}
320+
pytest ./poi-harness/ci/pytest/ -rs --deploy --ovf examples/iso/${ISO_OVF} --name ${VM_NAME} --ks_config examples/iso/${KS_FILE} --ova_config examples/iso/iso_ova.yaml --param ksdata64=${KSDATA64} --param boot_mode=${BOOT_MODE}
321+
322+
update-cayman-poi-master:
323+
runs-on: self-hosted
324+
needs:
325+
- cayman_poi
326+
- ova-poi-harness
327+
- iso-poi-harness
328+
if: github.ref_name == 'master'
329+
steps:
330+
- name: Checkout Cayman POI
331+
uses: actions/checkout@v3
332+
with:
333+
repository: vcf/cayman-poi
334+
ref: vmware-master
335+
path: ./cayman-poi
336+
submodules: "true"
337+
fetch-depth: 0
338+
ssh-key: ${{ secrets.POI_CICD_SSH_KEY }}
339+
ssh-strict: "false"
340+
341+
- name: update poi/src and push to vmware-master
342+
run: |
343+
cd ./cayman-poi
344+
git config --global user.email "poi-cd@broadcom.com"
345+
git config --global user.name "POI CI/CD"
346+
347+
pushd poi/src
348+
349+
# sometimes the sha is not available yet
350+
git fetch
351+
352+
git checkout ${GITHUB_SHA::7}
353+
popd
354+
355+
git add poi/src
356+
357+
# Only commit if there are changes
358+
if git diff --cached --quiet; then
359+
echo "No changes to commit, skipping commit step"
360+
else
361+
git commit -m "update poi/src to ${GITHUB_SHA::7}"
362+
git push origin vmware-master
363+
fi
296364
297365
github-public:
298366
runs-on: self-hosted

docker/Dockerfile

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,16 @@ RUN echo '%__transaction_unshare %{nil}' >> /etc/rpm/macros.unshare && \
6262
tdnf ${TDNF_OPTIONS} -y install \
6363
zlib tar \
6464
gzip cpio \
65-
util-linux coreutils findutils gawk binutils cdrkit \
65+
util-linux coreutils findutils gawk binutils file xorriso \
6666
gptfdisk grub2 \
6767
e2fsprogs btrfs-progs xfsprogs kpartx lvm2 dosfstools \
6868
createrepo rpm jq jc \
6969
python3-PyYAML \
7070
python3-rpm \
7171
qemu-img \
7272
open-vmdk \
73+
squashfs-tools \
74+
erofs-utils \
7375
stig-hardening \
7476
mkpasswd \
7577
$([ "${TARGETARCH}" == "amd64" ] && echo grub2-pc) && \
@@ -88,8 +90,9 @@ COPY \
8890
create-azure \
8991
create-cherrypicks-repo \
9092
create-pkg \
91-
poi-pkglist yjson \
92-
/usr/bin
93+
poi-pkglist \
94+
yjson \
95+
/usr/bin/
9396

9497
VOLUME /repo /workdir
9598

docker/poi-pkglist

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,8 @@ def main():
106106
pkg_list.add("grub2-efi-image")
107107

108108
# needed for ISO
109-
pkg_list.add("photon-iso-config")
110109
if arch == "x86_64":
111-
pkg_list.add("syslinux")
110+
pkg_list.add("grub2-pc")
112111

113112
for p in pkg_list:
114113
print(p)

docs/ks_config.md

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -674,9 +674,12 @@ Used to configure the network.
674674

675675
### _"prepkgsinstall":_ (optional)
676676
- Contains list of lines to be executed as a single script on
677-
the target just after "filesystem" and "rpm" package is installed
678-
and before installing list of packages defined in _"packages"_/
679-
_"packagelist_file"_/_"packagelist_files"_.
677+
the host (not chrooted into the target) just after "filesystem" and "rpm"
678+
package is installed and before installing list of packages defined in
679+
_"packages"_/_"packagelist_file"_/_"packagelist_files"_.
680+
- The target's root filesystem is available under the path in the
681+
_$POI_ROOT_ environment variable, which the installer sets before running
682+
the script.
680683

681684
Example:
682685
```json
@@ -688,10 +691,13 @@ Used to configure the network.
688691
}
689692
```
690693
### _"prepkgsinstallscripts":_ (optional)
691-
- Contains list of scripts to execute on the target
692-
just after "filesystem" and "rpm" package is installed
693-
and before installing list of packages defined in _"packages"_/
694-
_"packagelist_file"_/_"packagelist_files"_.
694+
- Contains list of scripts to execute on the host (not chrooted into the
695+
target) just after "filesystem" and "rpm" package is installed and before
696+
installing list of packages defined in _"packages"_/_"packagelist_file"_/
697+
_"packagelist_files"_.
698+
- The target's root filesystem is available under the path in the
699+
_$POI_ROOT_ environment variable, which the installer sets before running
700+
the script.
695701
- Scripts will be looked up in _"search_path"_ list.
696702

697703
Example:
@@ -703,7 +709,8 @@ Used to configure the network.
703709

704710
### _"preinstall":_ (optional)
705711
- Contains list of lines to be executed as a single script on
706-
the target before installation starts.
712+
the host before installation starts. At this point the target disk has
713+
not been partitioned yet, so there is no target filesystem to chroot into.
707714
- if ks file defines any value($VALUE) that need to be populated dynamically
708715
during runtime then it should be determined and exported in preinstall script.
709716

@@ -719,7 +726,9 @@ Used to configure the network.
719726
}
720727
```
721728
### _"preinstallscripts":_ (optional)
722-
- Contains list of scripts to execute on the target before installation starts.
729+
- Contains list of scripts to execute on the host before installation
730+
starts. At this point the target disk has not been partitioned yet, so
731+
there is no target filesystem to chroot into.
723732
- Scripts will be looked up in _"search_path"_ list.
724733

725734
Example:
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Testing the interactive (curses) installer
2+
3+
The interactive installer's dialog flow (`photon_installer/iso_config.py`
4+
and the screens under `photon_installer/`) is covered by an automated,
5+
scripted walkthrough in `tests/test_iso_dialogs.py`. This doc explains how
6+
that works and how to extend it.
7+
8+
Out of scope here: booting the real ISO end-to-end in a VM. That's handled
9+
separately, and there's already coverage for installing from an ISO image
10+
via other tests/tooling.
11+
12+
## Why this is possible
13+
14+
`Installer.configure()` (`photon_installer/installer.py`) cleanly separates
15+
gathering answers from acting on them:
16+
17+
- The curses dialogs (driven by `IsoConfig.configure()` in
18+
`photon_installer/iso_config.py`) only build up an `install_config` dict.
19+
- `Installer.execute()` is the part that actually partitions disks, installs
20+
packages, etc., and is a separate call that these tests never make.
21+
22+
`iso_config.py` also has a `main()` ("for debugging") that runs just the
23+
dialog flow standalone via `curses.wrapper(IsoConfig.configure, ui_config)`
24+
and prints the resulting JSON:
25+
26+
```
27+
cd photon_installer
28+
python3 iso_config.py -f sample_ui_config.json
29+
```
30+
31+
`photon_installer/sample_ui_config.json` is a minimal working ui_config
32+
(`eula_file_path`/`license_display_title` set to `null`, picking up the
33+
defaults). Both keys are required — `add_ui_pages()` reads them with plain
34+
dict indexing (`ui_config['eula_file_path']`), so a `ui_config.json` missing
35+
either one raises a `KeyError` before the first screen even renders; the
36+
debug `main()` doesn't fill in defaults for them itself. This run uses your
37+
machine's *real* disks (`SelectDisk` isn't stubbed here — only the
38+
automated tests stub it) but is still safe: this path only ever calls
39+
`IsoConfig.configure()`, never `Installer.execute()`, so nothing is written
40+
to disk regardless of what you select.
41+
42+
That's the manual, human-drivable way to click through the screens for a
43+
quick sanity check. `tests/test_iso_dialogs.py` automates the same idea.
44+
45+
Only one screen touches the real system: `SelectDisk` /
46+
`CustomPartition` call `Device.refresh_devices()`
47+
(`photon_installer/device.py`), which shells out to `lsblk`. That's stubbed
48+
out for the tests (see below). `NetworkConfigure` does **not** probe real
49+
network interfaces — it only ever offers a fixed menu of DHCP/static/VLAN
50+
options — so it needs no stubbing.
51+
52+
## Driving the TUI with `pexpect`
53+
54+
`tests/tui_driver.py` wraps `pexpect`: it spawns the dialog process in a
55+
pty and lets tests `expect()` known marker text (window titles, prompts)
56+
and `send()` key sequences (arrows, tab, enter, plain text) in response —
57+
no terminal-emulation/screen-buffer library needed. Each curses screen
58+
renders distinctive, literal strings (e.g. `"Select a disk"`, `"Choose the
59+
hostname for your system"`) that survive being interleaved with
60+
ANSI/cursor-movement bytes, so sequential `expect()` calls reliably confirm
61+
which screen is up before the next batch of keys is sent.
62+
63+
Known limitation: `pexpect` has no screen model, so it can't tell which
64+
menu item currently has the highlight (that's conveyed via color/reverse
65+
video, not text). Tests work around this by scripting a fixed, known-good
66+
key sequence per screen (e.g. "press Down once, then Enter" to move off the
67+
default choice) instead of reading back highlight state.
68+
69+
`TERM` is pinned to `"linux"` in `tui_driver.py` rather than left to the
70+
environment: the `"linux"` terminfo entry maps arrow keys to the plain
71+
`ESC [ A/B/C/D` byte sequences, whereas e.g. `"xterm"`'s terminfo uses
72+
`ESC O A/B/C/D` — an easy way to get silently-ignored keystrokes if `TERM`
73+
varies by environment.
74+
75+
## Fixtures/stubs (`tests/fixtures/`)
76+
77+
`tests/fixtures/iso_config_stub_entrypoint.py` is what `pexpect` actually
78+
spawns. Before calling `IsoConfig().configure()` it patches out three real
79+
external dependencies the dialogs would otherwise pull in, so the tests
80+
have no system-package requirements and no dependence on the machine
81+
they're running on:
82+
83+
- `Device.refresh_devices` → returns a small fixed list of fake disks
84+
(default: one 10 GiB disk at `/dev/fakea`), overridable via the
85+
`POI_TEST_FAKE_DISKS` env var (JSON list of
86+
`{"model", "path", "size_bytes"}`).
87+
- `cracklib.VeryFascistCheck` → faked to accept any password. Password
88+
strength policy isn't what these tests exercise.
89+
- `CommandUtils.generate_password_hash` → faked to avoid shelling out to
90+
the real `mkpasswd` binary (from the `whois` package). The dialogs only
91+
care that *some* string ends up in `install_config['shadow_password']`.
92+
93+
`tests/fixtures/packages_options.json` is a package-options file (the
94+
format normally passed as `--options-file`/`ui_config['options_file']`)
95+
with two visible options, both including `"linux"` and `"linux-rt"` in
96+
their package lists. Two visible options keeps the package-selection screen
97+
active (a single-option file gets silently auto-skipped by
98+
`PackageSelector`), and having two non-conflicting kernel flavors present
99+
keeps the linux-kernel-selection screen active too (`LinuxSelector`
100+
auto-skips itself when fewer than two flavors are available). This makes
101+
both screens deterministically part of the flow regardless of what
102+
environment the test runs in.
103+
104+
## The tests: `tests/test_iso_dialogs.py`
105+
106+
`_run_dialogs()` walks the full screen sequence — license, disk selection,
107+
packages, network, kernel flavor, STIG, hostname, root password (x2),
108+
final confirmation — accepting the default choice at each screen unless a
109+
test overrides one step (currently only the STIG screen, via the
110+
`on_stig_screen` callback). It returns the final `install_config` as a
111+
parsed dict for assertions.
112+
113+
Two tests today:
114+
115+
- `test_happy_path_auto_partition_dhcp` — every default accepted.
116+
- `test_stig_hardening_enabled` — same flow, but presses Down+Enter on the
117+
STIG screen and asserts `ansible`/`additional_packages` show up.
118+
119+
To add another variant (e.g. static IP networking, custom partitioning),
120+
add a new test that calls `_run_dialogs()` with an override callback for
121+
the relevant screen, following the `on_stig_screen` pattern — or extend
122+
`_run_dialogs()` with another optional callback parameter if the new
123+
variant needs to diverge earlier/later in the sequence.
124+
125+
Since this only ever calls `IsoConfig.configure()` (never
126+
`Installer.execute()`), these tests touch no real disks/packages/network
127+
and run in ~1s each — a normal, fast part of CI
128+
(`.github/workflows/photon-os-installer.yml`), unlike a real install.
129+
130+
## Dependency
131+
132+
- `pexpect` (installed in CI via `pip install pytest pexpect`; not added to
133+
`requirements.txt` since it's test-only, not a runtime dependency of the
134+
installer itself).
135+
136+
## Explicitly not covered here
137+
138+
- Booting the actual ISO in QEMU/a VM and driving the installer over a
139+
serial console for a true end-to-end interactive install. This is handled
140+
separately; there's already coverage for installing from an ISO image via
141+
other tests/tooling.

examples/iso/iso_ova.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ system:
22
name: minimal
33
type: vmx-14 vmx-20 vmx-21
44
os_vmw: vmwarePhoton64Guest
5-
firmware: efi
5+
firmware: !param boot_mode=efi
66
secure_boot: false
77

88
networks:

examples/iso/minimal_ks.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ password:
44

55
hostname: minimal
66

7+
bootmode: dualboot
8+
79
disks:
810
default:
911
device: /dev/sda

0 commit comments

Comments
 (0)