diff --git a/configs/installer-init.sh b/configs/installer-init.sh index 77d57a9..257d1d8 100644 --- a/configs/installer-init.sh +++ b/configs/installer-init.sh @@ -19,6 +19,24 @@ fail() { log "FATAL: $*"; sleep 3600; reboot -f; } report() { wget -q -O /dev/null "$HTTP/events/$1" 2>/dev/null; } log "planck installer starting" + +# test mode (kexec): verify kernel+initramfs+network without touching disks +if grep -q planck_test /proc/cmdline 2>/dev/null; then + SERIAL=$(awk '/Serial/ {print $3}' /proc/cpuinfo) + HOST=$(awk -v s="$SERIAL" 'tolower($1)==tolower(s) {print $2}' /conf/hostmap) + log "TEST MODE on $HOST" + report "$HOST-test-start" + ifconfig eth0 up + udhcpc -i eth0 -n -q -s /etc/udhcpc.script >/dev/null 2>&1 && log "dhcp ok" || log "dhcp FAIL" + IP=$(ip -4 -o addr show eth0 | awk '{print $4}' | cut -d/ -f1) + log "ip $IP" + report "$HOST-test-dhcp-$IP" + wget -q -O /tmp/test.bin "$HTTP/planck-http-server.py" && log "http get ok" || log "http FAIL" + report "$HOST-test-http" + sleep 5 + poweroff -f +fi + report "started" # --- identify this node ------------------------------------------------- diff --git a/configs/planck-tftp-server.py b/configs/planck-tftp-server.py index 2db6d91..032aa73 100644 --- a/configs/planck-tftp-server.py +++ b/configs/planck-tftp-server.py @@ -53,68 +53,44 @@ def send_err(sock, addr, code, msg): def handle_client(sock, data, addr, filepath, opts): + print(f"planck-tftp: opts={opts} file={filepath} size={os.path.getsize(filepath)}", flush=True) blksize = DEFAULT_BLKSIZE - window = DEFAULT_WINDOW - requested = {} if "blksize" in opts: try: blksize = min(BLKSIZE_MAX, max(512, int(opts["blksize"]))) - requested["blksize"] = str(blksize) except ValueError: - pass - if "windowsize" in opts: - try: - window = min(64, max(1, int(opts["windowsize"]))) - requested["windowsize"] = str(window) - except ValueError: - pass - - size = os.path.getsize(filepath) - if requested: - payload = b"\x00".join( - k.encode() + b"\x00" + v.encode() for k, v in requested.items() - ) - sock.sendto(struct.pack("!H", OP_OACK) + payload + b"\x00", addr) - else: - blksize = 512 - window = 1 + blksize = 512 + payload = ( + b"blksize\x00" + str(blksize).encode() + b"\x00" + + b"tsize\x00" + str(os.path.getsize(filepath)).encode() + b"\x00" + ) + sock.sendto(struct.pack("!H", OP_OACK) + payload, addr) with open(filepath, "rb") as f: block = 0 - eof = False - while not eof: - # send one window - for _ in range(window): - block = (block % 65536) + 1 - chunk = f.read(blksize) - pkt = struct.pack("!HH", OP_DATA, block) + chunk + while True: + chunk = f.read(blksize) + block = (block % 65536) + 1 + pkt = struct.pack("!HH", OP_DATA, block) + chunk + for attempt in range(RETRIES + 1): sock.sendto(pkt, addr) - if len(chunk) < blksize: - eof = True - # wait for the ACK of the last block of this window - expect = block - retries = 0 - while True: sock.settimeout(TIMEOUT) try: rdata, raddr = sock.recvfrom(4 + blksize) except socket.timeout: - retries += 1 - if retries > RETRIES: - return - # retransmit window - f.seek((expect - window) * blksize if expect >= window else 0) - block = expect - window if expect >= window else 0 - eof = os.path.getsize(filepath) <= f.tell() + window * blksize - break + continue if len(rdata) >= 4 and struct.unpack("!H", rdata[:2])[0] == OP_ACK: acked = struct.unpack("!H", rdata[2:4])[0] - if acked == expect: + if acked == block: break - elif acked < expect: - continue # stale ack - else: - continue # shouldn't happen + if acked == 0 and block == 1: + break # ack of the OACK, counts as ack of pending state + else: + print(f"planck-tftp: TRANSFER STALLED at block {block} {filepath} {addr}", flush=True) + return # retries exhausted + if len(chunk) < blksize: + print(f"planck-tftp: TRANSFER DONE {filepath} {addr}", flush=True) + return def main(): diff --git a/docs/PLAN.md b/docs/PLAN.md index a536064..4968599 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -58,6 +58,54 @@ Fully unattended network reimage, no per-node physical access: - UniFi MFA blocks API access; DHCP reservations deferred — mDNS (`planck0NN.local`) works across subnets today. +## Current debug state (paused here, swap to stronger model) + +First live test on planck020 (EEPROM install-mode staged, works — confirmed +via `vcgencmd bootloader_config` showing BOOT_ORDER=0xf142, TFTP_IP, +TFTP_PREFIX=1): + +- **TFTP chain works**: bootloader + firmware fetch firmware, config.txt, + cmdline.txt, kernel8.img, and the initramfs (`initramfs8` via + `auto_initramfs=1` — the explicit `initramfs followkernel` syntax + did NOT work on this bootloader; use auto_initramfs + file named + `initramfs8`). All transfers show DONE on the Synology TFTP server + (`journalctl -u planck-tftp`). "STALLED" log lines are harmless — they're + duplicate/speculative client sessions that never get ACKed. +- **Custom TFTP server gotcha**: firmware requires `tsize` in the OACK — + without it transfers stall. Fixed in + `configs/planck-tftp-server.py` (deployed version on Synology is current). +- **Kernel boots** (node pings ~25s after reboot with a DHCP-assigned IP — + init reached DHCP), but **no HTTP requests ever arrive** at the + planck-netboot helper (no `/events/` GETs, no template GET, journal + always empty). Then the node reboots into the OLD OS (~2-3 min cycle). + So initramfs `/init` is either crashing before/inside wget, or HTTP egress + from the initramfs fails silently, and the test-mode `poweroff -f` acts + like a reboot (node returns on old OS — actually likely USB fallback after + a failed netboot pass). +- planck020 EEPROM is still in install-mode; every reboot retries netboot + then falls back to old OS. To restore it: stage revert via + `rpi-eeprom-config --config revert.boot.conf` flow from the old OS (see + EEPROM configs in git history / regenerate from + `/lib/firmware/raspberrypi/bootloader/stable/pieeprom-2023-01-11.bin`). + +### Next steps for whoever continues + +1. Get console visibility on the netbooted node: add per-log-line HTTP + reporting in `configs/installer-init.sh` (each `log()` also does + `wget -q -O /dev/null "$HTTP/events/-"` once DHCP is up), + plus flush a tmpfs log file after DHCP. Alternatively boot with + `console=serial0,115200` and attach a USB-serial cable to planck020. +2. Suspects for the silent init death: busybox `wget` HTTP behavior in + initramfs (try `wget -O-` verbose, test with numeric IP + port 8000 — + verified reachable from other nodes), or udhcpc script path issues. + Consider testing initramfs content standalone (gunzip + cpio listing). +3. Template tarball + all services are already on the Synology + (`/volume1/plancknetboot/`): rootfs.tar (sha256 e37837bee03ae2b3...), + tftp/ (firmware, initramfs8, auto_initramfs config), HTTP helper :8000, + TFTP :69. Credentials for Synology SSH are session-only (not stored). +4. Once one node completes end-to-end, the rest is a loop: stage + install-mode EEPROM per node + reboot (script this in `scripts/`). + ## Phases - [x] Phase 0 — Access + inventory (SSH keys on all 20, docs/)