Checkpoint: netboot debug state before model swap

configs/installer-init.sh

@@ -19,6 +19,24 @@ fail() { log "FATAL: $*"; sleep 3600; reboot -f; }
 report() { wget -q -O /dev/null "$HTTP/events/$1" 2>/dev/null; }
 
 log "planck installer starting"
+
+# test mode (kexec): verify kernel+initramfs+network without touching disks
+if grep -q planck_test /proc/cmdline 2>/dev/null; then
+    SERIAL=$(awk '/Serial/ {print $3}' /proc/cpuinfo)
+    HOST=$(awk -v s="$SERIAL" 'tolower($1)==tolower(s) {print $2}' /conf/hostmap)
+    log "TEST MODE on $HOST"
+    report "$HOST-test-start"
+    ifconfig eth0 up
+    udhcpc -i eth0 -n -q -s /etc/udhcpc.script >/dev/null 2>&1 && log "dhcp ok" || log "dhcp FAIL"
+    IP=$(ip -4 -o addr show eth0 | awk '{print $4}' | cut -d/ -f1)
+    log "ip $IP"
+    report "$HOST-test-dhcp-$IP"
+    wget -q -O /tmp/test.bin "$HTTP/planck-http-server.py" && log "http get ok" || log "http FAIL"
+    report "$HOST-test-http"
+    sleep 5
+    poweroff -f
+fi
+
 report "started"
 
 # --- identify this node -------------------------------------------------

configs/planck-tftp-server.py

@@ -53,68 +53,44 @@ def send_err(sock, addr, code, msg):
 
 
 def handle_client(sock, data, addr, filepath, opts):
+    print(f"planck-tftp: opts={opts} file={filepath} size={os.path.getsize(filepath)}", flush=True)
     blksize = DEFAULT_BLKSIZE
-    window = DEFAULT_WINDOW
-    requested = {}
     if "blksize" in opts:
         try:
             blksize = min(BLKSIZE_MAX, max(512, int(opts["blksize"])))
-            requested["blksize"] = str(blksize)
         except ValueError:
-            pass
+            blksize = 512
-    if "windowsize" in opts:
+    payload = (
-        try:
+        b"blksize\x00" + str(blksize).encode() + b"\x00"
-            window = min(64, max(1, int(opts["windowsize"])))
+        + b"tsize\x00" + str(os.path.getsize(filepath)).encode() + b"\x00"
-            requested["windowsize"] = str(window)
+    )
-        except ValueError:
+    sock.sendto(struct.pack("!H", OP_OACK) + payload, addr)
-            pass
-
-    size = os.path.getsize(filepath)
-    if requested:
-        payload = b"\x00".join(
-            k.encode() + b"\x00" + v.encode() for k, v in requested.items()
-        )
-        sock.sendto(struct.pack("!H", OP_OACK) + payload + b"\x00", addr)
-    else:
-        blksize = 512
-        window = 1
 
     with open(filepath, "rb") as f:
         block = 0
-        eof = False
+        while True:
-        while not eof:
+            chunk = f.read(blksize)
-            # send one window
+            block = (block % 65536) + 1
-            for _ in range(window):
+            pkt = struct.pack("!HH", OP_DATA, block) + chunk
-                block = (block % 65536) + 1
+            for attempt in range(RETRIES + 1):
-                chunk = f.read(blksize)
-                pkt = struct.pack("!HH", OP_DATA, block) + chunk
                 sock.sendto(pkt, addr)
-                if len(chunk) < blksize:
-                    eof = True
-            # wait for the ACK of the last block of this window
-            expect = block
-            retries = 0
-            while True:
                 sock.settimeout(TIMEOUT)
                 try:
                     rdata, raddr = sock.recvfrom(4 + blksize)
                 except socket.timeout:
-                    retries += 1
+                    continue
-                    if retries > RETRIES:
-                        return
-                    # retransmit window
-                    f.seek((expect - window) * blksize if expect >= window else 0)
-                    block = expect - window if expect >= window else 0
-                    eof = os.path.getsize(filepath) <= f.tell() + window * blksize
-                    break
                 if len(rdata) >= 4 and struct.unpack("!H", rdata[:2])[0] == OP_ACK:
                     acked = struct.unpack("!H", rdata[2:4])[0]
-                    if acked == expect:
+                    if acked == block:
                         break
-                    elif acked < expect:
+                    if acked == 0 and block == 1:
-                        continue  # stale ack
+                        break  # ack of the OACK, counts as ack of pending state
-                    else:
+            else:
-                        continue  # shouldn't happen
+                print(f"planck-tftp: TRANSFER STALLED at block {block} {filepath} {addr}", flush=True)
+                return  # retries exhausted
+            if len(chunk) < blksize:
+                print(f"planck-tftp: TRANSFER DONE {filepath} {addr}", flush=True)
+                return
 
 
 def main():

docs/PLAN.md

@@ -58,6 +58,54 @@ Fully unattended network reimage, no per-node physical access:
 - UniFi MFA blocks API access; DHCP reservations deferred — mDNS
   (`planck0NN.local`) works across subnets today.
 
+## Current debug state (paused here, swap to stronger model)
+
+First live test on planck020 (EEPROM install-mode staged, works — confirmed
+via `vcgencmd bootloader_config` showing BOOT_ORDER=0xf142, TFTP_IP,
+TFTP_PREFIX=1):
+
+- **TFTP chain works**: bootloader + firmware fetch firmware, config.txt,
+  cmdline.txt, kernel8.img, and the initramfs (`initramfs8` via
+  `auto_initramfs=1` — the explicit `initramfs <file> followkernel` syntax
+  did NOT work on this bootloader; use auto_initramfs + file named
+  `initramfs8`). All transfers show DONE on the Synology TFTP server
+  (`journalctl -u planck-tftp`). "STALLED" log lines are harmless — they're
+  duplicate/speculative client sessions that never get ACKed.
+- **Custom TFTP server gotcha**: firmware requires `tsize` in the OACK —
+  without it transfers stall. Fixed in
+  `configs/planck-tftp-server.py` (deployed version on Synology is current).
+- **Kernel boots** (node pings ~25s after reboot with a DHCP-assigned IP —
+  init reached DHCP), but **no HTTP requests ever arrive** at the
+  planck-netboot helper (no `/events/` GETs, no template GET, journal
+  always empty). Then the node reboots into the OLD OS (~2-3 min cycle).
+  So initramfs `/init` is either crashing before/inside wget, or HTTP egress
+  from the initramfs fails silently, and the test-mode `poweroff -f` acts
+  like a reboot (node returns on old OS — actually likely USB fallback after
+  a failed netboot pass).
+- planck020 EEPROM is still in install-mode; every reboot retries netboot
+  then falls back to old OS. To restore it: stage revert via
+  `rpi-eeprom-config --config revert.boot.conf` flow from the old OS (see
+  EEPROM configs in git history / regenerate from
+  `/lib/firmware/raspberrypi/bootloader/stable/pieeprom-2023-01-11.bin`).
+
+### Next steps for whoever continues
+
+1. Get console visibility on the netbooted node: add per-log-line HTTP
+   reporting in `configs/installer-init.sh` (each `log()` also does
+   `wget -q -O /dev/null "$HTTP/events/<serial>-<msg>"` once DHCP is up),
+   plus flush a tmpfs log file after DHCP. Alternatively boot with
+   `console=serial0,115200` and attach a USB-serial cable to planck020.
+2. Suspects for the silent init death: busybox `wget` HTTP behavior in
+   initramfs (try `wget -O-` verbose, test with numeric IP + port 8000 —
+   verified reachable from other nodes), or udhcpc script path issues.
+   Consider testing initramfs content standalone (gunzip + cpio listing).
+3. Template tarball + all services are already on the Synology
+   (`/volume1/plancknetboot/`): rootfs.tar (sha256 e37837bee03ae2b3...),
+   tftp/ (firmware, initramfs8, auto_initramfs config), HTTP helper :8000,
+   TFTP :69. Credentials for Synology SSH are session-only (not stored).
+4. Once one node completes end-to-end, the rest is a loop: stage
+   install-mode EEPROM per node + reboot (script this in `scripts/`).
+
 ## Phases
 
 - [x] Phase 0 — Access + inventory (SSH keys on all 20, docs/)