diff --git a/.dev/bin/cooldctl b/.dev/bin/cooldctl new file mode 100755 index 000000000..57a106667 Binary files /dev/null and b/.dev/bin/cooldctl differ diff --git a/dev/coold-dev.md b/dev/coold-dev.md new file mode 100644 index 000000000..249c522b5 --- /dev/null +++ b/dev/coold-dev.md @@ -0,0 +1,182 @@ +# coold Dev Environment Notes + +This file documents the current local v5/coold dev setup in Coolify. + +## Roles + +- `scripts/dev.sh` owns local developer convenience. +- Lima VMs act like real deployment servers. +- `cooldctl init bootstrap` owns host wiring: + - WireGuard + - Podman mesh networks + - Corrosion config/schema/service + - coold install/service + - builder install/config + - default-deny firewall service +- Coolify/Laravel dev owns Flux-specific wiring: + - starts Flux inside the Coolify container + - mints dev host JWTs + - installs `/etc/coolify/host-jwt` into each VM + - adds a `coold.service` systemd drop-in with `COOLIFY_COOLD_FLUX_URL` + +## Main commands + +```bash +scripts/dev.sh up +scripts/dev.sh down +scripts/dev.sh clean-vms +scripts/dev.sh list +``` + +`clean-vms` deletes the Lima VM instances and VM-local state, including disks, +containers, WireGuard keys, Corrosion DB, installed binaries, and firewall state. +It does not delete the Coolify repo. + +## cooldctl helper + +```bash +scripts/dev.sh cooldctl install +scripts/dev.sh cooldctl path +scripts/dev.sh cooldctl bootstrap-command +scripts/dev.sh cooldctl run +``` + +On macOS, the helper builds `cooldctl` from the local coold repo because the +nightly release currently publishes Linux binaries. On Linux, it downloads the +nightly release artifact. + +The generated bootstrap command uses Lima's forwarded SSH ports and dev +WireGuard endpoint overrides, for example: + +```bash +.dev/bin/cooldctl init bootstrap \ + --nodes "127.0.0.1:,127.0.0.1:" \ + --ssh-key "$HOME/.lima/_config/user" \ + --ssh-user "$USER" \ + --wg-listen-port-overrides "127.0.0.1:=51821,127.0.0.1:=51822" \ + --wg-endpoint-overrides "127.0.0.1:=host.lima.internal:51821,127.0.0.1:=host.lima.internal:51822" \ + --coold-version "nightly" \ + --corrosion-version "v1.0.0" \ + --yes +``` + +Lima does not allow direct root SSH by default, so dev uses the normal Lima user +with passwordless sudo. `cooldctl` wraps remote commands in `sudo -n bash -lc` +when the SSH user is not `root`. + +## Default dev topology + +After `cooldctl init bootstrap`, defaults are: + +| VM | WireGuard IP | WireGuard endpoint | Podman subnet | Gateway | +| --- | --- | --- | --- | --- | +| `coold-dev` | `100.64.0.1` | `host.lima.internal:51821` | `10.210.0.0/24` | `10.210.0.1` | +| `coold-dev-2` | `100.64.0.2` | `host.lima.internal:51822` | `10.210.1.0/24` | `10.210.1.1` | + +## Checking state + +```bash +scripts/dev.sh corrosion check +scripts/dev.sh corrosion containers +scripts/dev.sh corrosion config +scripts/dev.sh corrosion logs 1 +scripts/dev.sh corrosion logs 2 +``` + +`corrosion containers` shows both Corrosion `service_endpoints` and rootful / +rootless Podman containers. + +## Example nginx containers + +```bash +scripts/dev.sh example-nginx up +scripts/dev.sh example-nginx check-dns +scripts/dev.sh example-nginx down +``` + +The example containers are started with coold DNS: + +```bash +--dns +--dns-search default.coolify.internal +``` + +Expected service discovery format: + +```text +.default.coolify.internal +``` + +Example: + +```text +coolify-example-nginx-2.default.coolify.internal -> 10.210.1.x +``` + +## Firewall behavior + +The mesh firewall is default-deny for inter-container traffic. Host-to-host +WireGuard traffic can work while container-to-container traffic is blocked. + +Allow traffic: + +```bash +scripts/dev.sh firewall allow 10.210.0.2 10.210.1.2 tcp 80 +scripts/dev.sh firewall list +``` + +Revoke traffic: + +```bash +scripts/dev.sh firewall revoke +scripts/dev.sh firewall revoke +``` + +### Why dev adds allow rules to both hosts + +`business traffic` from container A on host 1 to container B on host 2 crosses +forwarding/firewall logic on both sides: + +```text +10.210.0.2 + -> source host bridge/firewall + -> coold-dev wg0 + -> WireGuard + -> coold-dev-2 wg0 + -> destination host bridge/firewall + -> 10.210.1.2 +``` + +The default-deny hooks can drop the packet on either the source or destination +host. For the two-node dev setup, `scripts/dev.sh firewall allow` writes the +same allow tuple to every coold VM so the path works reliably. + +Production should become topology-aware instead of blindly writing to every +host: + +- cross-host traffic: write allow rules to the source and destination hosts +- same-host traffic: write the allow rule only to that host +- larger clusters: do not install unrelated allow rules on unaffected hosts + +## Manual connectivity checks + +Before allow rule, this should time out: + +```bash +scripts/dev.sh shell 1 +sudo podman exec coolify-example-nginx wget -T 3 -qO- http://10.210.1.2 +``` + +After allow rule, this should return nginx HTML: + +```bash +scripts/dev.sh firewall allow 10.210.0.2 10.210.1.2 tcp 80 +scripts/dev.sh shell 1 +sudo podman exec coolify-example-nginx wget -T 5 -qO- http://10.210.1.2 | head -n 1 +``` + +Expected: + +```html + +``` diff --git a/dev/lima/coold.yaml b/dev/lima/coold.yaml index d0354b215..b1fcd3713 100644 --- a/dev/lima/coold.yaml +++ b/dev/lima/coold.yaml @@ -1,6 +1,5 @@ -# Lima VM for testing Coolify v5 against a packaged coold endpoint. +# Lima VM for testing Coolify v5 against cooldctl-provisioned hosts. # Start with: scripts/coold-vm.sh up -# Run endpoint: scripts/coold-vm.sh dev # # This file is copied into .dev/lima/coold.generated.yaml by the wrapper script, # with {{COOLIFY_REPO}}, {{COOLIFY_COOLD_VERSION}}, and {{COOLIFY_CORROSION_VERSION}} replaced before Lima sees it. @@ -41,85 +40,22 @@ provision: export DEBIAN_FRONTEND=noninteractive echo "[coold-vm] Updating Ubuntu package index..." apt-get update - echo "[coold-vm] Installing VM dependencies..." + echo "[coold-vm] Installing minimal VM dependencies..." apt-get install -y --no-install-recommends \ bash \ - buildah \ ca-certificates \ - clang \ curl \ - git \ iproute2 \ - iptables \ jq \ - libssl-dev \ - mold \ - nftables \ openssl \ - perl \ - pkg-config \ - podman \ - protobuf-compiler \ sqlite3 \ - sudo \ - wireguard-tools - echo "[coold-vm] Enabling Podman socket..." - systemctl enable --now podman.socket + sudo cat >/usr/local/bin/rtk <<'SH' #!/usr/bin/env bash exec "$@" SH chmod +x /usr/local/bin/rtk - mkdir -p /etc/coolify /var/lib/corrosion /run/corrosion /var/lib/coolify-dev /var/lib/coolify-builder/work + mkdir -p /etc/coolify /var/lib/coolify-dev chmod 755 /etc/coolify - chmod 777 /var/lib/corrosion /run/corrosion - install_coolify_binary() { - name="$1" - version="{{COOLIFY_COOLD_VERSION}}" - arch_raw="$(uname -m)" - case "$arch_raw" in - x86_64) arch=amd64 ;; - aarch64) arch=arm64 ;; - *) echo "unsupported arch: $arch_raw" >&2; exit 1 ;; - esac - - url="https://github.com/coollabsio/coold/releases/download/${version}/${name}-linux-${arch}.tar.gz" - echo "[coold-vm] Installing ${name} from ${url}..." - tmpdir="$(mktemp -d)" - trap 'rm -rf "$tmpdir"' RETURN - curl -fsSL --retry 3 --max-time 120 -o "$tmpdir/${name}.tar.gz" "$url" - tar -xzf "$tmpdir/${name}.tar.gz" -C "$tmpdir" - test -f "$tmpdir/$name" - install -m 0755 "$tmpdir/$name" "/usr/local/bin/${name}.tmp" - mv "/usr/local/bin/${name}.tmp" "/usr/local/bin/${name}" - echo "$version" > "/usr/local/bin/${name}.version" - } - - install_corrosion_binary() { - version="{{COOLIFY_CORROSION_VERSION}}" - arch_raw="$(uname -m)" - case "$arch_raw" in - x86_64) arch=x86_64-unknown-linux-gnu ;; - aarch64) arch=aarch64-unknown-linux-gnu ;; - *) echo "unsupported arch: $arch_raw" >&2; exit 1 ;; - esac - - url="https://github.com/superfly/corrosion/releases/download/${version}/corrosion-${arch}.tar.gz" - echo "[coold-vm] Installing corrosion from ${url}..." - tmpdir="$(mktemp -d)" - trap 'rm -rf "$tmpdir"' RETURN - curl -fsSL --retry 3 --max-time 180 -o "$tmpdir/corrosion.tar.gz" "$url" - tar -xzf "$tmpdir/corrosion.tar.gz" -C "$tmpdir" - test -f "$tmpdir/corrosion" - install -m 0755 "$tmpdir/corrosion" /usr/local/bin/corrosion.tmp - mv /usr/local/bin/corrosion.tmp /usr/local/bin/corrosion - echo "$version" > /usr/local/bin/corrosion.version - } - - install_coolify_binary coold - install_coolify_binary builder - install_corrosion_binary - echo "[coold-vm] Endpoint binaries installed." - - echo "[coold-vm] Provisioning complete." + echo "[coold-vm] Minimal provisioning complete. cooldctl bootstrap will install WireGuard, Podman, Corrosion, coold, and builder." diff --git a/scripts/coold-vm.sh b/scripts/coold-vm.sh index 627717181..b43f50c32 100755 --- a/scripts/coold-vm.sh +++ b/scripts/coold-vm.sh @@ -48,7 +48,7 @@ usage() { Usage: scripts/coold-vm.sh Commands: - up Create/start the Lima VM and install packaged coold endpoint binaries + up Create/start the Lima VM with minimal server prerequisites dev Start packaged coold + Corrosion inside the VM start-agent Start production-like coold.service + corrosion.service inside the VM @@ -490,13 +490,8 @@ wait_for_guest_provisioning() { fi echo "==> Final guest provisioning status:" - lima_shell bash -lc 'cloud-init status 2>/dev/null || true; if command -v coold >/dev/null; then coold --version; else echo "coold not installed yet"; fi; if command -v corrosion >/dev/null; then echo "corrosion installed"; else echo "corrosion not installed yet"; fi; if command -v builder >/dev/null; then echo "builder installed"; else echo "builder not installed yet"; fi; true' | awk '{ print "[guest] " $0; fflush(); }' || true - - if ! lima_shell bash -lc 'command -v coold >/dev/null && command -v corrosion >/dev/null && command -v builder >/dev/null' >/dev/null 2>&1; then - echo "ERROR: VM provisioning finished but coold, corrosion, or builder is missing." >&2 - echo "Check with: scripts/coold-vm.sh shell" >&2 - exit 1 - fi + lima_shell bash -lc 'cloud-init status 2>/dev/null || true; echo "minimal VM ready"; true' \ + | awk '{ print "[guest] " $0; fflush(); }' || true } up_with_logs() { @@ -508,7 +503,7 @@ up_with_logs() { wait_for_lima_start wait_for_guest_provisioning - echo "==> VM is ready. Run: scripts/coold-vm.sh dev" + echo "==> VM is ready. Run cooldctl bootstrap via: scripts/dev.sh up" } cmd="${1:-}" @@ -561,7 +556,7 @@ case "$cmd" in shell) require_lima start_vm >/dev/null - exec bash -lc "cd /tmp && limactl shell '$INSTANCE' -- env TERM=xterm-256color SYSTEMD_PAGER=cat SYSTEMD_LESS=FRXMK bash -l" + exec bash -lc "cd /tmp && limactl shell '$INSTANCE' -- sudo env TERM=xterm-256color SYSTEMD_PAGER=cat SYSTEMD_LESS=FRXMK bash -l" ;; status) require_lima diff --git a/scripts/dev.sh b/scripts/dev.sh index f361a0fac..1d389a0c3 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -52,7 +52,7 @@ coold_vm_instance() { coold_vm_wg_ip() { local index="$1" - read_coolify_env "COOLIFY_COOLD_VM_WG_IP_${index}" "100.64.0.$((9 + index))" + read_coolify_env "COOLIFY_COOLD_VM_WG_IP_${index}" "100.64.0.${index}" } coold_vm_wg_port() { @@ -70,6 +70,239 @@ coold_vm_container_gateway() { read_coolify_env "COOLIFY_COOLD_VM_CONTAINER_GATEWAY_${index}" "10.210.$((index - 1)).1" } +host_arch() { + case "$(uname -m)" in + x86_64|amd64) + printf '%s\n' amd64 + ;; + arm64|aarch64) + printf '%s\n' arm64 + ;; + *) + echo "ERROR: unsupported host architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +} + +host_os() { + case "$(uname -s)" in + Darwin) + printf '%s\n' darwin + ;; + Linux) + printf '%s\n' linux + ;; + *) + echo "ERROR: unsupported host OS: $(uname -s)" >&2 + exit 1 + ;; + esac +} + +cooldctl_bin() { + printf '%s\n' "$ROOT/.dev/bin/cooldctl" +} + +build_local_cooldctl() { + local bin + local coold_repo + + bin="$(cooldctl_bin)" + coold_repo="$(read_coolify_env COOLIFY_COOLD_REPO /Users/heyandras/devel/coold)" + + if [ ! -d "$coold_repo/cooldctl" ]; then + echo "ERROR: macOS cooldctl release artifact is not available yet and local coold repo was not found at ${coold_repo}." >&2 + echo "Set COOLIFY_COOLD_REPO=/path/to/coold or build/copy cooldctl to ${bin}." >&2 + exit 1 + fi + + echo "==> Building local macOS cooldctl from ${coold_repo}" + (cd "$coold_repo" && cargo build -p cooldctl) + mkdir -p "$(dirname "$bin")" + install -m 0755 "$coold_repo/target/debug/cooldctl" "$bin" +} + +ensure_cooldctl() { + local bin + local version + local arch + local os + local url + local tmpdir + + bin="$(cooldctl_bin)" + version="$(read_coolify_env COOLIFY_COOLDCTL_VERSION "$(read_coolify_env COOLIFY_COOLD_VERSION nightly)")" + arch="$(host_arch)" + os="$(host_os)" + + if [ "$os" = "darwin" ]; then + build_local_cooldctl + return + fi + + if [ -x "$bin" ] && "$bin" --version >/dev/null 2>&1 && [ "${COOLIFY_COOLDCTL_FORCE_DOWNLOAD:-false}" != "true" ]; then + return + fi + + url="https://github.com/coollabsio/coold/releases/download/${version}/cooldctl-${os}-${arch}.tar.gz" + echo "==> Installing cooldctl from ${url}" + + mkdir -p "$(dirname "$bin")" + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' RETURN + curl -fsSL --retry 3 --max-time 120 -o "$tmpdir/cooldctl.tar.gz" "$url" + tar -xzf "$tmpdir/cooldctl.tar.gz" -C "$tmpdir" + install -m 0755 "$tmpdir/cooldctl" "$bin" +} + +lima_ssh_target() { + local index="$1" + local instance + local ssh + + instance="$(coold_vm_instance "$index")" + ssh="$(limactl list 2>/dev/null | awk -v name="$instance" 'NR > 1 && $1 == name {print $3}')" + + if [ -z "$ssh" ] || [ "$ssh" = "-" ]; then + echo "ERROR: could not determine SSH target for Lima VM ${instance}. Start it first with scripts/dev.sh up." >&2 + exit 1 + fi + + printf '%s\n' "$ssh" +} + +cooldctl_nodes_arg() { + local count + local nodes="" + count="$(coold_vm_count)" + + for index in $(seq 1 "$count"); do + if [ -n "$nodes" ]; then + nodes="${nodes}," + fi + nodes="${nodes}$(lima_ssh_target "$index")" + done + + printf '%s\n' "$nodes" +} + +cooldctl_wg_listen_overrides_arg() { + local count + local overrides="" + local node + count="$(coold_vm_count)" + + for index in $(seq 1 "$count"); do + node="$(lima_ssh_target "$index")" + if [ -n "$overrides" ]; then + overrides="${overrides}," + fi + overrides="${overrides}${node}=$(coold_vm_wg_port "$index")" + done + + printf '%s\n' "$overrides" +} + +cooldctl_wg_endpoint_overrides_arg() { + local count + local overrides="" + local node + count="$(coold_vm_count)" + + for index in $(seq 1 "$count"); do + node="$(lima_ssh_target "$index")" + if [ -n "$overrides" ]; then + overrides="${overrides}," + fi + overrides="${overrides}${node}=host.lima.internal:$(coold_vm_wg_port "$index")" + done + + printf '%s\n' "$overrides" +} + +cooldctl_ssh_key() { + read_coolify_env COOLIFY_COOLDCTL_SSH_KEY "$HOME/.lima/_config/user" +} + +cooldctl_ssh_user() { + read_coolify_env COOLIFY_COOLDCTL_SSH_USER "$USER" +} + +cooldctl_bootstrap_command() { + ensure_cooldctl + + cat < + +Commands: + install Download/install the nightly cooldctl dev binary + path Print the local cooldctl path + bootstrap-command Print the dev Lima bootstrap command without running it + run Run cooldctl with arbitrary args + +Example: + scripts/dev.sh cooldctl bootstrap-command +USAGE + ;; + *) + echo "unknown cooldctl command: $command" >&2 + echo "Run: scripts/dev.sh cooldctl help" >&2 + exit 1 + ;; + esac +} + coold_vm() { local index="$1" shift @@ -108,29 +341,6 @@ mint_host_jwt_for_host() { return 1 } -setup_wireguard_mesh() { - local count="$1" - - if [ "$count" -lt 2 ]; then - return - fi - - echo "==> Configuring WireGuard mesh for ${count} coold VMs..." - - local pub1 pub2 ip1 ip2 port1 port2 subnet1 subnet2 - pub1="$(coold_vm 1 wg-public-key)" - pub2="$(coold_vm 2 wg-public-key)" - ip1="$(coold_vm_wg_ip 1)" - ip2="$(coold_vm_wg_ip 2)" - port1="$(coold_vm_wg_port 1)" - port2="$(coold_vm_wg_port 2)" - subnet1="$(coold_vm_container_subnet 1)" - subnet2="$(coold_vm_container_subnet 2)" - - coold_vm 1 setup-wireguard "$ip1" "$ip2" "host.lima.internal" "$pub2" "$port1" "$port2" "$subnet2" - coold_vm 2 setup-wireguard "$ip2" "$ip1" "host.lima.internal" "$pub1" "$port2" "$port1" "$subnet1" -} - follow_logs() { local coold_vm_enabled="$1" local count @@ -168,6 +378,33 @@ follow_logs() { spin logs -f } +configure_flux_dev_for_vm() { + local index="$1" + local host_id + local host_jwt + local flux_url + + host_id="$(coold_vm_wg_ip "$index")" + flux_url="$(read_coolify_env COOLIFY_COOLD_VM_FLUX_URL http://host.lima.internal:6443)" + + echo "==> Minting Flux dev host JWT for ${host_id}..." + host_jwt="$(mint_host_jwt_for_host "$host_id")" + echo "==> Installing Flux dev env/JWT into $(coold_vm_instance "$index")..." + coold_vm "$index" install-host-jwt "$host_jwt" + + COOLIFY_COOLD_LIMA_INSTANCE="$(coold_vm_instance "$index")" scripts/coold-vm.sh shell </dev/null < Starting ${count} Coolify coold VM(s) before Spin..." + vm_pids="" for index in $(seq 1 "$count"); do - coold_vm "$index" up + ( + coold_vm "$index" up + ) & + vm_pids="${vm_pids} $!" + done + for pid in $vm_pids; do + wait "$pid" done - setup_wireguard_mesh "$count" else echo "==> COOLIFY_COOLD_VM_ENABLED=false; skipping coold VM." fi @@ -190,25 +433,11 @@ up() { spin up -d "$@" if [ "$coold_vm_enabled" != "false" ]; then - for index in $(seq 1 "$count"); do - host_id="$(coold_vm_instance "$index")" - echo "==> Minting Flux dev host JWT for ${host_id}..." - host_jwt="$(mint_host_jwt_for_host "$host_id")" - echo "==> Installing host JWT into ${host_id}..." - coold_vm "$index" install-host-jwt "$host_jwt" - done + echo "==> Bootstrapping coold VM mesh with cooldctl..." + cooldctl_bootstrap for index in $(seq 1 "$count"); do - echo "==> Starting coold VM agent service on $(coold_vm_instance "$index")..." - if [ "$count" -ge 2 ]; then - peer_index=1 - if [ "$index" = "1" ]; then - peer_index=2 - fi - COOLIFY_COOLD_VM_WG_PEER_IP="$(coold_vm_wg_ip "$peer_index")" coold_vm "$index" start-agent - else - coold_vm "$index" start-agent - fi + configure_flux_dev_for_vm "$index" done fi @@ -244,6 +473,48 @@ down() { fi } +clean_vms() { + local count + count="$(coold_vm_count)" + + echo "==> Deleting ${count} coold Lima VM(s). This removes VM-local state." + for index in $(seq 1 "$count"); do + instance="$(coold_vm_instance "$index")" + echo "==> Deleting ${instance}..." + limactl stop --force --tty=false "$instance" >/dev/null 2>&1 || true + if ! run_with_timeout 60 limactl delete --force --tty=false "$instance"; then + echo "WARN: limactl delete timed out for ${instance}; killing matching limactl clients." >&2 + pkill -f "limactl.*${instance}" >/dev/null 2>&1 || true + rm -rf "$HOME/.lima/${instance}" + fi + done +} + +run_with_timeout() { + local timeout_seconds="$1" + shift + local pid + local elapsed=0 + + "$@" & + pid="$!" + + while kill -0 "$pid" >/dev/null 2>&1; do + if [ "$elapsed" -ge "$timeout_seconds" ]; then + kill "$pid" >/dev/null 2>&1 || true + sleep 2 + kill -9 "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + return 124 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + wait "$pid" +} + corrosion_for_each_vm() { local label="$1" shift @@ -511,7 +782,7 @@ firewall_api_for_each_vm() { COOLIFY_COOLD_LIMA_INSTANCE="$instance" scripts/coold-vm.sh shell < Inspect Corrosion state, config, logs, and registered containers firewall Manage dev coold firewall allow rules example-nginx Start/check example nginx containers with coold DNS + cooldctl Install/run the released cooldctl dev helper USAGE } @@ -660,6 +933,9 @@ case "$cmd" in list) limactl list ;; + clean-vms|clean-vm|reset-vms) + clean_vms + ;; corrosion) corrosion "$@" ;; @@ -669,6 +945,9 @@ case "$cmd" in example-nginx) example_nginx "$@" ;; + cooldctl) + cooldctl_dev "$@" + ;; -h|--help|help|"") usage ;;