diff --git a/.pct-helpers b/.pct-helpers index 108aa1a..4c79715 100644 --- a/.pct-helpers +++ b/.pct-helpers @@ -302,33 +302,81 @@ xreadpass(){ } +# Like cat but a prettier... +# +# listFile PATH +# +listFile(){ + if [ -e "$1" ] ; then + echo "--- $1 ---" + cat "$1" + echo '---' + else + echo "$FUNCNAME: $1: No such file or directory." + return 1 + fi +} + + # Review changes in PATH.new, then edit/apply changes to PATH # -# reviewApplyChanges PATH +# reviewApplyChanges PATH [apply|edit|skip] +# # # NOTE: if changes are not applied this will return non-zero making this # usable in conditionals... reviewApplyChanges(){ local file=$1 + if ! [ -e "$file".new ] ; then + echo "$FUNCNAME: $1: No such file or directory." + return 1 + fi + + # default option... + local dfl= + local a=a + local e=e + local s=s + case "${2,,}" in + a|apply) + a=A + dfl=a + ;; + e|edit) + e=E + dfl=e + ;; + s|skip) + s=S + dfl=s + ;; + esac + echo "# Review updated: ${file}.new:" - @ cat ${file}.new - echo '---' + listFile ${file}.new local res while true ; do - read -ep "# [a]pply, [e]dit, [s]kip? " res + read -ep "# [$a]pply, [$e]dit, [$s]kip? " res + if [ -z $res ] ; then + if [ -z $dfl ] ; then + continue + fi + res=$dfl + fi case "${res,,}" in a|apply) break ;; e|edit) ${EDITOR} "${file}.new" + listFile ${file}.new ;; s|skip) - echo "# file saved as: ${file}.new" + echo "# Changes kept as: ${file}.new" return 1 ;; *) - echo "ERROR: unknown command: \"$res\"" >&2 + echo "ERROR: Unknown command: \"$res\"" >&2 continue ;; esac diff --git a/Makefile b/Makefile index 6ad588c..2d2bdb5 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ # #---------------------------------------------------------------------- -EDITOR ?= vim +EDITOR ?= nano # CTs... @@ -62,6 +62,7 @@ FORCE: %: config %/make.sh FORCE $*/make.sh + @echo %.config: %/config.example @@ -94,7 +95,10 @@ bootstrap-clean: host-bootstrap-clean # Finalize: reconect admin port/bridge correctly... .PHONY: finalize -finalize: bootstrap-clean gate-bootstrap-clean +finalize: gate-bootstrap-clean + # cleanup: stage 1... + make host-bootstrap-clean + # cleanup: stage 2... make host-bootstrap-clean diff --git a/README.md b/README.md index c6e8e57..ed12063 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,28 @@ A set of scripts for automating setup and tasks in proxmox. ## TODO -- revise defaults +- CT updates / upgrades + Right now the simplest way to update the infrastructure CT's if the + sources changed is to simply rebuild them -- add rebuild command. + - backup + - build (new reserve) + - destroy + - clone + - cleanup +- backup/restore +- config manager -- save/use/.. +- mail server +- which is better? + - Makefile (a-la ./wireguard/templates/root/Makefile) + - shell (a-la ./shadow/templates/root/update-shadowsocks.sh) - separate templates/assets into distribution and user directories ...this is needed to allow the user to change the configs without the fear of them being overwritten by git (similar to how config is handlerd) -- might be a good idea to export a specific ct script that can be used - for updates for that ct -- which is better? - - Makefile (a-la wireguard) - - shell (a-la shadow) -- ct updates -- backup/restore -- mail + + + + + ## Motivation @@ -25,21 +35,25 @@ functionality in Ansible. _NOTE: for a fair assessment of viability of further development an Ansible version will be implemented next as a direct comparison._ +Fun. + ## Architecture -Goals: -- Separate concerns +### Goals + +- _Separate concerns_ Preferably one service/role per CT -- Keep things as light as possible +- _Keep things as light as possible_ This for the most part rules out Docker as a nested virtualization - layer under Proxmox while preferring light distributions like Alpine + layer under Proxmox, and preferring light distributions like Alpine Linux -- Pragmatic simplicity - This goal yields some compromises to previous goals, for example [TKL]() - is used as a base for [Nextcloud]() effectively simplifying the setup +- _Pragmatic simplicity_ + This goal yields some compromises to previous goals, for example + [TKL](https://www.turnkeylinux.org/) is used as a base for + [Nextcloud](https://nextcloud.com/) effectively simplifying the setup and administration of all the related components at the cost of a - heavier CT transparently integrating multiple related services + heavier CT, transparently integrating multiple related services ### Network @@ -77,60 +91,96 @@ Goals: +---------------------------------------------------------------+ ``` -XXX +The system defines two networks: +- LAN + Hosts all the service CT's (`*.srv`) +- ADMIN + Used for administration (`*.adm`) +The ADMIN network is connected to the admin port. -### Services +Both networks are provided DNS and DHCP services by the `ns` CT. -XXX +Services on either network are connected to the outside world (WAN) via +a NAT router implemented by the `gate` CT (`iptables`). + +The `gate` CT also implements a reverse proxy ([`traefik`](https://traefik.io/traefik/)), +routing requests from the WAN (`$WAN_IP`) to appropriate service CT's on +the LAN. + +Services expose their administration interfaces only on the ADMIN network +when possible. + +The host Proxmox (`pve.adm`) is only accessible through the ADMIN network. + +The `gate` and `ns` CT's are only accessible for administration from the +host (i.e. via `lxc-attach ..`). + +Three ways of access to the ADMIN network are provided: +- [`wireguard`](https://www.wireguard.com/) VPN (CT) via `gate` reverse proxy, +- `ssh` service (CT) via the `gate` reverse proxy, +- `ssh` service (CT) via the direct `$WAN_SSH_IP` (fail-safe). -## Setup +## Getting started ### Prerequisites Install Proxmox and connect it to your device/network. -Proxmox will need to have internet access +Proxmox will need to have access to the internet to download assets and +updates. + + +#### Notes This setup will use three IP addresses: -1. IP address used for setup only, this is the static (usually) IP - initially assigned to Proxmox on install and it will not be used after - setup is done, -2. WAN IP adress to be used for the main set of applications, this is +1. The static (usually) IP initially assigned to Proxmox on install. This + will not be used after setup is done, +2. WAN IP address to be used for the main set of applications, this is the address that all the requests will be routed from to various - services internally, + services on the LAN network, 3. Fail-safe ssh IP address, this is the connection used for recovery in case the internal routing fails. -### Semi-automated setup +### Setup + +Open a terminal on the host, either `ssh` (recommended) or via the UI. + +Optionally, set a desired default editor (default: `nano`) via: +```shell +export EDITOR=nano +``` Download the [`bootstrap.sh`](./scripts/bootstrap.sh) script and execute it: ```shell curl 'https://raw.githubusercontent.com/flynx/proxmox-utils/refs/heads/master/scripts/bootstrap.sh' | sudo bash ``` +_It is recommended to review the script/code before starting._ + This will: -- Install basic dependencies -- Clone this repo -- Run `make bootstrap` on the repo +- Install basic dependencies, +- Clone this repo, +- Run `make bootstrap` on the repo: + - bootstrap configure the network (2 out of 3 stages) + - build and infrastructure start CT's (`gate`, `ns`, `ssh`, and `wireguard`) -After the basic setup is done connect the device to the network via the -selcted WAN port and **disconnect** the ADMIN port. - -The WAN interface exposes two IPs: +At this point WAN interface exposes two IPs: - Main server (config: `$DFL_WAN_IP` / `$WAN_IP`) - ssh:23 - wireguard:51820 - Fail-safe ssh (config: `$DFL_WAN_SSH_IP` / `$WAN_SSH_IP`) - ssh:22 +The Proxmox administrative interface is available behind the +[Wireguard](https://www.wireguard.com/) proxy or on the ADMIN port, both +on https://10.0.0.254:8006. -The Proxmox administrative interface is available behind the Wireguard -proxy or on the ADMIN port, both on https://10.0.0.254:8006. +Additional administrative tasks can be performed now if needed. To finalize the setup run: ```shell @@ -138,56 +188,150 @@ make finalize ``` This will -- detach the host from any external ports and make it accessible only - from the internal network. - See: [Architecture](#architecture) and [Bootstrapping](#bootstrapping) -- setup firewall rules. +- Setup firewall rules. Note that the firewall will not be enabled, this should be done manually after rule review. - +- Detach the host from any external ports and make it accessible only + from the internal network. + See: [Architecture](#architecture) and [Bootstrapping](#bootstrapping) + +This will break the ssh connection when done, reconnect via the WAN port +to continue (see: [Accessing the host](#accessing-the-host)), or connect +directly to the ADMIN port (DHCP) and ssh into `$HOST_ADMIN_IP` (default: 10.0.0.254). + +_Note that the ADMIN port is configured for direct connections only, +connecting it to a configured network can lead to unexpected behavior -- +DHCP races, IP clashes... etc._ -*Note that the ADMIN port is configured for direct connections only (DHCP), -connecting it to a configured network can lead to unexpected behavior.* #### Accessing the host -XXX +The simplest way is to connect to `wireguard` VPN and open http://pve.adm:8006 +in a browser (a profile was created during the setup process and stored +in the `/root/clients/` directory on the `wireguard` CT). + +The second approach is to `ssh` to either: + +```shell +ssh -p 23 @ +``` + +or: +```shell +ssh @ +``` + +The later will also work if the `gate` CT is down or not accessible. -#### Setup additional services +And from the `ssh` CT: +```shell +ssh root@pve +``` + +_WARNING: NEVER store any ssh keys on the `ssh` CT, use `ssh-agent` instead!_ + + + +#### Configuration XXX +The following CT's interfaces can not be configured in the Proxmox UI: +- `gate` +- `ns` +- `nextcloud` +- `wireguard` + +This is done mostly to keep Proxmox from touching the `hostname $(hostname)` +directive (used by the DNS server to assigned predefined IP's) and in +the case of `gate` and `wireguard` to keep it from touching the additional +bridges or interfaces defined. +(XXX this restriction may be lifted in the future) + + + +## Services + +Install all user services: ```shell make all ``` +Includes: +- [`syncthing`](#syncthing) +- [`nextcloud`](#nextcloud) + + +Install development services: ```shell make dev ``` +Includes: +- [`gitea`](#gitea) -Or individually: -```shell -make nextcloud -``` + + +### Syncthing ```shell make syncthing ``` +Syncthing administration interface is accessible via https://syncthing.adm/ +on the ADMIN network, it is recommended to set an admin password on +the web interface as soon as possible. + +No additional routing or network configuration is required, Syncthing is +smart enough to handle its own connections itself. + +For more info see: https://syncthing.net/ + + +### Nextcloud + +```shell +make nextcloud +``` + +Nextcloud will get mapped to subdomain `$NEXTCLOUD_SUBDOMAIN` of +`$NEXTCLOUD_DOMAIN` (defaulting to `$DOMAIN`, if not defined). + +For basic configuration edit the generated: [config.global](./config.global) +and for defaults: [config.global.example](./config.global.example). + +For deeper management use the [TKL](https://www.turnkeylinux.org/) consoles +(via https://nextcloud.srv, on the LAN network) and `ssh`, for more details +see: https://www.turnkeylinux.org/nextcloud + +For more info on Nextcloud see: https://nextcloud.com/ + + +### Gitea + ```shell make gitea ``` +Gitea is mapped to the subdomain `$GITEA_SUBDOMAIN` of `$GITEA_DOMAIN` +or `$DOMAIN` if the former is not defined. -#### Setup and configure custom services +For basic configuration edit the generated: [config.global](./config.global) +and for defaults: [config.global.example](./config.global.example). + +For more info see: https://gitea.com/ + + +### Custom services XXX traefik rules + + ## Extending @@ -325,11 +470,14 @@ XXX firewall ### Directory structure ``` -/ +proxmox-utils/ +- / | +- templates/ +| | +- ... | +- assets/ +| | +- ... | +- staging/ +| | +- ... | +- make.sh | +- config | +- config.last-run diff --git a/config.global.example b/config.global.example index 8ad34f1..fac9161 100644 --- a/config.global.example +++ b/config.global.example @@ -1,4 +1,4 @@ -#------------------------------------------------------------------------ +#---------------------------------------------------------------------- # # Global config file # @@ -16,15 +16,74 @@ # It is not recomended to set passwords here or in other config files. # # -#------------------------------------------------------------------------ +#---------------------------------------------------------------------- +# These options need to be revised or changed... +# (remove "DFL_" prefix to disable promting) +# + +# Domain and email configuration +# +DFL_DOMAIN=example.com +DFL_EMAIL=user@example.com + + +# Network configuration... +# +# NOTE: it is simpler to statically assign these than to configure DHCP +# plus port forewarding to the dynamically assigned IP. +# NOTE: if installing on a different network than the target, these can +# be changed for target deployment in: +# - gate CT's /etc/network/interfaces (NOT in the Proxmox UI) +# - ssh CT's network configuration (Proxmox UI) +DFL_WAN_IP=192.168.1.101/24 +DFL_WAN_GATE=192.168.1.252 + +# IP used for fail-safe conection to the ADMIN network +DFL_WAN_SSH_IP=192.168.1.102/24 + + +# Web app/service domain configuration +# +# Here two optional variables are provided per service: +# - _DOMAIN=... +# Overrides the $DOMAIN option above for +# - _SUBDOMAIN=... +# Sets the subdomain of $DOMAIN (or $_DOMAIN) for + +# Nextcloud +#NEXTCLOUD_DOMAIN= +NEXTCLOUD_SUBDOMAIN=nc. + +# Gitea +#GITEA_DOMAIN= +#GITEA_SUBDOMAIN=git. + + +# Extra options passed to each CT when created. +# +# This can be used for passing in ssh keys, etc... +# +# see: +# man pct +# +# Example: +# DFL_PCT_EXTRA="--ssh-public-keys /path/to/autohrized_keys" +# +DFL_PCT_EXTRA=SKIP + + + +#---------------------------------------------------------------------- +# +# Options afetr this point are sane defaults and in the general case +# can be left as-is. +# # Bootsrap configuration... # # Usually this is the default bridge created in Proxmox, so there is no # need to touch this. BOOTSTRAP_BRIDGE=0 -# XXX -#BOOTSTRAP_PORT=none # CT interface bridge configuration. @@ -39,71 +98,21 @@ BOOTSTRAP_BRIDGE=0 # bridges with numbers greater than X (10 in the example below) # # Example: -# WAN_BRIDGE=0 -# ADMIN_BRIDGE=3 -# LAN_BRIDGE=10 +# ADMIN_BRIDGE=_admin +# WAN_BRIDGE=_wan +# LAN_BRIDGE=_lan # -# XXX revise numbering... ADMIN_BRIDGE=_admin WAN_BRIDGE=_wan LAN_BRIDGE=_lan -# NOTE: it is simpler to statically assign these than to configure dhcp -# plus port forewarding to the dynamically assigned IP. -DFL_WAN_IP=192.168.1.101/24 -DFL_WAN_GATE=192.168.1.252 - -DFL_WAN_SSH_IP=192.168.1.102/24 - - -# Domain and email configuration -# -DOMAIN=example.com -EMAIL=user@example.com - - -# Web app/service domain configuration -# -# Here two optional variables are provided per service: -# - _DOMAIN=... -# Overrides the $DOMAIN option above for -# - _SUBDOMAIN=... -# Sets the subdomain of $DOMAIN (or $_DOMAIN) for -# - -# Nextcloud -#NEXTCLOUD_DOMAIN= -NEXTCLOUD_SUBDOMAIN=nc. - -# Gitea -#GITEA_DOMAIN= -#GITEA_SUBDOMAIN=git. - - -# Extra options passed to each CT created. -# -# This can be used for passing in ssh keys, etc... -# -# see: -# man pct -# -# Example: -# DFL_PCT_EXTRA="--ssh-public-keys /path/to/autohrized_keys" -# -DFL_PCT_EXTRA=SKIP - - - -#------------------------------------------------------------------------ -# -# Options afetr this point are sane defaults and in the general case -# can be left as-is. -# +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # host HOST_ADMIN_IP=10.0.0.254/24 + # Nameserver NS_HOSTNAME=ns NS_ID=100 @@ -113,6 +122,7 @@ NS_LAN_IP=10.1.1.1/24 RESERVE_NS_ID=101 TEMPLATE_NS_ID=200 + # Gateway / Reverse proxy GATE_HOSTNAME=gate GATE_ID=110 @@ -123,4 +133,5 @@ RESERVE_GATE_ID=111 TEMPLATE_GATE_ID=210 -#------------------------------------------------------------------------ + +#---------------------------------------------------------------------- diff --git a/gate-traefik/make.sh b/gate-traefik/make.sh index 8ff7a92..3050c1f 100755 --- a/gate-traefik/make.sh +++ b/gate-traefik/make.sh @@ -25,8 +25,8 @@ RAM=128 SWAP=$RAM DRIVE=0.5 -DFL_WAN_IP=${DFL_WAN_IP} -DFL_WAN_GATE=${DFL_WAN_GATE} +#DFL_WAN_IP=${DFL_WAN_IP} +#DFL_WAN_GATE=${DFL_WAN_GATE} # XXX revise... DFL_ADMIN_IP=${GATE_ADMIN_IP:=${DFL_ADMIN_IP:=10.0.0.2/24}} diff --git a/host/make.sh b/host/make.sh index e16c240..fb77960 100755 --- a/host/make.sh +++ b/host/make.sh @@ -38,7 +38,9 @@ SOFTWARE=( INTERFACES=/etc/network/interfaces -BRIDGES_TPL=bridges.tpl +BOOTSTRAP_PORT=${BOOTSTRAP_PORT:-none} + +BRIDGES_TPL=${BRIDGES_TPL:-bridges.tpl} # XXX #readVars @@ -47,9 +49,23 @@ BRIDGES_TPL=bridges.tpl # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Bootstrap... +# cleanup... if ! [ -z $BOOTSTRAP_CLEAN ] ; then @ cp "$INTERFACES"{,.bak} + __finalize(){ + if reviewApplyChanges "$INTERFACES" apply ; then + # XXX this must be done in nohup to avoid breaking on connection lost... + if ! @ ifreload -a ; then + # reset settings back if ifreload fails... + @ cp "$INTERFACES"{.bak,} + @ ifreload -a + fi + fi + # clear self to avoid a second deffered execution... + unset -f __finalize + } + # stage 1: bootstrap -> clean if [ -e "$INTERFACES".clean ] ; then @ mv "$INTERFACES"{.clean,.new} @@ -60,6 +76,9 @@ if ! [ -z $BOOTSTRAP_CLEAN ] ; then DFL_DNS=1 DFL_FIREWALL=SKIP + # NOTE: in general this is non-destructive and can be done inline. + __finalize + # stage 2: clean -> final elif [ -e "$INTERFACES".final ] ; then @ mv "$INTERFACES"{.final,.new} @@ -70,6 +89,8 @@ if ! [ -z $BOOTSTRAP_CLEAN ] ; then DFL_DNS=SKIP DFL_FIREWALL=1 + # NOTE: __finalize is deferred to just before reboot... + REBOOT=1 # done @@ -77,15 +98,6 @@ if ! [ -z $BOOTSTRAP_CLEAN ] ; then exit fi - if reviewApplyChanges "$INTERFACES" ; then - # XXX this must be done in nohup to avoid breaking on connection lost... - if ! @ ifreload -a ; then - # reset settings back if ifreload fails... - @ cp "$INTERFACES"{.bak,} - @ ifreload -a - fi - fi - # Bootstrap... elif ! [ -z $BOOTSTRAP ] ; then DFL_BOOTSTRAP_PORT=${DFL_BOOTSTRAP_PORT:-none} @@ -105,20 +117,20 @@ fi #---------------------------------------------------------------------- -# System... +# system... if xreadYes "# Update system?" UPDATE ; then @ apt update @ apt upgrade fi -# Tools... +# tools... if xreadYes "# Install additional apps?" APPS ; then @ apt install ${SOFTWARE[@]} fi -# Bridges... +# bridges... if xreadYes "# Create bridges?" BRIDGES ; then xread "WAN port: " WAN_PORT xread "ADMIN port: " ADMIN_PORT @@ -201,7 +213,7 @@ if xreadYes "# Create bridges?" BRIDGES ; then fi # interfaces - if reviewApplyChanges "$INTERFACES" ; then + if reviewApplyChanges "$INTERFACES" apply ; then # XXX this must be done in nohup to avoid breaking on connection lost... if ! @ ifreload -a ; then # reset settings back if ifreload fails... @@ -219,7 +231,7 @@ if xreadYes "# Update /etc/hosts?" HOSTS ; then @ sed -i \ -e 's/^[^#].* \(pve.local.*\)$/'${HOST_ADMIN_IP/\/*}' \1/' \ /etc/hosts.new - reviewApplyChanges /etc/hosts + reviewApplyChanges /etc/hosts apply fi @@ -238,7 +250,7 @@ if xreadYes "# Update DNS?" DNS ; then build file=/etc/resolv.conf @ cp "staging/${file}" "${file}".new - reviewApplyChanges "${file}" + reviewApplyChanges "${file}" apply fi @@ -247,7 +259,7 @@ if xreadYes "# Update firewall rules?" FIREWALL ; then build file=/etc/pve/firewall/cluster.fw @ cp "staging/${file}" "${file}".new - reviewApplyChanges "${file}" + reviewApplyChanges "${file}" apply fi @@ -255,6 +267,14 @@ showNotes echo "# Done." +# finalize... +if [[ $( type -t __finalize ) == "function" ]] ; then + echo "# Finalizing ${INTERFACES}..." + __finalize +fi + + +# reboot... if ! [ -z $REBOOT ] ; then echo "# Rebooting..." @ reboot diff --git a/ns/templates/etc/dnsmasq.conf b/ns/templates/etc/dnsmasq.conf index b8df181..9afbc86 100644 --- a/ns/templates/etc/dnsmasq.conf +++ b/ns/templates/etc/dnsmasq.conf @@ -43,12 +43,16 @@ dhcp-range=interface:admin,10.0.0.20,10.0.0.200,12h dhcp-range=interface:lan,10.1.1.20,10.1.1.200,12h # ns -address=/${CTHOSTNAME}/${ADMIN_IPn} address=/${CTHOSTNAME}/${LAN_IPn} +address=/${CTHOSTNAME}.srv/${LAN_IPn} +#address=/${CTHOSTNAME}/${ADMIN_IPn} +address=/${CTHOSTNAME}.adm/${ADMIN_IPn} # gate -address=/${GATE_HOSTNAME}/${GATE_ADMIN_IPn} address=/${GATE_HOSTNAME}/${LAN_GATE} +address=/${GATE_HOSTNAME}.srv/${LAN_GATE} +#address=/${GATE_HOSTNAME}/${GATE_ADMIN_IPn} +address=/${GATE_HOSTNAME}.adm/${GATE_ADMIN_IPn} dhcp-option=admin,option:router,${GATE_ADMIN_IPn} dhcp-option=lan,option:router,${LAN_GATE} dhcp-host=admin,gate,${GATE_ADMIN_IPn},infinite @@ -56,6 +60,7 @@ dhcp-host=lan,gate,${LAN_GATE},infinite # pve address=/pve/10.0.0.254 +address=/pve.adm/10.0.0.254 diff --git a/ssh/make.sh b/ssh/make.sh index c9d9a5e..e9d4a0d 100755 --- a/ssh/make.sh +++ b/ssh/make.sh @@ -24,10 +24,10 @@ readConfig DFL_ID=${DFL_ID:=120} DFL_CTHOSTNAME=${DFL_CTHOSTNAME:=ssh} -DFL_CORES=${DFL_CORES:=1} -DFL_RAM=${DFL_RAM:=1024} -DFL_SWAP=${DFL_SWAP:=${DFL_RAM}} -DFL_DRIVE=${DFL_DRIVE:=16} +DFL_CORES=${DFL_SSH_CORES:=1} +DFL_RAM=${DFL_SSH_RAM:=512} +DFL_SWAP=${DFL_SSH_SWAP:=${DFL_RAM}} +DFL_DRIVE=${DFL_SSH_DRIVE:=16} WAN_IP=SKIP WAN_GATE=SKIP