#!/usr/bin/env bash set -o errexit set -o nounset set -o pipefail checkBashVersion() { if (( "${BASH_VERSION%%.*}" < 4 )); then echo "This script requires Bash major version 4 or greater." echo "Detected version: $BASH_VERSION" if [[ ${OSTYPE:-} =~ [Dd]arwin ]]; then echo -e "\nOn macOS, you can install bash with Homebrew:" echo "brew install bash" echo -e "\nThen run:" #shellcheck disable=SC2016 echo '"$(brew --prefix bash)/bin/bash" ./miniziti.bash ...' fi exit 1; fi } banner(){ local profile="${1-}" cat <&2 else banner fi echo -e "\n Basic Commands:\n"\ " start\t\tstart miniziti\n"\ " delete\t\tdelete miniziti\n"\ " console\t\topen ziti-console in browser\n"\ " creds\t\tprints admin user updb credentials\n"\ " status\t\tprint ziti component status\n"\ " ziti\t\tziti cli wrapper\n"\ " help\t\tshow these usage hints\n"\ "\n Profile Commands:\n"\ " profile list\tlist available profiles\n"\ " profile show\tshow default profile\n"\ " profile use\t\tset default profile\n"\ "\n Advanced Commands:\n"\ " shell\t\trun interactive shell inside the ziti-controller container\n"\ " login\t\trun local ziti binary edge login\n"\ "\n Other Commands:\n"\ " kubectl\t\tkubectl cli wrapper\n"\ " minikube\t\tminikube cli wrapper\n"\ "\n Options:\n"\ " --quiet\t\tsuppress INFO messages\n"\ " --verbose\t\tshow DEBUG messages\n"\ " --profile\t\tMINIKUBE_PROFILE (miniziti)\n"\ " --namespace\t\tZITI_NAMESPACE (MINIKUBE_PROFILE)\n"\ " --no-hosts\t\tdon't use local hosts DB; use wildcard DNS zone based on minikube node IP\n"\ " --modify-hosts\tadd entries to local hosts database. Requires sudo if not running as root. Linux only.\n"\ "\n Debug:\n"\ " --charts\t\tZITI_CHARTS_REF (openziti) alternative charts repo\n"\ " --values-dir\tEXTRA_VALUES_DIR with Helm values files named ziti-controller.yaml or ziti-router.yaml\n"\ " --devel\t\tinclude Helm chart prereleases (passes --devel to OpenZiti chart installs/upgrades)\n"\ " --now\t\teliminate safety waits, e.g., before deleting miniziti\n"\ " --\t\t\tMINIKUBE_START_ARGS args after -- passed to minikube start\n" } _usageProfileUse(){ echo -e "\nSet miniziti default profile\n"\ "\n Usage:\n"\ " ziti profile use [profile]\n" } _usageProfile(){ if (( $# )); then echo -e "\nERROR: unexpected arg '$1'" >&2 fi echo -e "\nProfile management commands\n"\ "\n Usage:\n"\ " ziti profile [command]\n"\ "\n Available Commands:\n"\ " list\tlist available profiles\n"\ " show\tshow default profile\n"\ " use\t\tset miniziti profile\n" } checkDns(){ # only param is an IPv4 [[ $# -eq 1 && $1 =~ ([0-9]{1,3}\.){3}[0-9]{1,3} ]] || { logError "need IPv4 of miniziti ingress as only param to checkDns()" return 1 } logDebug "checking dns to ensure ${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE} resolves to $1" if grep -qE "^${1//./\\.}.*${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" /etc/hosts \ || nslookup "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" | grep -q "${1//./\\.}"; then logDebug "host dns found expected minikube ingress IP '$1'" return 0 else logError "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE} does not resolve to '$1'. Did you add the record in /etc/hosts?" return 1 fi } deleteMiniziti(){ local WAIT=30 if [[ $1 =~ ^[0-9]+$ ]]; then WAIT="$1" shift else logDebug "no integer param detected to deleteMiniziti(), using default wait time ${WAIT}s" fi (( SAFETY_WAIT )) && { logWarn "deleting ${MINIKUBE_PROFILE} in ${WAIT}s, use --now to skip wait" >&2 sleep "$WAIT" } logInfo "waiting for ${MINIKUBE_PROFILE} to be deleted" minikube --profile "${MINIKUBE_PROFILE}" delete >&3 } detectOs(){ if grep -qi "microsoft" /proc/sys/kernel/osrelease 2>/dev/null; then logDebug "detected Windows OS" echo "Windows" elif [[ ${OSTYPE:-} =~ [Dd]arwin ]]; then logDebug "detected macOS OS" echo "macOS" elif [[ ${OSTYPE:-} =~ [Ll]inux ]]; then logDebug "detected Linux OS" echo "Linux" else logError "failed to detect OS" return 1 fi } makeMinizitiStateDir() { case "$DETECTED_OS" in macOS) state_dir="${XDG_STATE_DIR:-$HOME/Library/Application Support}/miniziti" ;; Linux|Windows) state_dir="${XDG_STATE_DIR:-$HOME/.local/state}/miniziti" ;; *) logError "Unknown os: $DETECTED_OS" exit 1 esac if [[ ! -d "$state_dir" ]]; then logDebug "Creating miniziti state directory: $state_dir" mkdir -p "$state_dir" fi echo "$state_dir" } getZitiCliHome() { case "$DETECTED_OS" in Linux) echo "$HOME/.config/ziti" ;; *) echo "$HOME/.ziti" ;; esac } pathToNative() { local path="$1" case "$DETECTED_OS" in Windows) wslpath -w "$path" ;; *) echo "$path" ;; esac } testClusterDns(){ if kubectlWrapper run "dnstest" --rm --tty --stdin --image busybox --restart Never -- \ nslookup "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" | grep "$1" >&3; then logInfo "cluster dns test succeeded" else logError "cluster dns test failed" return 1 fi } validateDnsName(){ if ! [[ $# -eq 1 ]]; then logError "validateDnsName() takes one string param" return 1 fi if grep -qE '^[a-zA-Z0-9]([-a-zA-Z0-9]{2,251})[a-zA-Z0-9]$' <<<"$1"; then logDebug "'$1' is a valid DNS name" return 0 else logError "'$1' could not be validated as an unqualified DNS name which is limited to at least four alphanumeric and hyphen characters and does not start nor end with a hyphen." return 1 fi } checkSudoRequired() { if (( EUID != 0 )); then logInfo "sudo is required when not running as root" fi } HOSTS_FILE='/etc/hosts' cleanHosts() { local marker="$1" logWarn "Removing stale miniziti entries from $HOSTS_FILE" if (( EUID != 0 )); then sudo sed -i "/$marker/d" "$HOSTS_FILE" else sed -i "/$marker/d" "$HOSTS_FILE" fi } installHosts() { hosts=( "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" "${MINIZITI_ROUTER_NAME}.${MINIZITI_INGRESS_ZONE}" ) hosts_line="$MINIKUBE_NODE_EXTERNAL ${hosts[*]}" if ! grep -q "$hosts_line" "$HOSTS_FILE"; then checkSudoRequired if grep -q "$MINIZITI_INGRESS_ZONE" "$HOSTS_FILE"; then cleanHosts "$MINIZITI_INGRESS_ZONE" fi logInfo "Adding miniziti entries to $HOSTS_FILE" if (( EUID != 0 )); then echo "$hosts_line" | sudo tee -a "$HOSTS_FILE" > /dev/null else echo "$hosts_line" | tee -a "$HOSTS_FILE" > /dev/null fi fi } getAdminSecret() { kubectlWrapper get secrets "ziti-controller1-admin-secret" \ --namespace "$ZITI_NAMESPACE" \ --output go-template='{{index .data "admin-password" | base64decode }}' } showAdminCreds() { logInfo "The password for 'admin' is: '$(getAdminSecret)'" } getConfigMapKey() { local configmap="$1" local key="$2" kubectlWrapper get configmap "$configmap" \ --ignore-not-found=true \ --output jsonpath="{.data.$key}" 2> /dev/null } getIngressZone() { local ingress_zone local traefik_lb_ip ingress_zone="$(getConfigMapKey "$MINIZITI_CONFIGMAP" 'ingress-zone')" if [[ -z "$ingress_zone" ]]; then logError "Failed to retrieve ingress zone. Did you start profile '$MINIKUBE_PROFILE' successfully?" exit 1 elif [[ "$ingress_zone" =~ \.sslip\.io$ ]]; then # For no-hosts mode, prefer the currently assigned Traefik LB IP in case # a stale configmap still points at the minikube node IP. traefik_lb_ip=$(kubectlWrapper get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2> /dev/null || true) if [[ -n "$traefik_lb_ip" ]]; then ingress_zone="${traefik_lb_ip}.sslip.io" fi else echo "$ingress_zone" return fi echo "$ingress_zone" } minizitiLogin() { checkCommand ziti ingress_zone="$(getIngressZone)" getAdminSecret | xargs ziti edge login "https://${ZITI_NETWORK_NAME}.${ingress_zone}:443/edge/management/v1" \ --cli-identity "$MINIKUBE_PROFILE" \ --yes \ --username "admin" \ --password logInfo "Setting default ziti identity to: $MINIKUBE_PROFILE" ziti edge use "$MINIKUBE_PROFILE" >&3 } minizitiConsole() { ingress_zone="$(getIngressZone)" console_url="https://${ZITI_NETWORK_NAME}.${ingress_zone}/zac/" case "$DETECTED_OS" in "Windows") checkCommand wslview wslview "$console_url" ;; "macOS") checkCommand open open "$console_url" ;; *) checkCommand xdg-open xdg-open "$console_url" ;; esac logInfo "Opening ziti-console: $console_url ..." } listProfiles() { dir --format=single-column --ignore=default --sort=time --reverse "$PROFILES_DIR" } logger() { local caller="${FUNCNAME[1]}" if (( $# < 1 )); then echo "ERROR: $caller() takes 1 or more args" >&2 return 1 fi local message="$*" if [[ "$message" =~ ^r\'(.+)\'$ ]]; then raw_message="${BASH_REMATCH[1]}" message="$raw_message" fi caller_level="${caller##log}" if (( MINIZITI_DEBUG )); then line="${caller_level^^} ${FUNCNAME[2]}:${BASH_LINENO[1]}: $message" else line="${caller_level^^} $message" fi if [[ -n "${raw_message-}" ]]; then echo -E "$line" else echo -e "$line" fi } logInfo() { logger "$*" } logWarn() { logger "$*" >&2 } logError() { logger "$*" >&2 } logDebug() { logger "$*" >&3 } controllerPod() { kubectlWrapper get pods \ --selector app.kubernetes.io/component=ziti-controller \ --output jsonpath='{.items[0].metadata.name}' } getPodStatus() { local pod_name="$1" kubectlWrapper get pod \ --selector app.kubernetes.io/component="$pod_name" \ --ignore-not-found=true \ --output jsonpath='{.items[0].status.phase}' 2> /dev/null } showStatus() { COMPONENTS=("ziti-controller" "ziti-router") for component in "${COMPONENTS[@]}"; do status="$(getPodStatus "$component")" if [[ -z "$status" ]]; then logError "Could not retrieve status for component $component" exit 1 fi echo "$component: $status" done } getDefaultProfile() { default_profile="$DEFAULT_PROFILE" if [[ -L "$default_profile" ]]; then basename "$(realpath "$default_profile")" else echo "miniziti" fi } setProfile() { profile="$1" profiles="$(listProfiles)" if ! echo "$profiles" | grep -q "$profile"; then logError "Invalid profile: $profile\nValid profiles are:\n$profiles" exit 1 else ln -nsf "$PROFILES_DIR/$profile" "$DEFAULT_PROFILE" logInfo "Set default profile to: '$profile'" fi } zitiWrapper() { kubectlWrapper exec "$(controllerPod)" --container ziti-controller -- zitiLogin > /dev/null kubectlWrapper exec "$(controllerPod)" --container ziti-controller -- ziti "$@" } shellWrapper() { kubectlWrapper exec "$(controllerPod)" --container ziti-controller --tty --stdin -- bash } kubectlWrapper() { minikube kubectl --profile "$MINIKUBE_PROFILE" -- --context "$MINIKUBE_PROFILE" "$@" } minikubeWrapper() { minikube --profile "$MINIKUBE_PROFILE" "$@" } helmWrapper() { helm --kube-context "$MINIKUBE_PROFILE" "$@" } checkCommand() { if ! command -v "$1" &>/dev/null; then logError "this script requires command '$1'. Please install on the search PATH and try again." $1 fi } main(){ # xtrace opt implies --verbose if [[ $- =~ x ]]; then MINIZITI_DEBUG=1 exec 3>&1 else MINIZITI_DEBUG=0 fi checkBashVersion >&2 # require commands declare -a BINS=(awk grep helm jq minikube nslookup pgrep sed xargs) for BIN in "${BINS[@]}"; do checkCommand "$BIN" done # open a descriptor for debug messages exec 3>/dev/null # local strings with defaults that never produce an error declare CHECK_CERT_SUBJECT=0 \ DELETE_MINIZITI=0 \ DETECTED_OS \ DO_ZITI_LOGIN=0 \ HELM_INCLUDE_DEVEL=0 \ MINIKUBE_NODE_EXTERNAL \ MINIKUBE_PROFILE \ MINIZITI_HOSTS=1 \ MINIZITI_MODIFY_HOSTS=0 \ OPEN_ZITI_CONSOLE=0 \ RUN_ZITI_CLI=0 \ SAFETY_WAIT=1 \ SHOW_ADMIN_CREDS=0 \ START_MINIZITI=0 \ ZITI_CHARTS_ALT=0 \ ZITI_CHARTS_REF="openziti" \ ZITI_CHARTS_URL="https://raw.githubusercontent.com/openziti/helm-charts/main/charts" \ EXTRA_VALUES_DIR \ ZITI_NAMESPACE # local arrays with defaults that never produce an error declare -a MINIKUBE_START_ARGS=() # local defaults that are inherited or may error DETECTED_OS="$(detectOs)" : "${DEBUG_MINIKUBE_TUNNEL:=0}" # set env = 1 to trigger the minikube tunnel probe : "${MINIZITI_TIMEOUT_SECS:=240}" MINIZITI_CONFIGMAP="miniziti-config" ZITI_CLI_HOME="$(getZitiCliHome)" ZITI_CLI_CERTS_DIR="$ZITI_CLI_HOME/certs" STATE_DIR="$(makeMinizitiStateDir)" PROFILES_DIR="$STATE_DIR/profiles" [[ ! -d "$PROFILES_DIR" ]] && mkdir "$PROFILES_DIR" DEFAULT_PROFILE="$PROFILES_DIR/default" MINIKUBE_PROFILE="$(getDefaultProfile)" while (( $# )); do case "$1" in start) START_MINIZITI=1 shift ;; delete) DELETE_MINIZITI=1 shift ;; console) OPEN_ZITI_CONSOLE=1 shift ;; creds) SHOW_ADMIN_CREDS=1 shift ;; profile) shift if (( $# == 0 )); then _usageProfile exit 1 fi case "$1" in list) shift listProfiles exit ;; use) shift if (( $# != 1 )); then _usageProfileUse exit 1 fi setProfile "$1" exit ;; show) getDefaultProfile exit ;; *) _usageProfile "$1" exit 1 ;; esac ;; login) DO_ZITI_LOGIN=1 shift ;; status) showStatus exit ;; ziti) RUN_ZITI_CLI=1 shift ziti_cli_args=("$@") shift "${#ziti_cli_args[@]}" ;; kubectl) shift kubectlWrapper "${@:-}" exit ;; minikube) shift minikubeWrapper "${@:-}" exit ;; shell) shift shellWrapper "${@:-}" exit ;; -p|--profile) validateDnsName "$2" MINIKUBE_PROFILE="$2" # sanity check the profile name input if [[ ${MINIKUBE_PROFILE} =~ ^- ]]; then # in case the arg value is another option instead of the profile name logError "--profile needs a profile name not starting with a hyphen" _usage exit 1 fi shift 2 ;; -n|--namespace) ZITI_NAMESPACE="$2" shift 2 ;; --charts) ZITI_CHARTS_REF="${2%"/"}" ZITI_CHARTS_URL="${2%"/"}" ZITI_CHARTS_ALT=1 shift 2 ;; --values-dir) EXTRA_VALUES_DIR="${2%"/"}" shift 2 ;; --devel) HELM_INCLUDE_DEVEL=1 shift ;; -q|--quiet) exec > /dev/null shift ;; -v|--verbose|--debug) MINIZITI_DEBUG=1 exec 3>&1 shift ;; --now) SAFETY_WAIT=0 shift ;; --check-cert-subject) CHECK_CERT_SUBJECT=1 shift ;; --no-hosts) MINIZITI_HOSTS=0 shift ;; --modify-hosts) if [[ "$DETECTED_OS" != "Linux" ]]; then logError "The '--modify-hosts' option is only available for Linux" exit 1 fi MINIZITI_MODIFY_HOSTS=1 shift ;; --) shift mapfile -t -n1 MINIKUBE_START_ARGS <<< "$*" shift $# if (( ${#MINIKUBE_START_ARGS[*]} )) && [[ -n "${MINIKUBE_START_ARGS[0]}" ]]; then logDebug "passing ${#MINIKUBE_START_ARGS[*]} args to minikube start: '${MINIKUBE_START_ARGS[*]}'" else MINIKUBE_START_ARGS=() fi ;; -h|*help) _usage exit 0 ;; *) _usage "$1" exit ;; esac done : "${ZITI_NAMESPACE:=${MINIKUBE_PROFILE}}" : "${ZITI_NETWORK_NAME:=${MINIKUBE_PROFILE}-controller}" MINIZITI_INGRESS_ZONE="$MINIKUBE_PROFILE.internal" MINIZITI_INTERCEPT_ZONE="$MINIKUBE_PROFILE.private" PROFILE_DIR="$PROFILES_DIR/${MINIKUBE_PROFILE}" IDENTITIES_DIR="$PROFILE_DIR/identities" if (( DO_ZITI_LOGIN )); then minizitiLogin exit 0 fi if (( SHOW_ADMIN_CREDS )); then showAdminCreds exit 0 fi if (( OPEN_ZITI_CONSOLE )); then minizitiConsole exit 0 fi if (( RUN_ZITI_CLI )); then zitiWrapper "${ziti_cli_args[@]}" exit 0 fi if (( DELETE_MINIZITI )); then banner "$MINIKUBE_PROFILE" trap EXIT if ingress_zone="$(getIngressZone)"; then if (( MINIZITI_MODIFY_HOSTS )) && grep -q "$ingress_zone" "$HOSTS_FILE"; then checkSudoRequired cleanHosts "$ingress_zone" fi CERT_FILE="$(find "$ZITI_CLI_CERTS_DIR" -maxdepth 1 -type f -name "${ZITI_NETWORK_NAME}.$ingress_zone" -print -quit 2> /dev/null)" if [[ -n "$CERT_FILE" ]]; then logWarn "Deleting miniziti certificate file: $CERT_FILE" rm -f "$CERT_FILE" fi fi trap - EXIT deleteMiniziti 10 if [[ -d "$PROFILE_DIR" ]]; then logWarn "Deleting miniziti profile directory: $PROFILE_DIR" rm -rf "$PROFILE_DIR" fi if [[ -L "$DEFAULT_PROFILE" ]]; then if [[ "$(basename "$(realpath "$DEFAULT_PROFILE")")" == "$MINIKUBE_PROFILE" ]]; then unlink "$DEFAULT_PROFILE" fi fi # Cannot nicely call logout until https://github.com/openziti/ziti/issues/1305 is addressed. # if checkCommand ziti &>/dev/null; then # logWarn "Removing $MINIKUBE_PROFILE profile identity from ziti-cli.json" # ziti edge logout --cli-identity "$MINIKUBE_PROFILE" >&3 # fi exit 0 fi if (( START_MINIZITI != 1 )); then _usage exit 0; fi banner "$MINIKUBE_PROFILE" if [[ ! -d "$IDENTITIES_DIR" ]]; then logDebug "Creating miniziti identities directory: ($IDENTITIES_DIR)" mkdir -p "$IDENTITIES_DIR" fi # ## Ensure Minikube is Started and Configured # # run 'minikube start' if not running or any extra start args are present logInfo "waiting for minikube to be ready" if ! minikube status \ --profile "${MINIKUBE_PROFILE}" 2>/dev/null \ | grep -q "apiserver: Running" \ || (( ${#MINIKUBE_START_ARGS[*]} )); then logDebug "apiserver not running or got extra start args, running 'minikube start'" eval minikube start \ --profile "${MINIKUBE_PROFILE}" \ "${MINIKUBE_START_ARGS[*]}" >&3 else logDebug "apiserver is running, not starting minikube" fi MINIKUBE_NODE_EXTERNAL=$(minikube --profile "${MINIKUBE_PROFILE}" ip) # if --no-hosts then build a new zone name for RFC-1918 wildcard DNS (( MINIZITI_HOSTS )) || { MINIZITI_INGRESS_ZONE="${MINIKUBE_NODE_EXTERNAL}.sslip.io" logDebug "DNS wildcard zone for ingresses is ${MINIZITI_INGRESS_ZONE}" } if [[ -n "${MINIKUBE_NODE_EXTERNAL:-}" ]]; then logDebug "the minikube external IP is ${MINIKUBE_NODE_EXTERNAL}" else logError "failed to find minikube external IP" exit 1 fi if (( MINIZITI_MODIFY_HOSTS )); then installHosts fi # verify current context can connect to apiserver kubectlWrapper cluster-info >&3 logDebug "kubectl successfully obtained cluster-info from apiserver" # ingress controller setup happens after Helm repositories are configured. declare -A HELM_REPOS HELM_REPOS[openziti]="openziti.io/helm-charts" HELM_REPOS[jetstack]="charts.jetstack.io" HELM_REPOS[traefik]="traefik.github.io/charts" for REPO in "${!HELM_REPOS[@]}"; do if helmWrapper repo list | cut -f1 | grep -qE "^${REPO}(\s+)?$"; then logDebug "refreshing ${REPO} Helm Charts" helmWrapper repo update "${REPO}" >&3 else logInfo "subscribing to ${REPO} Helm Charts" helmWrapper repo add "${REPO}" "https://${HELM_REPOS[${REPO}]}" >&3 fi done # Enable minikube MetalLB so LoadBalancer services (e.g. Traefik) get an # external IP without requiring a separate `minikube tunnel` step. logInfo "ensuring minikube metallb addon is enabled" minikubeWrapper addons enable metallb >&3 kubectlWrapper wait deployments \ --namespace metallb-system \ --for condition=Available=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" \ --all >&3 local metallb_pool_name local metallb_pool_start local metallb_pool_end local existing_metallb_pool local existing_metallb_legacy_pool local existing_metallb_legacy_config local existing_metallb_legacy_address local traefik_lb_ip metallb_pool_name="" metallb_pool_start="${MINIKUBE_NODE_EXTERNAL%.*}.240" metallb_pool_end="${MINIKUBE_NODE_EXTERNAL%.*}.250" if kubectlWrapper get crd ipaddresspools.metallb.io >/dev/null 2>&1; then existing_metallb_pool=$(kubectlWrapper get ipaddresspool -n metallb-system -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) if [[ -n "${existing_metallb_pool}" ]]; then metallb_pool_name="${existing_metallb_pool}" logInfo "using existing metallb IPAddressPool '${metallb_pool_name}'" else metallb_pool_name="miniziti-pool" logInfo "configuring fallback metallb address pool ${metallb_pool_start}-${metallb_pool_end}" cat <&3 apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: ${metallb_pool_name} namespace: metallb-system spec: addresses: - ${metallb_pool_start}-${metallb_pool_end} --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: ${metallb_pool_name} namespace: metallb-system spec: ipAddressPools: - ${metallb_pool_name} EOF fi else existing_metallb_legacy_config=$(kubectlWrapper get configmap -n metallb-system config -o jsonpath='{.data.config}' 2>/dev/null || true) existing_metallb_legacy_pool=$(sed -n 's/^[[:space:]]*-[[:space:]]*name:[[:space:]]*\([^[:space:]]\+\).*$/\1/p' <<< "${existing_metallb_legacy_config}" | head -n1 || true) existing_metallb_legacy_address=$(sed -n 's/^[[:space:]]*-[[:space:]]*\([^[:space:]]\+\).*$/\1/p' <<< "${existing_metallb_legacy_config}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 || true) if [[ -n "${existing_metallb_legacy_pool}" && -n "${existing_metallb_legacy_address}" ]]; then metallb_pool_name="${existing_metallb_legacy_pool}" logInfo "using existing metallb legacy pool '${metallb_pool_name}'" else metallb_pool_name="miniziti-pool" logInfo "configuring fallback metallb address pool ${metallb_pool_start}-${metallb_pool_end}" # legacy addon path: use non-interactive configmap instead of # `minikube addons configure metallb`, which prompts for stdin. cat <&3 apiVersion: v1 kind: ConfigMap metadata: namespace: metallb-system name: config data: config: | address-pools: - name: ${metallb_pool_name} protocol: layer2 addresses: - ${metallb_pool_start}-${metallb_pool_end} EOF fi fi if kubectlWrapper get deployment "traefik" --namespace traefik >/dev/null 2>&1; then logDebug "traefik is already installed" else logInfo "installing traefik" local -a _traefik_cmd=(upgrade --install traefik traefik/traefik --namespace traefik --create-namespace --set providers.kubernetesCRD.enabled=true --set service.type=LoadBalancer ) helmWrapper "${_traefik_cmd[@]}" >&3 fi logInfo "waiting for traefik to be ready" kubectlWrapper wait deployments traefik \ --namespace traefik \ --for condition=Available=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 kubectlWrapper wait pods \ --namespace traefik \ --for condition=ready \ --selector app.kubernetes.io/name=traefik \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 logInfo "waiting for traefik LoadBalancer external IP" for ((i=0; i/dev/null || true) [[ -n "${traefik_lb_ip}" ]] && break sleep 1 done if [[ -z "${traefik_lb_ip}" ]]; then logError "traefik service external IP is still pending after ${MINIZITI_TIMEOUT_SECS}s" logError "metallb pools:"; kubectlWrapper get ipaddresspool -n metallb-system >&2 || true logError "metallb advertisements:"; kubectlWrapper get l2advertisement -n metallb-system >&2 || true logError "traefik service:"; kubectlWrapper get service traefik -n traefik -o wide >&2 || true exit 1 fi logDebug "traefik LoadBalancer external IP is ${traefik_lb_ip}" # In --no-hosts mode, route advertised hosts through the LoadBalancer IP # instead of minikube node IP so routers use the same reachable north-south # entrypoint as external clients. if (( ! MINIZITI_HOSTS )); then MINIZITI_INGRESS_ZONE="${traefik_lb_ip}.sslip.io" logDebug "DNS wildcard zone for ingresses is ${MINIZITI_INGRESS_ZONE}" fi helmWrapper upgrade --install cert-manager jetstack/cert-manager \ --namespace cert-manager --create-namespace \ --set crds.enabled=true >&3 kubectlWrapper wait deployments -n cert-manager --for condition=Available --timeout="${MINIZITI_TIMEOUT_SECS}s" --all >&3 kubectlWrapper get namespace "${ZITI_NAMESPACE}" &>/dev/null || kubectlWrapper create namespace "${ZITI_NAMESPACE}" >&3 helmWrapper upgrade --install trust-manager jetstack/trust-manager \ --namespace cert-manager \ --set crds.keep=false \ --set app.trust.namespace="${ZITI_NAMESPACE}" >&3 # Wait for trust-manager-tls secret to be created by cert-manager logInfo "waiting for trust-manager-tls secret" for ((i=0; i/dev/null 2>&1; then logDebug "trust-manager-tls secret exists" break fi sleep 1 done kubectlWrapper wait deployments -n cert-manager --for condition=Available --timeout="${MINIZITI_TIMEOUT_SECS}s" trust-manager >&3 # Wait for trust-manager pods to be ready (not just deployment available) logInfo "waiting for trust-manager pods to be ready" kubectlWrapper wait pods -n cert-manager -l app.kubernetes.io/name=trust-manager --for condition=Ready --timeout="${MINIZITI_TIMEOUT_SECS}s" >&3 # Wait for trust-manager webhook endpoint to actually be responsive # Pod Ready doesn't guarantee the webhook service endpoint is registered and accepting connections logInfo "waiting for trust-manager webhook to be responsive" for ((i=0; i/dev/null || true) if [[ -n "${endpoints}" ]]; then logDebug "trust-manager endpoints registered: ${endpoints}" # Give the webhook a moment to fully initialize after endpoint registration sleep 3 break fi sleep 1 done # ## Ensure OpenZiti Controller is Upgraded and Ready # # Capture the current controller pod UID (if any) so we can detect whether # the helm upgrade causes a pod replacement. When the controller restarts it # generates a new ephemeral OIDC JWT signing key; the router must also be # restarted to re-fetch the JWKS or it will reject JWTs signed by the new key. local ctrl_pod_uid_before ctrl_pod_uid_before=$(kubectlWrapper get pods \ --selector app.kubernetes.io/component=ziti-controller \ --namespace "${ZITI_NAMESPACE}" \ --output jsonpath='{.items[0].metadata.uid}' 2>/dev/null || true) logInfo "installing openziti controller chart" if (( ZITI_CHARTS_ALT )) && [[ -s "${ZITI_CHARTS_REF}/ziti-controller/Chart.lock" ]]; then helmWrapper dependency build "${ZITI_CHARTS_REF}/ziti-controller" >&3 fi local -a _controller_cmd=(upgrade --install "ziti-controller1" "${ZITI_CHARTS_REF}/ziti-controller" --namespace "${ZITI_NAMESPACE}" --create-namespace --set clientApi.advertisedHost="${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" --set cluster.mode=cluster-init --set "cluster.trustDomain=${MINIZITI_INGRESS_ZONE}" --set cluster.nodeName=ziti-controller1 --values "${ZITI_CHARTS_URL}/ziti-controller/values-traefik-ingressroutetcp.yaml" ) if (( HELM_INCLUDE_DEVEL )) && (( ! ZITI_CHARTS_ALT )); then _controller_cmd+=(--devel) fi if [[ -n "${EXTRA_VALUES_DIR:-}" && -s "${EXTRA_VALUES_DIR}/ziti-controller.yaml" ]]; then _controller_cmd+=(--values "${EXTRA_VALUES_DIR}/ziti-controller.yaml") fi helmWrapper "${_controller_cmd[@]}" >&3 local web_identity_cert_name web_identity_cert_name="ziti-controller1-web-identity-cert" logInfo "waiting for cert-manager certificate '${web_identity_cert_name}' to be ready" for ((i=0; i/dev/null 2>&1; then logDebug "certificate '${web_identity_cert_name}' exists" break fi sleep 1 done kubectlWrapper wait certificate.cert-manager.io "${web_identity_cert_name}" \ --namespace "${ZITI_NAMESPACE}" \ --for condition=Ready=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 local web_identity_secret_name="ziti-controller1-web-identity-secret" logDebug "setting default namespace '${ZITI_NAMESPACE}' in kubeconfig context '${MINIKUBE_PROFILE}'" kubectlWrapper config set-context "${MINIKUBE_PROFILE}" \ --namespace "${ZITI_NAMESPACE}" >&3 logInfo "waiting for ziti-controller1 to be ready" kubectlWrapper wait deployments ziti-controller1 \ --namespace "${ZITI_NAMESPACE}" \ --for condition=Available=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 # Detect whether the controller pod was replaced during the helm upgrade. local ctrl_pod_uid_after ctrl_restarted=0 ctrl_pod_uid_after=$(kubectlWrapper get pods \ --selector app.kubernetes.io/component=ziti-controller \ --namespace "${ZITI_NAMESPACE}" \ --output jsonpath='{.items[0].metadata.uid}' 2>/dev/null || true) if [[ -n "${ctrl_pod_uid_before}" && "${ctrl_pod_uid_before}" != "${ctrl_pod_uid_after}" ]]; then ctrl_restarted=1 logInfo "controller pod was replaced during upgrade (OIDC signing key regenerated)" fi if (( CHECK_CERT_SUBJECT )); then checkCommand openssl checkCommand base64 local cert_subject if cert_subject=$(kubectlWrapper get secret "${web_identity_secret_name}" --namespace "${ZITI_NAMESPACE}" -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null); then if [[ ! "${cert_subject}" == *"OpenZiti Community"* ]]; then logError "controller server certificate missing expected subject x509 property 'OpenZiti Community': ${cert_subject}" exit 1 fi else logWarn "failed to inspect certificate subject for ${web_identity_secret_name}" fi logInfo "inspecting hosted controller server certificate subject" local cert_subject_hosted if cert_subject_hosted=$(echo | openssl s_client -showcerts -connect "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}:443" 2>/dev/null | openssl x509 -noout -subject 2>/dev/null); then if [[ ! "${cert_subject_hosted}" == *"OpenZiti Community"* ]]; then logError "hosted controller server certificate missing expected subject x509 property 'OpenZiti Community': ${cert_subject_hosted}" exit 1 fi else logWarn "failed to inspect hosted certificate subject for ${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" fi fi # ## Ensure Minikube Tunnel is Running on macOS and WSL # # wait to probe for the minikube tunnel until after controller deployment so there's at least one # ingress causing minikube to immediately prompt for sudo password if (( DEBUG_MINIKUBE_TUNNEL )) || [[ "${DETECTED_OS}" =~ Windows|macOS ]]; then logDebug "detected OS is ${DETECTED_OS}, probing for minikube tunnel" if ! pgrep -f "minikube --profile ${MINIKUBE_PROFILE} tunnel" >&3; then echo -e "ERROR: ${DETECTED_OS} OS requires a running minikube tunnel for ingresses."\ " In another terminal, run the following command. Then re-run this script."\ "\n\n\tminikube --profile ${MINIKUBE_PROFILE} tunnel\n" >&2 exit 1 else logDebug "minikube tunnel is running" fi # recommend /etc/hosts change unless dns is configured to reach the minikube node IP checkDns "127.0.0.1" else checkDns "$MINIKUBE_NODE_EXTERNAL" fi # ## Ensure Cluster DNS is Resolving ${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE} # if ! testClusterDns "${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}" 2>/dev/null; then logDebug "initial cluster dns test failed, doing cluster dns setup" # xargs trims whitespace because minikube ssh returns a stray trailing '\r' after remote command output logDebug "probing minikube node for internal host record" MINIKUBE_NODE_INTERNAL=$(minikube --profile "${MINIKUBE_PROFILE}" ssh 'grep host.minikube.internal /etc/hosts') if [[ -n "${MINIKUBE_NODE_INTERNAL:-}" ]]; then # strip surrounding whitespace MINIKUBE_NODE_INTERNAL=$(xargs <<< "${MINIKUBE_NODE_INTERNAL}") logDebug "the minikube internal host record is \"${MINIKUBE_NODE_INTERNAL}\"" else logError "failed to find minikube internal IP" exit 1 fi (( MINIZITI_HOSTS )) && { logDebug "patching coredns configmap with *.${MINIZITI_INGRESS_ZONE} forwarder to minikube node external IP" kubectlWrapper patch configmap "coredns" \ --namespace kube-system \ --patch " data: Corefile: | .:53 { log errors health { lameduck 5s } ready kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure fallthrough in-addr.arpa ip6.arpa ttl 30 } prometheus :9153 hosts { ${MINIKUBE_NODE_INTERNAL} fallthrough } forward . /etc/resolv.conf { max_concurrent 1000 } cache 30 loop reload loadbalance } ${MINIZITI_INGRESS_ZONE}:53 { errors cache 30 forward . ${MINIKUBE_NODE_EXTERNAL} } " >&3 logDebug "deleting coredns pod so a new one will have modified Corefile" kubectlWrapper delete pods \ --context "$MINIKUBE_PROFILE" \ --namespace kube-system \ --selector k8s-app=kube-dns >&3 logDebug "waiting for cluster dns to be ready" kubectlWrapper wait deployments "coredns" \ --namespace kube-system \ --for condition=Available=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 } # perform a DNS query in a pod so we know wildcard zone forwarding is working inside the cluster testClusterDns "${MINIKUBE_NODE_EXTERNAL}" fi logInfo "Applying $MINIZITI_CONFIGMAP configmap" cat <&3 apiVersion: v1 kind: ConfigMap metadata: name: "$MINIZITI_CONFIGMAP" namespace: "$ZITI_NAMESPACE" data: ingress-zone: "$MINIZITI_INGRESS_ZONE" EOF logInfo "Setting default ziti identity to: $MINIKUBE_PROFILE" zitiWrapper edge use "$MINIKUBE_PROFILE" >&3 ROUTER_NAME="${MINIKUBE_PROFILE}-router" ROUTER_OTT="$IDENTITIES_DIR/$ROUTER_NAME.jwt" if zitiWrapper edge list edge-routers "name=\"$ROUTER_NAME\"" \ | grep -qF "$ROUTER_NAME"; then logDebug "updating $ROUTER_NAME" zitiWrapper edge update edge-router "$ROUTER_NAME" \ --role-attributes "public-routers" >&3 else logDebug "creating $ROUTER_NAME" zitiWrapper edge create edge-router "$ROUTER_NAME" \ --role-attributes "public-routers" \ --tunneler-enabled >&3 zitiWrapper edge list edge-routers \ "name=\"$ROUTER_NAME\"" \ --output-json \ | jq --exit-status --raw-output '.data[0].enrollmentJwt' > "$ROUTER_OTT" fi logDebug "installing router chart as 'ziti-router'" if (( ZITI_CHARTS_ALT )) && [[ -s "${ZITI_CHARTS_REF}/ziti-router/Chart.lock" ]]; then helmWrapper dependency build "${ZITI_CHARTS_REF}/ziti-router" >&3 fi local -a _router_cmd=(upgrade --install "ziti-router" "${ZITI_CHARTS_REF}/ziti-router" --namespace "${ZITI_NAMESPACE}" --set-file enrollmentJwt="$ROUTER_OTT" --set edge.advertisedHost="${ROUTER_NAME}.${MINIZITI_INGRESS_ZONE}" --set ctrl.endpoint="${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}:443" --set "tunnel.mode=host" --values "${ZITI_CHARTS_URL}/ziti-router/values-traefik-ingressroutetcp.yaml" --set linkListeners.transport.service.enabled=false ) if (( HELM_INCLUDE_DEVEL )) && (( ! ZITI_CHARTS_ALT )); then _router_cmd+=(--devel) fi if [[ -n "${EXTRA_VALUES_DIR:-}" && -s "${EXTRA_VALUES_DIR}/ziti-router.yaml" ]]; then _router_cmd+=(--values "${EXTRA_VALUES_DIR}/ziti-router.yaml") fi helmWrapper "${_router_cmd[@]}" >&3 logInfo "waiting for ziti-router to be ready" kubectlWrapper wait deployments "ziti-router" \ --namespace "${ZITI_NAMESPACE}" \ --for condition=Available=True \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 # If the controller was replaced during its helm upgrade, the router must be # restarted so it re-fetches the controller's JWKS endpoint and picks up the # new OIDC signing key. Without this, the router rejects JWTs signed by the # new controller with "public key not found". if (( ctrl_restarted )); then logInfo "restarting ziti-router to refresh OIDC JWKS cache after controller restart" kubectlWrapper rollout restart deployment/ziti-router \ --namespace "${ZITI_NAMESPACE}" >&3 kubectlWrapper rollout status deployment/ziti-router \ --namespace "${ZITI_NAMESPACE}" \ --timeout "${MINIZITI_TIMEOUT_SECS}s" >&3 fi logDebug "probing ${ROUTER_NAME} for online status" if zitiWrapper edge list edge-routers "name=\"$ROUTER_NAME\"" \ | awk -v rname="${ROUTER_NAME}" '$0 ~ rname {print $6}' \ | grep -q true; then logInfo "${ROUTER_NAME} is online" else logError "${ROUTER_NAME} is offline" exit 1 fi logDebug "setting default namespace to '${ZITI_NAMESPACE}' in kubeconfig context '${MINIKUBE_PROFILE}'" kubectlWrapper config set-context "${MINIKUBE_PROFILE}" \ --namespace "${ZITI_NAMESPACE}" >&3 # ## Ensure OpenZiti Identities and Services are Created # CLIENT_NAME="${MINIKUBE_PROFILE}-client" CLIENT_OTT="$IDENTITIES_DIR/$CLIENT_NAME.jwt" if ! zitiWrapper edge list identities "name=\"$CLIENT_NAME\"" --csv \ | grep -q "$CLIENT_NAME"; then logDebug "creating identity $CLIENT_NAME" zitiWrapper edge create identity "$CLIENT_NAME" \ --role-attributes httpbin-clients >&3 zitiWrapper edge list identities "name=\"$CLIENT_NAME\"" \ --output-json \ | jq --exit-status --raw-output '.data[0].enrollment.ott.jwt' > "$CLIENT_OTT" else logDebug "ignoring identity $CLIENT_NAME" fi HTTPBIN_NAME="httpbin-host" HTTPBIN_OTT="$IDENTITIES_DIR/$HTTPBIN_NAME.jwt" if ! zitiWrapper edge list identities "name=\"$HTTPBIN_NAME\"" --csv \ | grep -q "$HTTPBIN_NAME"; then logDebug "creating identity $HTTPBIN_NAME" zitiWrapper edge create identity "$HTTPBIN_NAME" \ --role-attributes httpbin-hosts >&3 zitiWrapper edge list identities "name=\"$HTTPBIN_NAME\"" \ --output-json \ | jq --exit-status --raw-output '.data[0].enrollment.ott.jwt' > "$HTTPBIN_OTT" else logDebug "ignoring identity $HTTPBIN_NAME" fi if ! zitiWrapper edge list configs 'name="httpbin-intercept-config"' --csv \ | grep -q "httpbin-intercept-config"; then logDebug "creating config httpbin-intercept-config" zitiWrapper edge create config "httpbin-intercept-config" intercept.v1 \ '{"protocols":["tcp"],"addresses":["httpbin.'"${MINIZITI_INTERCEPT_ZONE}"'"], "portRanges":[{"low":80, "high":80}]}' >&3 else logDebug "ignoring config httpbin-intercept-config" fi if ! zitiWrapper edge list configs 'name="httpbin-host-config"' --csv \ | grep -q "httpbin-host-config"; then logDebug "creating config httpbin-host-config" zitiWrapper edge create config "httpbin-host-config" host.v1 \ '{"protocol":"tcp", "address":"httpbin","port":8080}' >&3 else logDebug "ignoring config httpbin-host-config" fi if ! zitiWrapper edge list services 'name="httpbin-service"' --csv \ | grep -q "httpbin-service"; then logDebug "creating service httpbin-service" zitiWrapper edge create service "httpbin-service" \ --configs httpbin-intercept-config,httpbin-host-config >&3 else logDebug "ignoring service httpbin-service" fi if ! zitiWrapper edge list service-policies 'name="httpbin-bind-policy"' --csv \ | grep -q "httpbin-bind-policy"; then logDebug "creating service-policy httpbin-bind-policy" zitiWrapper edge create service-policy "httpbin-bind-policy" Bind --semantic "AnyOf" \ --service-roles '@httpbin-service' --identity-roles '#httpbin-hosts' >&3 else logDebug "ignoring service-policy httpbin-bind-policy" fi if ! zitiWrapper edge list service-policies 'name="httpbin-dial-policy"' --csv \ | grep -q "httpbin-dial-policy"; then logDebug "creating service-policy httpbin-dial-policy" zitiWrapper edge create service-policy "httpbin-dial-policy" Dial --semantic "AnyOf" \ --service-roles '@httpbin-service' --identity-roles '#httpbin-clients' >&3 else logDebug "ignoring service-policy httpbin-dial-policy" fi if ! zitiWrapper edge list edge-router-policies 'name="public-routers"' --csv \ | grep -q "public-routers"; then logDebug "creating edge-router-policy public-routers" zitiWrapper edge create edge-router-policy "public-routers" \ --edge-router-roles '#public-routers' --identity-roles '#all' >&3 else logDebug "ignoring edge-router-policy public-routers" fi if ! zitiWrapper edge list service-edge-router-policies 'name="public-routers"' --csv \ | grep -q "public-routers"; then logDebug "creating service-edge-router-policy public-routers" zitiWrapper edge create service-edge-router-policy "public-routers" \ --edge-router-roles '#all' --service-roles '#all' >&3 else logDebug "ignoring service-edge-router-policy public-routers" fi if [[ -s "$HTTPBIN_OTT" ]]; then logDebug "installing httpbin chart as 'miniziti-httpbin'" _httpbin_cmd=( install "miniziti-httpbin" "${ZITI_CHARTS_REF}/httpbin" --set-file zitiEnrollment="$HTTPBIN_OTT" --set zitiServiceName=httpbin-service ) if (( HELM_INCLUDE_DEVEL )) && (( ! ZITI_CHARTS_ALT )); then _httpbin_cmd+=(--devel) fi if [[ -n "${EXTRA_VALUES_DIR:-}" && -s "${EXTRA_VALUES_DIR}/httpbin.yaml" ]]; then _httpbin_cmd+=(--values "${EXTRA_VALUES_DIR}/httpbin.yaml") fi helmWrapper "${_httpbin_cmd[@]}" >&3 rm -f "$HTTPBIN_OTT" logDebug "deleted $HTTPBIN_OTT after installing successfully with miniziti-httpbin chart" else logDebug "upgrading httpbin chart as 'miniziti-httpbin'" _httpbin_cmd=( upgrade "miniziti-httpbin" "${ZITI_CHARTS_REF}/httpbin" --set zitiServiceName=httpbin-service ) if (( HELM_INCLUDE_DEVEL )) && (( ! ZITI_CHARTS_ALT )); then _httpbin_cmd+=(--devel) fi if [[ -n "${EXTRA_VALUES_DIR:-}" && -s "${EXTRA_VALUES_DIR}/httpbin.yaml" ]]; then _httpbin_cmd+=(--values "${EXTRA_VALUES_DIR}/httpbin.yaml") fi helmWrapper "${_httpbin_cmd[@]}" >&3 fi echo -e "\n\n" logInfo "Your OpenZiti Console is here: https://${ZITI_NETWORK_NAME}.${MINIZITI_INGRESS_ZONE}/zac/" showAdminCreds echo -e "\n\n" logInfo "r'Success! Remember to add your edge client identity '$(pathToNative "$CLIENT_OTT")' in your tunneler, e.g. Ziti Desktop Edge.'" } main "$@"