Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make relay init container script more resilient: #112

Merged
merged 1 commit into from
Jul 19, 2024
Merged

Make relay init container script more resilient: #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 116 additions & 0 deletions tinkerbell/stack/templates/init_configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{{- if and .Values.stack.enabled .Values.stack.relay.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: host-interface-script
data:
host_interface.sh: |
#!/usr/bin/env sh

# This script allows us to listen and respond to DHCP requests on a host network interface and interact with Smee properly.
# This is used instead of `hostNetwork: true` because the dhcp relay requires clear paths for listening for broadcast traffic
# and sending/receiving unicast traffic to/from Smee.

set -xeuo pipefail

function usage() {
echo "Usage: $0 [OPTION]..."
echo "Init script for setting up a network interface to listen and respond to DHCP requests from the Host and move it into a container."
echo
echo "Options:"
echo " -s, --src Source interface for listening and responding to DHCP requests (default: default gateway interface)"
echo " -t, --type Create the interface of type, must be either ipvlan or macvlan (default: macvlan)"
echo " -c, --clean Clean up any interfaces created"
echo " -h, --help Display this help and exit"
}

function binary_exists() {
command -v "$1" >/dev/null 2>&1
}

function main() {
local src_interface="$1"
local interface_type="$2"
local interface_mode="$3"
local interface_name="${interface_type}0"

# Preparation
# Delete existing interfaces in the container
ip link del macvlan0 || true
ip link del ipvlan0 || true
ip link del ipvlan0-wa || true
# Delete existing interfaces in the host namespace
nsenter -t1 -n ip link del macvlan0 || true
nsenter -t1 -n ip link del ipvlan0 || true
nsenter -t1 -n ip link del ipvlan0-wa || true
# Create the interface
nsenter -t1 -n ip link add "${interface_name}" link "${src_interface}" type "${interface_type}" mode "${interface_mode}" || true
# Move the interface into the Pod container
pid=$(echo $$)
nsenter -t1 -n ip link set "${interface_name}" netns ${pid} || nsenter -t1 -n ip link delete "${interface_name}"
# Bring up the interface
ip link set dev "${interface_name}" up
# Set the IP address
ip addr add 127.1.1.1/32 dev "${interface_name}" noprefixroute || true
# Run ipvlan workaround
# There is an issue with ipvlan interfaces. They do not start receiving broadcast packets after creation.
# This is a workaround to get broadcast packets flowing.
# TODO(jacobweinstock): Investigate this deeper and see if this is a kernel bug.
if [[ "${interface_type}" == "ipvlan" ]]; then
nsenter -t1 -n nmap --script broadcast-dhcp-discover
nsenter -t1 -n ip link add "${interface_name}"-wa link "${src_interface}" type "${interface_type}" mode "${interface_mode}" bridge || true
nsenter -t1 -n nmap --script broadcast-dhcp-discover
fi
}

src_interface=$(nsenter -t1 -n ip route | awk '/default/ {print $5}' | head -n1)
interface_type="macvlan"
interface_mode="bridge"
clean=false
args=$(getopt -a -o s:t:ch --long src:,type:,clean,help -- "$@")
if [[ $? -gt 0 ]]; then
usage
fi

eval set -- ${args}
while :
do
case $1 in
-s | --src)
if [[ ! -z $2 ]]; then
src_interface=$2
fi
shift 2 ;;
-t | --type)
if [[ "$2" == "ipvlan" ]]; then
interface_type="ipvlan"
interface_mode="l2"
fi
shift 2 ;;
-c | --clean)
clean=true
shift ;;
-h | --help)
usage
exit 1
shift ;;
# -- means the end of the arguments; drop this, and break out of the while loop
--) shift; break ;;
*) >&2 echo Unsupported option: $1
usage ;;
esac
done

if "${clean}"; then
# Delete existing interfaces in the container
ip link del macvlan0 || true
ip link del ipvlan0 || true
ip link del ipvlan0-wa || true
# Delete existing interfaces in the host namespace
nsenter -t1 -n ip link del macvlan0 || true
nsenter -t1 -n ip link del ipvlan0 || true
nsenter -t1 -n ip link del ipvlan0-wa || true
exit 0
fi
main "${src_interface}" "${interface_type}" "${interface_mode}"
{{- end }}
46 changes: 12 additions & 34 deletions tinkerbell/stack/templates/nginx.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ spec:
{{- toYaml . | nindent 6 }}
{{- end }}
replicas: 1
strategy:
type: {{ .Values.stack.deployment.strategy.type }}
template:
metadata:
annotations:
Expand Down Expand Up @@ -113,6 +115,12 @@ spec:
items:
- key: nginx.conf
path: nginx.conf.template
{{- if and .Values.stack.relay.enabled $listenBroadcast }}
- name: script
configMap:
name: host-interface-script
defaultMode: 0500
{{- end }}
{{- if .Values.stack.hook.enabled }}
- name: hook-artifacts
hostPath:
Expand All @@ -122,41 +130,11 @@ spec:
{{- if and .Values.stack.relay.enabled $listenBroadcast }}
initContainers:
- name: relay-broadcast-interface
command:
- /bin/sh
- -c
- |
# This script allows us to listen and respond to DHCP requests on a host network interface and interact with Smee properly.
# This is used instead of `hostNetwork: true` because the dhcp relay requires clear paths for listening for broadcast traffic
# and sending/receiving unicast traffic to/from Smee.
set -xe
# if sourceInterface is not set use the interface from the default route
srcInterface="{{ $sourceInterface }}"
if [ -z "$srcInterface" ]; then
srcInterface=$(nsenter -t1 -n ip route | awk '/default/ {print $5}' | head -n1)
fi
# Create the interface. TODO: If this fails, try again with a different name?
{{- if eq $dhcpInterfaceType "ipvlan" }}
nsenter -t1 -n ip link add {{ $dhcpInterfaceName }} link ${srcInterface} type ipvlan mode l2
{{- else }}
nsenter -t1 -n ip link add {{ $dhcpInterfaceName }} link ${srcInterface} type macvlan mode bridge
{{- end }}
# Move the interface into the POD.
pid=$(echo $$)
nsenter -t1 -n ip link set {{ $dhcpInterfaceName }} netns ${pid} || nsenter -t1 -n ip link delete {{ $dhcpInterfaceName }}
# Set the interface up
ip link set {{ $dhcpInterfaceName }} up
# Set the IP address
ip addr add 127.1.1.1/32 dev {{ $dhcpInterfaceName }} noprefixroute || true
{{- if eq $dhcpInterfaceType "ipvlan" }}
# There is an issue with ipvlan interfaces. They do not start receiving broadcast packets after creation.
# This is a workaround to get broadcast packets flowing.
# TODO(jacobweinstock): Investigate this deeper and see if this is a kernel bug.
nsenter -t1 -n ip link del {{ $dhcpInterfaceName }}-wa || true
nsenter -t1 -n nmap --script broadcast-dhcp-discover
nsenter -t1 -n ip link add {{ $dhcpInterfaceName }}-wa link ${srcInterface} type ipvlan mode l2 bridge || true
{{- end }}
image: {{ .Values.stack.relay.initImage }}
command: ["/script/host_interface.sh", "-s", "{{ $sourceInterface }}", "-t", "{{ $dhcpInterfaceType }}"]
volumeMounts:
- name: script
mountPath: "/script"
securityContext:
privileged: true
{{- end }}
Expand Down
3 changes: 3 additions & 0 deletions tinkerbell/stack/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ stack:
selector:
app: tink-stack
nodeSelector: {}
deployment:
strategy:
type: RollingUpdate
# stack needs to resolve DNS names in the cluster (in .svc.clusterDomain)
clusterDomain: cluster.local
# &publicIP is a YAML anchor. It allows us to define a value once and reference it multiple times.
Expand Down
Loading