steven/kubernetes-bootstrapping
steven
/
kubernetes-bootstrapping
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

a "lighter" footprint for node provisioning

This commit is contained in:
Steven Polley 2023-12-18 18:18:20 -07:00
parent 9105e8ee12
commit a4ea60d253
9 changed files with 36 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
00-provisioning/README.md
View File

@ -2,24 +2,48 @@
Adding new nodes
1. Create a boot and full butane yaml file for the new host in the butane directory
2. Generate ignition JSON files from butane YAML files using the below snippet
3. Check the resulting .json files into version control
1. Call lighter and pass the new node's name to generate ignition files for the node, see below snippet.
2. Commit and check the resulting ignition/*.json files into version control at deadbeef.codes, they need to be present before booting the node.
```bash
# Be sure to run from 00-provisioning directory
cd 00-provisioning
# Loop through all butane files and generate ignition files
for i in butane/*.yaml; do
FILENAME=$( echo $i | cut -c 8- | head -c -6)
echo running butane on $FILENAME
butane butane/$FILENAME.yaml > ignition/$FILENAME.json
done
# Templating for Butane files to replace hostname with name passed
# to lighter, then it calls butane to generate ignition files
lighter() {
if [ -z "$1" ]; then
echo "error: lighter() called without specifying a VM name"
echo "Usage: lighter() <name>"
return
fi
# Create temporary working copies
cp butane/boot.yaml butane/boot~.yaml
cp butane/full.yaml butane/full~.yaml
# Replace hostname token with name provided
hostnameToken="{{HOSTNAME}}"
sed -i -e "s/$hostnameToken/$1/g" butane/boot~.yaml
sed -i -e "s/$hostnameToken/$1/g" butane/full~.yaml
# Butane transpile to ignition files
butane butane/boot~.yaml > ignition/$1-boot.json
butane butane/full~.yaml > ignition/$1-full.json
# Cleanup mess
rm -f butane/*~.yaml
}
lighter kube-control01
lighter kube-node01
lighter kube-node02
lighter kube-node03
```
After you've checked the ignition files into version control, provision the server, either on baremetal or VM - example with virtualbox.
After you've checked the ignition files into version control, provision the server, either on baremetal or VM - example with virtualbox. Use a method to point it to the boot ignition file, in virtual box guest properties can be used.
```bash

2
00-provisioning/butane/kube-node02-boot.yaml → 00-provisioning/butane/boot.yaml
View File

@ -3,4 +3,4 @@ version: 1.5.0
ignition:
config:
replace:
source: https://deadbeef.codes/steven/deadbeef.codes-k8s/raw/branch/main/00-provisioning/ignition/kube-node02-full.json
source: https://deadbeef.codes/steven/deadbeef.codes-k8s/raw/branch/main/00-provisioning/ignition/{{HOSTNAME}}-full.json

2
00-provisioning/butane/kube-node02-full.yaml → 00-provisioning/butane/full.yaml
View File

@ -8,7 +8,7 @@ storage:
overwrite: true
contents:
inline: |
kube-node02
{{HOSTNAME}}
# CRI-O DNF module
- path: /etc/dnf/modules.d/cri-o.module
mode: 0644

6
00-provisioning/butane/kube-control01-boot.yaml
View File

@ -1,6 +0,0 @@
variant: fcos
version: 1.5.0
ignition:
config:
replace:
source: https://deadbeef.codes/steven/deadbeef.codes-k8s/raw/branch/main/00-provisioning/ignition/kube-control01-full.json

114
00-provisioning/butane/kube-control01-full.yaml
View File

@ -1,114 +0,0 @@
variant: fcos
version: 1.5.0
storage:
files:
# Hostname
- path: /etc/hostname
mode: 420
overwrite: true
contents:
inline: |
kube-control01
# CRI-O DNF module
- path: /etc/dnf/modules.d/cri-o.module
mode: 0644
overwrite: true
contents:
inline: |
[cri-o]
name=cri-o
stream=1.29
profiles=
state=enabled
# YUM repository for kubeadm, kubelet and kubectl
- path: /etc/yum.repos.d/kubernetes.repo
mode: 0644
overwrite: true
contents:
inline: |
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
# configuring automatic loading of br_netfilter on startup
- path: /etc/modules-load.d/br_netfilter.conf
mode: 0644
overwrite: true
contents:
inline: br_netfilter
# setting kernel parameters required by kubelet
- path: /etc/sysctl.d/kubernetes.conf
mode: 0644
overwrite: true
contents:
inline: |
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
# script to install kubelet dependencies
- path: /usr/local/bin/install-kubelet.sh
mode: 0744
overwrite: true
contents:
inline: |
#!/bin/bash
rpm-ostree install kubelet kubeadm kubectl cri-o vim
systemctl enable crio kubelet
sleep 1 # systemctl enable and reboot race condition?
systemctl reboot
- path: /home/steven/cluster-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
controllerManager:
extraArgs:
flex-volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
networking:
podSubnet: "10.244.0.0/16"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
- path: /home/steven/join-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
passwd: # setting login credentials
users:
- name: steven
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAsPq55j525p1ntk37oeel83i6FVm9+ptWk/4csqZivKRrtfhSebtWj0GHg7mnN+XIvQFEXY1HuPSfAByuwURN7LrBTKYNDHM5VxyHSP3s6Ik9OYngbVDCcDRc94teivYalYPyC5rCfIFYRg+vrxD1Gl7eASpiS7z2YD5w6WSxEnQ9tk+GQgsbRcsDBpKTZL/yHZbNNOamUwv3FNmaDpq7V8d1IrKCQiivYQ5n/sWpQnOzMOnY5i7OKr2G56KbaVIXRe3JKIF7ifAK0aJ5q+45RmwdgVh+SgwIFBzQD4GZJbr8jbvxYO9NjbF9fm7qLYnbHNyT7cDx8ClQqAz/2cL0xw== taterwin10
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCf2HvSx2ls2KhHn3tISbYbx9NpYSKw+ESmOdxscZJuCEMKDEOdBuOJ5E3CpA+A+QiLY+qlXGPOyNKwmjVPFr6TzNwBciehcs3bFKAvar2vrJsQkUXAJiBzJWiQceGwto3zq6nIAO/tx7s3P6KVPuxegGyXAMv/7Fc8cytOk8q05Xt+7hUyz0LbCZ6j66/Qa7c8eJz8Vho1Oe1BpIhhcZbSovZrKgBOhpyIdUtxh/W5KnFsbIq4MPPVCRHN7IVrXcvkPsTQ6OGeJAsqun+zF3KupQs0Xqt157EmOn5D41x5QY7kts11QZiKmeeSFYt2gRaY7VtAlEfar0fgXWOyQ/Uf steven-pixel6a
groups: [ sudo, docker ]
systemd:
units:
- name: install-kubelet.service
enabled: true
contents: |
[Unit]
Before=systemd-user-sessions.service
Wants=network-online.target
After=network-online.target
ConditionPathExists=!/var/lib/install-kubelet
[Service]
Type=oneshot
ExecStart=/usr/local/bin/install-kubelet.sh
ExecStartPost=/usr/bin/touch /var/lib/install-kubelet
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target

6
00-provisioning/butane/kube-node01-boot.yaml
View File

@ -1,6 +0,0 @@
variant: fcos
version: 1.5.0
ignition:
config:
replace:
source: https://deadbeef.codes/steven/deadbeef.codes-k8s/raw/branch/main/00-provisioning/ignition/kube-node01-full.json

114
00-provisioning/butane/kube-node01-full.yaml
View File

@ -1,114 +0,0 @@
variant: fcos
version: 1.5.0
storage:
files:
# Hostname
- path: /etc/hostname
mode: 420
overwrite: true
contents:
inline: |
kube-node01
# CRI-O DNF module
- path: /etc/dnf/modules.d/cri-o.module
mode: 0644
overwrite: true
contents:
inline: |
[cri-o]
name=cri-o
stream=1.29
profiles=
state=enabled
# YUM repository for kubeadm, kubelet and kubectl
- path: /etc/yum.repos.d/kubernetes.repo
mode: 0644
overwrite: true
contents:
inline: |
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
# configuring automatic loading of br_netfilter on startup
- path: /etc/modules-load.d/br_netfilter.conf
mode: 0644
overwrite: true
contents:
inline: br_netfilter
# setting kernel parameters required by kubelet
- path: /etc/sysctl.d/kubernetes.conf
mode: 0644
overwrite: true
contents:
inline: |
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
# script to install kubelet dependencies
- path: /usr/local/bin/install-kubelet.sh
mode: 0744
overwrite: true
contents:
inline: |
#!/bin/bash
rpm-ostree install kubelet kubeadm kubectl cri-o vim
systemctl enable crio kubelet
sleep 1 # systemctl enable and reboot race condition?
systemctl reboot
- path: /home/steven/cluster-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
controllerManager:
extraArgs:
flex-volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
networking:
podSubnet: "10.244.0.0/16"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
- path: /home/steven/join-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
passwd: # setting login credentials
users:
- name: steven
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAsPq55j525p1ntk37oeel83i6FVm9+ptWk/4csqZivKRrtfhSebtWj0GHg7mnN+XIvQFEXY1HuPSfAByuwURN7LrBTKYNDHM5VxyHSP3s6Ik9OYngbVDCcDRc94teivYalYPyC5rCfIFYRg+vrxD1Gl7eASpiS7z2YD5w6WSxEnQ9tk+GQgsbRcsDBpKTZL/yHZbNNOamUwv3FNmaDpq7V8d1IrKCQiivYQ5n/sWpQnOzMOnY5i7OKr2G56KbaVIXRe3JKIF7ifAK0aJ5q+45RmwdgVh+SgwIFBzQD4GZJbr8jbvxYO9NjbF9fm7qLYnbHNyT7cDx8ClQqAz/2cL0xw== taterwin10
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCf2HvSx2ls2KhHn3tISbYbx9NpYSKw+ESmOdxscZJuCEMKDEOdBuOJ5E3CpA+A+QiLY+qlXGPOyNKwmjVPFr6TzNwBciehcs3bFKAvar2vrJsQkUXAJiBzJWiQceGwto3zq6nIAO/tx7s3P6KVPuxegGyXAMv/7Fc8cytOk8q05Xt+7hUyz0LbCZ6j66/Qa7c8eJz8Vho1Oe1BpIhhcZbSovZrKgBOhpyIdUtxh/W5KnFsbIq4MPPVCRHN7IVrXcvkPsTQ6OGeJAsqun+zF3KupQs0Xqt157EmOn5D41x5QY7kts11QZiKmeeSFYt2gRaY7VtAlEfar0fgXWOyQ/Uf steven-pixel6a
groups: [ sudo, docker ]
systemd:
units:
- name: install-kubelet.service
enabled: true
contents: |
[Unit]
Before=systemd-user-sessions.service
Wants=network-online.target
After=network-online.target
ConditionPathExists=!/var/lib/install-kubelet
[Service]
Type=oneshot
ExecStart=/usr/local/bin/install-kubelet.sh
ExecStartPost=/usr/bin/touch /var/lib/install-kubelet
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target

6
00-provisioning/butane/kube-node03-boot.yaml
View File

@ -1,6 +0,0 @@
variant: fcos
version: 1.5.0
ignition:
config:
replace:
source: https://deadbeef.codes/steven/deadbeef.codes-k8s/raw/branch/main/00-provisioning/ignition/kube-node03-full.json

114
00-provisioning/butane/kube-node03-full.yaml
View File

@ -1,114 +0,0 @@
variant: fcos
version: 1.5.0
storage:
files:
# Hostname
- path: /etc/hostname
mode: 420
overwrite: true
contents:
inline: |
kube-node03
# CRI-O DNF module
- path: /etc/dnf/modules.d/cri-o.module
mode: 0644
overwrite: true
contents:
inline: |
[cri-o]
name=cri-o
stream=1.29
profiles=
state=enabled
# YUM repository for kubeadm, kubelet and kubectl
- path: /etc/yum.repos.d/kubernetes.repo
mode: 0644
overwrite: true
contents:
inline: |
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
# configuring automatic loading of br_netfilter on startup
- path: /etc/modules-load.d/br_netfilter.conf
mode: 0644
overwrite: true
contents:
inline: br_netfilter
# setting kernel parameters required by kubelet
- path: /etc/sysctl.d/kubernetes.conf
mode: 0644
overwrite: true
contents:
inline: |
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
# script to install kubelet dependencies
- path: /usr/local/bin/install-kubelet.sh
mode: 0744
overwrite: true
contents:
inline: |
#!/bin/bash
rpm-ostree install kubelet kubeadm kubectl cri-o vim
systemctl enable crio kubelet
sleep 1 # systemctl enable and reboot race condition?
systemctl reboot
- path: /home/steven/cluster-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
controllerManager:
extraArgs:
flex-volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
networking:
podSubnet: "10.244.0.0/16"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
- path: /home/steven/join-config.yaml
mode: 0600
overwrite: true
contents:
inline: |
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
volume-plugin-dir: "/var/lib/kubelet/volumeplugins/nodeagent~uds"
passwd: # setting login credentials
users:
- name: steven
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAsPq55j525p1ntk37oeel83i6FVm9+ptWk/4csqZivKRrtfhSebtWj0GHg7mnN+XIvQFEXY1HuPSfAByuwURN7LrBTKYNDHM5VxyHSP3s6Ik9OYngbVDCcDRc94teivYalYPyC5rCfIFYRg+vrxD1Gl7eASpiS7z2YD5w6WSxEnQ9tk+GQgsbRcsDBpKTZL/yHZbNNOamUwv3FNmaDpq7V8d1IrKCQiivYQ5n/sWpQnOzMOnY5i7OKr2G56KbaVIXRe3JKIF7ifAK0aJ5q+45RmwdgVh+SgwIFBzQD4GZJbr8jbvxYO9NjbF9fm7qLYnbHNyT7cDx8ClQqAz/2cL0xw== taterwin10
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCf2HvSx2ls2KhHn3tISbYbx9NpYSKw+ESmOdxscZJuCEMKDEOdBuOJ5E3CpA+A+QiLY+qlXGPOyNKwmjVPFr6TzNwBciehcs3bFKAvar2vrJsQkUXAJiBzJWiQceGwto3zq6nIAO/tx7s3P6KVPuxegGyXAMv/7Fc8cytOk8q05Xt+7hUyz0LbCZ6j66/Qa7c8eJz8Vho1Oe1BpIhhcZbSovZrKgBOhpyIdUtxh/W5KnFsbIq4MPPVCRHN7IVrXcvkPsTQ6OGeJAsqun+zF3KupQs0Xqt157EmOn5D41x5QY7kts11QZiKmeeSFYt2gRaY7VtAlEfar0fgXWOyQ/Uf steven-pixel6a
groups: [ sudo, docker ]
systemd:
units:
- name: install-kubelet.service
enabled: true
contents: |
[Unit]
Before=systemd-user-sessions.service
Wants=network-online.target
After=network-online.target
ConditionPathExists=!/var/lib/install-kubelet
[Service]
Type=oneshot
ExecStart=/usr/local/bin/install-kubelet.sh
ExecStartPost=/usr/bin/touch /var/lib/install-kubelet
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target