blob: 1766746b850124b34b00dc39a619fe73e2fa2b9f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
# Rinse Lather Repeat. K8s on trixie.
Found a decent example of how to run up kubernetes by someone else trying to get a CKA.
https://max-pfeiffer.github.io/installing-kubernetes-on-debian-13-trixie.html
Unfortunately there was a little piece missing '--set cni.binPath=/usr/lib/cni' which pretty much meant reinstalling everything about 4 times. On the third iteration I started seeing the point of getting cloud-init (or ansible, or puppet) involved in the bass systems.
## Attempt #1/2 (do it by hand and then batch a bunch)
```sh
incus init trixie-vm-cloud -c limits.cpu=6 -c limits.memory=12GiB -d root,size=24GiB --vm gru -p default -p merlot -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 192.168.129.130/17
gateway4: 192.168.128.1
nameservers:
addresses:
- 192.168.128.1
EOF
)"
incus start gru --console
incus init trixie-vm-cloud -c limits.cpu=6 -c limits.memory=12GiB -d root,size=24GiB --vm minion1 -p default -p merlot -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 192.168.129.131/17
gateway4: 192.168.128.1
nameservers:
addresses:
- 192.168.128.1
EOF
)"
incus start minion1
incus init trixie-vm-cloud -c limits.cpu=6 -c limits.memory=12GiB -d root,size=24GiB --vm minion2 -p default -p merlot -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 192.168.129.132/17
gateway4: 192.168.128.1
nameservers:
addresses:
- 192.168.128.1
EOF
)"
incus init trixie-vm-cloud -c limits.cpu=6 -c limits.memory=12GiB -d root,size=24GiB --vm minion3 -p default -p merlot -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 192.168.129.133/17
gateway4: 192.168.128.1
nameservers:
addresses:
- 192.168.128.1
EOF
)"
for h in gru minion1 minion2 minion3 ; do echo $h; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- modprobe overlay; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- modprobe overlay; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- modprobe br-netfilter; done
cat <<EOF | tee k8s.sysctl.d.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
for h in gru minion1 minion2 minion3 ; do incus file push k8s.sysctl.d.conf $h/etc/sysctl.d/k8s.conf; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- sysctl --system; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- apt update; done
; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- install -m 0755 -d /etc/apt/keyrings; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- chmod a+r /etc/apt/keyrings/docker.asc; done
for h in gru minion1 minion2 minion3 ; do incus file push /etc/apt/sources.list.d/docker.list $h/etc/apt/sources.list.d/docker.list; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- apt update; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- apt install containerd.io; done
containerd config default | tee config.toml
sed -e 's/SystemdCgroup = false/SystemdCgroup = true/g' -i config.toml
apt install containerd
nano /etc/resolv.conf
apt install containerd
containerd config default | tee config.toml
sed -e 's/SystemdCgroup = false/SystemdCgroup = true/g' -i config.toml
for h in gru minion1 minion2 minion3 ; do incus file push config.toml $h/etc/containerd/config.toml; done
for h in gru minion1 minion2 minion3 ; do incus exec $h -- systemctl restart containerd ; done
curl -fsSL https://packages.buildkite.com/helm-linux/helm-debian/gpgkey | gpg --dearmor | tee helm.gpg > /dev/null
for h in gru minion1 minion2 minion3 ; do incus file push helm.gpg $h/usr/share/keyrings/helm.gpg; done
echo "deb [signed-by=/usr/share/keyrings/helm.gpg] https://packages.buildkite.com/helm-linux/helm-debian/any/ any main" | sudo tee helm-stable-debian.list
for h in gru minion1 minion2 minion3 ; do incus file push helm.gpg $h/etc/apt/sources.list.d/helm-stable-debian.list; done
for h in gru minion1 minion2 minion3 minion4; do incus file push helm-stable-debian.list $h/etc/apt/sources.list.d/helm-stable-debian.list; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt update ; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt install helm; done
ls
mkdir cka
mv config.toml helm* cka/
cd cka
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key | gpg --dearmor -o kubernetes-apt-keyring.gpg
for h in gru minion1 minion2 minion3 minion4; do incus file push kubernetes-apt-keyring.gpg $h/etc/apt/keyrings/kubernetes-apt-keyring.gpg; done
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.33/deb/ /'|tee kubernetes.list
for h in gru minion1 minion2 minion3 minion4; do incus file push kubernetes.list $h/etc/apt/sources.list.d/kubernetes.list; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt update; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt dist-upgrade -y; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt install -y kubeadm=1.33.5-1.1 kubelet=1.33.5-1.1 kubectl=1.33.5-1.1; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- apt-mark hold kubelet kubeadm kubectl; done
cat <<EOF | sudo tee k8s.conf
overlay
br_netfilter
EOF
ls
for h in gru minion1 minion2 minion3 minion4; do incus file push k8s.conf $h/etc/sysctl.d/k8s.conf; done
cat <<EOF | tee k8s.sysctl.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- kubeadm config images list; done
incus file pull gru/etc/containerd/config.toml
incus file pull gru/etc/containerd/config.toml .
nano config.toml
for h in gru minion1 minion2 minion3 minion4; do incus file push config.toml $h/etc/containerd/config.toml; done
for h in gru minion1 minion2 minion3 minion4; do incus exec $h -- systemctl restart containerd; done
top
cat /etc/hosts
incus shell minion1
incus shell minion2
incus shell minion3
incus shell minion4
for h in gru minion1 minion2 minion3 minion4; do incus file push k8s.conf $h/etc/sysctl.d/k8s.conf; done
for h in gru minion1 minion2 minion3 minion4; do incus file push /etc/apt/sources.list.d/docker.list $h/etc/apt/sources.list.d/docker.list; done
```
## Attempt #3/4
### take the boring parts and make cloud init do the work
Incus allows you to provide cloud init sections to the profiles used to create incus containers.
The gist of the parts to install kubernetes and helm is here <https://gist.github.com/feurig/c7db29c4df8eade0a027411d1917a602>.
### also the key point
```sh
kubeadm init --kubernetes-version 1.33.5 --control-plane-endpoint gru
export KUBECONFIG=/etc/kubernetes/admin.conf
helm install cilium cilium/cilium --version 1.18.3 --namespace kube-system --set cni.binPath=/usr/lib/cni
```
### Make it work at the colo
```sh
incus init trixie-vm -c limits.cpu=16 -c limits.memory=24GiB -d root,size=24GiB --vm gru -p default -p k8s-colo -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 69.41.138.117/27
gateway4: 69.41.138.97
nameservers:
addresses:
- 69.41.138.98
- 8.8.4.4
EOF
)"
incus start gru
```
Initialize the control plane
```sh
kubeadm init --kubernetes-version 1.33.5 --control-plane-endpoint gru
export KUBECONFIG=/etc/kubernetes/admin.conf
helm install cilium cilium/cilium --version 1.18.3 --namespace kube-system --set cni.binPath=/usr/lib/cni
```
Set up the nodes
```sh
incus init trixie-vm -c limits.cpu=16 -c limits.memory=24GiB -d root,size=24GiB --vm minion1 -p default -p k8s-colo -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 69.41.138.118/27
gateway4: 69.41.138.97
nameservers:
addresses:
- 69.41.138.98
- 8.8.4.4
EOF
)"
incus start minion1
incus init trixie-vm -c limits.cpu=16 -c limits.memory=24GiB -d root,size=24GiB --vm minion2 -p default -p k8s-colo -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 69.41.138.119/27
gateway4: 69.41.138.97
nameservers:
addresses:
- 69.41.138.98
- 8.8.4.4
EOF
)"
incus start minion2
incus init trixie-vm -c limits.cpu=16 -c limits.memory=24GiB -d root,size=24GiB --vm minion3 -p default -p k8s-colo -c cloud-init.network-config="$(cat <<EOF
version: 2
ethernets:
enp5s0:
addresses:
- 69.41.138.120/27
gateway4: 69.41.138.97
nameservers:
addresses:
- 69.41.138.98
- 8.8.4.4
EOF
)"
incus start minion3
```
## references
- <https://discuss.linuxcontainers.org/t/how-to-use-cloud-init-to-set-up-a-vm-or-container-static-ip/22075/2>
- <https://forum.linuxfoundation.org/discussion/869484/issue-with-coredns-pods-after-initial-cluster-setup>
- <https://max-pfeiffer.github.io/installing-kubernetes-on-debian-13-trixie.html>
|
