Commit 5dc0d6dc authored by Larkin Heintzman's avatar Larkin Heintzman

init commit

parent 46a800b3
Pipeline #58 failed with stages
Commands to initialize cluster (pay attention to sudo useage):
sudo kubeadm init --pod-network-cidr=10.244.0.0/16
or, if using config file
sudo kubeadm init --config kubeadm-config.yaml
In either case, may need to turn off swap memory to get cluster up:
sudo swapoff -a
...
after thats done do, somehow the mkdir and chown need to be run every time:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
start up cluster networking with one of the following:
requires internet:
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
requires the kube-flannel.yml file locally:
kubectl apply -f kube-flannel.yml
check that pods/nodes are up, can take a while due to launch order:
kubectl get pods --all-namespaces
allow pods to run on control-plane node (by removing the taint on all nodes):
kubectl taint nodes --all node-role.kubernetes.io/master-
-----------------------------------------------------
ros_on_kubernetes set up steps:
kubectl create -f ros_on_kubernetes
kubectl get all
kubectl exec -it listener bash (deprech kubectl exec -it listener -- bash)
source .. yadda ya bumdadada
-----------------------------------------------------
turning off nodes:
first drain node (kubectl get nodes, for node name):
kubectl drain <node name> --delete-emptydir-data --force --ignore-daemonsets
reset cluster:
sudo kubeadm reset
reset ipv6 tables and such (seems to work fine without but eh):
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
another ipvs reset:
ipvsadm -C
now you can finally delete the node(s) in question:
kubectl delete node <node name>
--------------------------------------------------------
handy interaction commands
get into pod bash shell (remove -it and just put command for noninteractive):
kubectl exec <pod_name> -it -- bash
This diff is collapsed.
# syntax=docker/dockerfile:1
FROM python:3.7-alpine
WORKDIR /code
ENV FLASK_APP=app.py
ENV FLASK_RUN_HOST=0.0.0.0
RUN apk add --no-cache gcc musl-dev linux-headers
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
EXPOSE 5000
COPY . .
CMD ["flask", "run"]
import time
import redis
from flask import Flask
app = Flask(__name__)
cache = redis.Redis(host='redis', port=6379)
def get_hit_count():
retries = 5
while True:
try:
return cache.incr('hits')
except redis.exceptions.ConnectionError as exc:
if retries == 0:
raise exc
retries -= 1
time.sleep(0.5)
@app.route('/')
def hello():
count = get_hit_count()
return 'Hello World! I have been seen {} times.\n'.format(count)
version: "3.3"
services:
web:
build: .
ports:
- "5000:5000"
redis:
image: "redis:alpine"
To test a quick docker web app, with steps/code taken from https://docs.docker.com/compose/gettingstarted/
- from project directory, start up docker daemon (not sure why this isn't a service):
sudo dockerd
- bring up container (again from project dir, separate terminal):
sudo docker-compose up
- check http://localhost:5000/ to see if page loads, refresh to see counter incrementing
if dockerd comes back with an error the following command might be needed:
ps axf | grep docker | grep -v grep | awk '{print "kill -9 " $1}' | sudo sh
FROM busybox
CMD echo $((40 + 2))
{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo $((40 + 2))"],"Image":"sha256:42b97d3c2ae95232263a04324aaf656dc80e7792dee6629a9eff276cdfb806c0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"7844ba4fcb3e2573beee308f75094be8428cf1a99076340d971f9f0b2b61b9cc","container_config":{"Hostname":"7844ba4fcb3e","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\" \"-c\" \"echo $((40 + 2))\"]"],"Image":"sha256:42b97d3c2ae95232263a04324aaf656dc80e7792dee6629a9eff276cdfb806c0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"created":"2021-09-13T17:33:29.768365573Z","docker_version":"20.10.8","history":[{"created":"2021-08-20T21:20:01.375838656Z","created_by":"/bin/sh -c #(nop) ADD file:89552401a4d4fc6b2e7e019e9dff35d16de40ad30973d5ccdd78d3de33790155 in / "},{"created":"2021-08-20T21:20:01.541762445Z","created_by":"/bin/sh -c #(nop) CMD [\"sh\"]","empty_layer":true},{"created":"2021-09-13T17:33:29.768365573Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo $((40 + 2))\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:0fd05bf2930d72edc1aa0b9fa7e442295c2384512a32c0b1502392f5384acd81"]}}
\ No newline at end of file
{"id":"e93c093b65bb10672e77027c11ee804f192a401d94cf1b8f53fcc84d6335f6d3","created":"2021-09-13T17:33:29.768365573Z","container":"7844ba4fcb3e2573beee308f75094be8428cf1a99076340d971f9f0b2b61b9cc","container_config":{"Hostname":"7844ba4fcb3e","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\" \"-c\" \"echo $((40 + 2))\"]"],"Image":"sha256:42b97d3c2ae95232263a04324aaf656dc80e7792dee6629a9eff276cdfb806c0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"docker_version":"20.10.8","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo $((40 + 2))"],"Image":"sha256:42b97d3c2ae95232263a04324aaf656dc80e7792dee6629a9eff276cdfb806c0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}
\ No newline at end of file
[{"Config":"42ca20c5edcca8f3790773dd6111e6eb360f56a7a2fabcd4feba72a940b1eccb.json","RepoTags":["calc:latest"],"Layers":["e93c093b65bb10672e77027c11ee804f192a401d94cf1b8f53fcc84d6335f6d3/layer.tar"]}]
{"calc":{"latest":"e93c093b65bb10672e77027c11ee804f192a401d94cf1b8f53fcc84d6335f6d3"}}
How to do basic saving and loading of containers/images, steps taken from https://pspdfkit.com/blog/2019/docker-import-export-vs-load-save/
- start up docker daemon in separate terminal in project dir:
sudo dockerd
- assuming we already have a Dockerfile, build image using tag "calc" of current directory
sudo docker build --tag calc .
- check that docker actually built the image and stored it locally
sudo docker images OR sudo docker image ls
- run image tagged "calc" and see test output
sudo docker run calc
- save the image in question to a tar file in the current directory
sudo docker save calc > calc.tar
- check out the contents of the image after saving
mkdir calc && tar -xf calc.tar -C calc THEN tree calc
- delete the local image (if for some reason you want to rebuild/import it):
(get all containers)
sudo docker ps -a
(remove a specific container so we can delete the image later)
sudo docker rm <container_id>
(then delete the image via tag or id)
sudo docker image ls
sudo docker image rm <tag/id>
- also can delete all exited containers via this kickass command:
sudo docker ps -a | grep Exit | cut -d ' ' -f 1 | xargs sudo docker rm
- load image from tar file
sudo docker load < calc.tar
- run it again to confirm it works (can also look and make sure image is there)
sudo docker run calc
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unused in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni
image: quay.io/coreos/flannel:v0.14.0
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.14.0
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
# kubeadm-config.yaml
kind: ClusterConfiguration
apiVersion: kubeadm.k8s.io/v1beta2
kubernetesVersion: v1.21.3
networking:
podSubnet: "10.244.0.0/16"
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd
version: '2.4'
services:
master:
image: llhcluster/master:v0
build:
context: ./master/
args:
from: ros:noetic
node:
image: llhcluster/node:v0
build:
context: ./node/
args:
from: ros:noetic
ARG from=ros:noetic
FROM ${from}
# install build tools
# RUN apt-get update && apt-get install -y \
# python-catkin-tools \
# && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y python3-pip git
# work around as the apt-get version fails
RUN pip3 install git+https://github.com/catkin/catkin_tools.git
# clone ros package repo
ENV ROS_WS /opt/ros_ws
RUN mkdir -p $ROS_WS/src
WORKDIR $ROS_WS
RUN git -C src clone \
-b $ROS_DISTRO-devel \
https://github.com/ros/ros_tutorials.git
# install ros package dependencies
RUN apt-get update && rosdep update && rosdep install -y --from-paths src/ros_tutorials/roscpp_tutorials --ignore-src
# build ros package source
RUN catkin config \
--extend /opt/ros/$ROS_DISTRO && \
catkin build \
roscpp_tutorials
COPY ros_entrypoint.sh /usr/local/bin/ros_entrypoint.sh
RUN chmod 755 /usr/local/bin/ros_entrypoint.sh
ENTRYPOINT ["/usr/local/bin/ros_entrypoint.sh"]
CMD ["roscore"]
# source ros package from entrypoint
# RUN sed --in-place --expression \
# '$isource "$ROS_WS/devel/setup.bash"' \
# /ros_entrypoint.sh
# run ros package launch file
# CMD ["roslaunch", "roscpp_tutorials", "talker_listener.launch"]
#!/bin/bash
set -e
echo "==> Executing master image entrypoint ..."
# setup ros environment
source "/opt/ros/$ROS_DISTRO/setup.bash"
# might need to source the devel space as well
source "$ROS_WS/devel/setup.bash"
echo "==> Container ready"
exec "$@"
ARG from=ros:noetic
FROM ${from}
# install build tools
# RUN apt-get update && apt-get install -y \
# python-catkin-tools \
# && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y python3-pip git
RUN pip3 install git+https://github.com/catkin/catkin_tools.git
# clone ros package repo
ENV ROS_WS /opt/ros_ws
RUN mkdir -p $ROS_WS/src
WORKDIR $ROS_WS
RUN git -C src clone \
-b $ROS_DISTRO-devel \
https://github.com/ros/ros_tutorials.git
# install ros package dependencies
RUN apt-get update && \
rosdep update && \
rosdep install -y \
--from-paths \
src/ros_tutorials/roscpp_tutorials \
--ignore-src && \
rm -rf /var/lib/apt/lists/*
# build ros package source
RUN catkin config \
--extend /opt/ros/$ROS_DISTRO && \
catkin build \
roscpp_tutorials
COPY check_roscore.py /usr/local/bin/check_roscore.py
COPY ros_entrypoint.sh /usr/local/bin/ros_entrypoint.sh
RUN chmod 755 /usr/local/bin/check_roscore.py &&\
chmod 755 /usr/local/bin/ros_entrypoint.sh
ENTRYPOINT ["/usr/local/bin/ros_entrypoint.sh"]
CMD ["bash"]
# run ros package launch file
# CMD ["roslaunch", "roscpp_tutorials", "talker_listener.launch"]
#!/usr/bin/env python3
import sys
import rosgraph
if not rosgraph.is_master_online():
print("Waiting roscore ...")
sys.exit(1)
print("roscore found!")
#!/bin/bash
set -e
echo "==> Executing node image entrypoint ..."
# setup ros environment
source "/opt/ros/$ROS_DISTRO/setup.bash"
# might need to source the devel space as well
source "$ROS_WS/devel/setup.bash"
echo "==> Container ready"
exec "$@"
apiVersion: apps/v1
kind: Deployment
metadata:
name: listener-deployment
labels:
app: llhcluster
node: listener
spec:
replicas: 1
# The deployment handles all matching templated pods
selector:
matchLabels:
node: listener
# Template for a replica.
# The deployment makes sure that a POD containing the containers
# defined below is always running.
template:
metadata:
labels:
node: listener
spec:
# Wait roscore to be running before starting the pod.
# An initContainer is executed when this pod is created
# and tries to connect to the rocore which is running in
# a container in the master pod deployment.
# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
initContainers:
- name: init-service-listener
image: llhcluster/node:v0
args:
- check_roscore.py
env:
- name: ROS_MASTER_URI
value: http://service-master:11311
- name: ROS_HOSTNAME
value: service-listener
containers:
# The real node container
- name: llhclusterpublisher
image: llhcluster/node:v0
args:
- rostopic
- echo
- chatter
env:
- name: ROS_MASTER_URI
value: http://service-master:11311
- name: ROS_HOSTNAME
value: service-listener
# When the roscore container stops or fails, all the node
# containers need to be restarted because the ros network
# configuration is lost.
# This liveness probe restarts the node container if the
# 'chatter' topic is not anymore listed in the ROS network.
# This is required because the node doesn't fail automatically
# when roscore stops / restarts.
# https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
livenessProbe:
exec:
command:
- entrypoint.sh
- /opt/ros/melodic/bin/rostopic
- info
- chatter
initialDelaySeconds: 5
periodSeconds: 1
---
apiVersion: v1
kind: Service
metadata:
name: service-listener
labels:
app: llhcluster
node: listener
spec:
# Start a headless service
# https://kubernetes.io/docs/concepts/services-networking/service/#headless-services
clusterIP: None
ports:
# Dummy port
- port: 11311
selector:
node: listener
apiVersion: apps/v1
kind: Deployment
metadata:
name: roscore-deployment
labels:
app: llhcluster
node: roscore
spec:
replicas: 1
# The deployment handles all matching templated pods
selector:
matchLabels:
node: roscore
# Template for a replica.
# The deployment makes sure that a POD containing the containers
# defined below is always running.
template:
metadata:
labels:
node: roscore
spec:
containers:
- name: llhclustermaster
image: llhcluster/master:v0
args:
- roscore
ports:
- containerPort: 11311
name: roscoreport
---
apiVersion: v1
kind: Service
metadata:
name: service-master
labels:
app: llhcluster
node: roscore
spec:
clusterIP: None
ports:
- port: 11311
selector:
node: roscore
apiVersion: apps/v1
kind: Deployment
metadata:
name: talker-deployment
labels:
app: llhcluster
node: talker
spec:
replicas: 1
# The deployment handles all matching templated pods
selector:
matchLabels:
node: talker
# Template for a replica.
# The deployment makes sure that a POD containing the containers
# defined below is always running.
template:
metadata:
labels:
node: talker
spec:
# Wait roscore to be running before starting the pod.
# An initContainer is executed when this pod is created
# and tries to connect to the rocore which is running in
# a container in the master pod deployment.
# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
initContainers:
- name: init-service-talker
image: llhcluster/node:v0
args:
- check_roscore.py
env:
- name: ROS_MASTER_URI
value: http://service-master:11311
- name: ROS_HOSTNAME
value: service-talker
containers:
# The real node container
- name: llhclusterpublisher
image: llhcluster/node:v0
args:
- "rostopic"
- "pub"
- "-r 1"
- "chatter"
- "std_msgs/String"
- "Hello, world"
env:
- name: ROS_MASTER_URI
value: http://service-master:11311
- name: ROS_HOSTNAME
value: service-talker
# When the roscore container stops or fails, all the node
# containers need to be restarted because the ros network
# configuration is lost.
# This liveness probe restarts the node container if the
# 'chatter' topic is not anymore listed in the ROS network.
# This is required because the node doesn't fail automatically
# when roscore stops / restarts.
# https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
livenessProbe:
exec:
command:
- entrypoint.sh
- /opt/ros/melodic/bin/rostopic
- info
- chatter
initialDelaySeconds: 5
periodSeconds: 1
---
apiVersion: v1
kind: Service
metadata:
name: service-talker
labels:
app: llhcluster
node: talker
spec:
# Start a headless service
# https://kubernetes.io/docs/concepts/services-networking/service/#headless-services
clusterIP: None
ports:
# Dummy port
- port: 11311
selector:
node: talker
apiVersion: v1
kind: Pod
metadata:
name: listener
labels:
name: listener
spec:
containers:
- name: listener
image: ros:indigo-ros-core
env:
- name: ROS_HOSTNAME
value: listener
- name: ROS_MASTER_URI
value: http://master:11311
args:
- rostopic
- echo
- my_topic
apiVersion: v1
kind: Service
metadata:
name: listener
spec:
clusterIP: None
ports:
- port: 11311
protocol: TCP
selector:
name: listener
type: ClusterIP
apiVersion: v1
kind: Pod
metadata:
name: master
labels:
name: master
spec:
containers:
- name: master
image: roscluster/master:v0
ports:
- containerPort: 11311
name: master
# args:
# - roscore
apiVersion: v1
kind: Service
metadata:
name: master
spec:
clusterIP: None
ports:
- port: 11311
protocol: TCP
selector:
name: master
type: ClusterIP
apiVersion: v1
kind: Pod
metadata:
name: talker
labels:
name: talker
spec:
containers:
- name: talker
image: ros:indigo-ros-core
env:
- name: ROS_HOSTNAME
value: talker
- name: ROS_MASTER_URI
value: http://master:11311
args:
- rostopic
- pub
- "-r"
- "1"
- my_topic
- std_msgs/String
- "SPLab+ICClab"
apiVersion: v1
kind: Service
metadata:
name: talker
spec:
clusterIP: None
ports:
- port: 11311
protocol: TCP
selector:
name: talker
type: ClusterIP
ros-kubernetes-ex @ 316101e9
Subproject commit 316101e906c06ce40993a4398a90ea2ba46b8000
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment