|
Dobrica Pavlinušić's random unstructured stuff
LXC
 This are notes for my LXC workshop, in state of flux Cgroups
Systemd
launchd alterntive, somilar to inetd, but for unix sockets (mostly) LXCVirtual Servers and Checkpoint/Restart in Mainstream Linux http://lxc.sourceforge.net/doc/sigops/appcr.pdf
LXC inside KVM for testingsetup KVM LXC test machineStep 1: Create a root filesystem for the KVM system.Step 2: Build a kernel for KVM, with container support.Step 3: Boot the result under QEMU or KVMStep 4: ssh into the KVM instance.ssh root@127.0.0.1 -p 9876 Step 5: Set up a simple busybox-based container under the KVM system.wget http://busybox.net/downloads/binaries/latest/busybox-i686 -O busybox chmod +x busybox echo -e "lxc.utsname = container\nlxc.network.type = empty" > container.conf PATH=$(pwd):$PATH lxc-create -f container.conf -t busybox -n container Step 6: Launch the containerlxc-start -n container # console is broken, so start another lxc-console -n container Step 7: Stop the container, and the KVM system.lxc-stop -n container # remove container lxc-destroy -n container Setup networkingStep 1: Add a TAP interface to the Laptop.# FIXME change username tunctl -u dpavlin -t kvm0 ifconfig kvm0 192.168.254.1 netmask 255.255.255.0 echo 1 > /proc/sys/net/ipv4/ip_forward # iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE Step 2: Launch KVM with two ethernet interfaces.
kvm -m 1024 -kernel ../01-setup/linux-2.6.*/arch/x86/boot/bzImage -no-reboot \
-hda ../01-setup/squeeze.ext3 -append "root=/dev/hda rw panic=1" \
-net nic,model=e1000 -net user -redir tcp:9876::22 \
-net nic,model=e1000 -net tap,ifname=kvm0,script=no
Step 3: Set up a new container in the KVM system.root@kvm:~# cat > busybox.conf << EOF lxc.utsname = busybox lxc.network.type = phys lxc.network.flags = up lxc.network.link = eth1 #lxc.network.name = eth0 EOF PATH=$(pwd):$PATH lxc-create -f busybox.conf -t busybox -n busybox lxc-start -n busybox root@kvm:~# lxc-console -n busybox ifconfig eth1 192.168.254.2 netmask 255.255.255.0 route add default gw 192.168.254.1 Step 4: Fun with routing.On host, bring up loopback alias in KVM network: dpavlin@x200:~$ sudo ifconfig lo:1 10.0.2.200 netmask 255.255.255.0 busybox container can reach it, while KVM can't ! network hintsprepare host machinemacvlan
lxc.network.type=macvlan ip link add link <phys> name <vif> address <mac address> type macvlan mode (bridge|vepa|private) ip link add link bond200 name bond200:0 address 00:aa:bb:cc:dd:ee type macvlan mode bridge ip -d show link bond200:0
lxc.network.type = macvlan vethsudo apt-get install bridge-utils dnsmasq # setup hints sysctl -w net.ipv4.ip_forward=1 ptables -t nat -A POSTROUTING -o wlan0 -j SNAT --to-source=WLAN0_IP # or for nat iptables -t nat -A POSTROUTING -o wlan0 -j MASQUERADE
lxc.network.type=veth
lxc.network.name = eth0.12
lxc.network.veth.pair = veth12 host-only bridge
$ cat /etc/network/interfaces
auto br0
iface br0 inet static
bridge_ports dummy0
bridge_maxwait 0
address 172.16.16.1
netmask 255.255.255.0
pseudo-random mac?http://en.wikipedia.org/wiki/Mac_address
x2:xx:xx:xx:xx:xx
IP=192.168.0.50 # container nic IP slow network?
/usr/sbin/ethtool -K br0 sg off physkernel > 2.6.35
lxc.network.type=phys limit container resourcescpuset.cpusecho 1 > /cgroup/<name>/cpuset.cpus # 2nd CPU! echo 1,2,3 > /cgroup/<name>/cpuset.cpus echo 0-7 > /cgroup/<name>/cpuset.cpus lxc-execute -n foo -s lxc.cgroup.cpuset.cpus="1,2,3" myforks cpu.shareslxc-execute -n foo -s lxc.cgroup.cpu.shares=1 /bin/bash lxc-execute -n bar -s /bin/bash while $(true); do echo -n . ; done lxc-cgroup -n foo cpu.shares=1024 memory
lxc.cgroup.memory.limit_in_bytes = 256M diskusagestandard Linux tools:
limit disk bandwith using cgroup blkioRequired kernel configuration
CONFIG_BLK_CGROUP=y create containers for test
#!/bin/sh -xe
lxc-ls | xargs -i sh -c "lxc-stop -n {} ; lxc-destroy -n {}"
echo "lxc.network.type = empty" > blkio.conf
PATH=$(pwd):$PATH lxc-create -f blkio.conf -t busybox -n disk1
PATH=$(pwd):$PATH lxc-create -f blkio.conf -t busybox -n disk2
PATH=$(pwd):$PATH lxc-create -f blkio.conf -t busybox -n disk3
lxc-ls | xargs -i dd if=/dev/zero of=/var/lib/lxc/{}/rootfs/tmp/zero bs=1M count=100
cat > /tmp/speed.sh <<EOF
#!/bin/sh
while true ; do
sync ; echo 3 > /proc/sys/vm/drop_caches
dd if=/tmp/zero of=/dev/null 2>&1
done | grep MB
EOF
chmod +x /tmp/speed.sh
lxc-ls | xargs -i cp /tmp/speed.sh /var/lib/lxc/{}/rootfs/tmp/speed.sh
lxc-ls | xargs -i lxc-start -d -n {}
login into each container and run test root@kvm:~# lxc-console -n disk1 Type <Ctrl+a q> to exit the console disk1 login: root ~ # /tmp/speed.sh 104857600 bytes (100.0MB) copied, 0.958453 seconds, 104.3MB/s Test limits (be careful not to enter 1000, you might oops kernel!) root@kvm:~# echo 100 > /mnt/cgroup/disk1/blkio.weight root@kvm:~# echo 200 > /mnt/cgroup/disk2/blkio.weight root@kvm:~# echo 500 > /mnt/cgroup/disk3/blkio.weight root@kvm:~# cat /mnt/cgroup/disk?/blkio.weight 100 200 500 Limit /dev/hda to 1Mb/s read root@kvm:~# ls -al /dev/hda brw-rw---- 1 root disk 3, 0 May 15 00:10 /dev/hda root@kvm:~# echo "3:0 1048576" > /mnt/cgroup/disk1/blkio.throttle.read_bps_device network# mkdir -p /dev/cgroup # mount -t cgroup net_cls -o net_cls /dev/cgroup # mkdir /dev/cgroup/A # mkdir /dev/cgroup/B # cd /dev/cgroup # echo 0x1001 > A/net_cls.classid # 10:1 # echo 0x1002 > B/net_cls.classid # 10:2 # tc qdisc add dev eth0 root handle 10: htb # tc class add dev eth0 parent 10: classid 10:1 htb rate 40mbit # tc class add dev eth0 parent 10: classid 10:2 htb rate 30mbit # tc filter add dev eth0 parent 10: protocol ip prio 10 handle 1: cgroup LXC commandslxc-create/usr/lib/lxc/templates/
export MIRROR=http://192.168.1.20:3142/ftp.debian.org
cat > /tmp/lenny.conf
t61p:~# lxc-create -n lenny -t debian -f /tmp/lenny.conf lxc-executeapplication container (shares filesystem!) lxc-ssh
lxc-execute -n foo -s lxc.utsname=foo /bin/bash lxc-attachNeeds kernel patch
lxc-attach deviceshttp://lwn.net/Articles/273208/ lxc.cgroup.devices.allow = <type> <major>:<minor> <perm>
<type> : b (block), c (char), etc ... monitoringhtophtop - cgroups > r192
t61p:/tmp# apt-get source htop t61p:/tmp/htop-0.9# DEB_BUILD_OPTIONS="--enable-cgroup" fakeroot debian/rules binary
t61p:/tmp/htop-0.9# fakeroot debian/rules binary procfs
http://lxc.sourceforge.net/download/procfs/procfs.tar.gz (fuse, defunct) debugginglxc-start --logpriority=TRACE -o /tmp/trace.log --name my_container (must have redirect to file!) kernel patcheshttp://lxc.sourceforge.net/patches/linux/ Are we in container?on host:
dpavlin@stage:~$ cat /proc/$$/cgroup inside container:
dpavlin@narada:~$ cat /proc/$$/cgroup 32-bit guest on 64-bit kernel(lxc >= 0.7.3) lxc.arch=x86 Container tweaksudevecho udev hold | dpkg --set-selections nfskernel doesn't have nfs namespaces yet, use user-space nfs servers: chromium
pam
pam_netns allows to setup a private network namespace for every user X-server
Virtual PCI network cardsdon't delete filesdpkg-divert --rename /etc/init/theinitfile.conf .pre |
