[elbe-devel] [PATCH v2] initvm: create: Add COW logic for initvm
Torben Hohn
torben.hohn at linutronix.de
Wed Jun 24 17:27:07 CEST 2020
On Tue, Jun 16, 2020 at 10:18:01AM -0400, Olivier Dion wrote:
> Rewrote the rules and dependencies and added support for COW images.
> Also moved the generation of the initvm-tree into Python.
>
> The base image is generated by the installer. From this base image,
> the initvm is created using COW semantic. This allows to easily clone
> the base image, but also to make a fresh initvm in an instant.
>
> Here are the steps that Elbe does for generating the installer's initrd:
>
> 1) gunzip the base initrd downloaded, given us a cpio archive.
>
> 2) A temporary directory is created and a bunch of files is copied
> into it. That directory is archived using cpio and merged with the
> previous archive.
>
> 3) The big archive is then gzip again into the final initrd.gz.
>
> To make things more clear, the step 2) is moved into init.py at the
> end.
>
> Signed-off-by: Olivier Dion <dion at linutronix.de>
Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>
> ---
> elbepack/commands/init.py | 30 +++++++++++
> elbepack/init/Makefile.mako | 93 +++++++++++++++++-----------------
> elbepack/init/libvirt.xml.mako | 2 +-
> 3 files changed, 78 insertions(+), 47 deletions(-)
>
> diff --git a/elbepack/commands/init.py b/elbepack/commands/init.py
> index 81db95f2..3e6b5394 100644
> --- a/elbepack/commands/init.py
> +++ b/elbepack/commands/init.py
> @@ -24,6 +24,7 @@ from elbepack.directories import init_template_dir, elbe_dir
> from elbepack.config import cfg
> from elbepack.shellhelper import command_out, system, do
> from elbepack.log import elbe_logging
> +from elbepack.filesystem import Filesystem
>
>
> def run_command(argv):
> @@ -269,3 +270,32 @@ def run_command(argv):
> system('tar cfj "%s" %s -C "%s" .' % (tar_fname,
> " ".join(opts),
> elbe_dir))
> +
> + to_cpy = [("apt.conf", "etc/apt"),
> + ("init-elbe.sh", ""),
> + ("source.xml", ""),
> + ("initrd-cdrom.gz", ""),
> + ("vmlinuz", ""),
> + ("preseed.cfg", "")]
> +
> + elbe_in = Filesystem(out_path)
> +
> + if opt.devel:
> + to_cpy.append(("elbe-devel.tar.bz2", ""))
> +
> + # Convert relative rfs path to absolute in the system
> + to_cpy = [(elbe_in.fname(src), elbe_in.fname(os.path.join("initrd-tree", dst)))
> + for src, dst
> + in to_cpy]
> +
> + # These are already absolute path!
> + keyrings = elbe_in.fname(os.path.join("initrd-tree", "usr/share/keyrings"))
> + for gpg in elbe_in.glob("*.gpg"):
> + to_cpy.append((gpg, keyrings))
> +
> + for src, dst in to_cpy:
> + try:
> + os.makedirs(dst)
> + except FileExistsError:
> + pass
> + shutil.copy(src, dst)
> diff --git a/elbepack/init/Makefile.mako b/elbepack/init/Makefile.mako
> index 87ec10ad..ad694507 100644
> --- a/elbepack/init/Makefile.mako
> +++ b/elbepack/init/Makefile.mako
> @@ -16,7 +16,9 @@ memory = size_to_int(prj.text('mem', default=defs, key='mem')) // 1024 // 1024
> MEMSIZE?=${memory}
> SMP?=$$((`nproc` > ${max_cpus} ? ${max_cpus} : `nproc`))
> INTERPRETER?=${prj.text('interpreter', default=defs, key='interpreter')}
> -
> +% if defs["interpreter-args"] is not None:
> +INTERPRETER-ARGS= ${" ".join(defs["interpreter-args"])}
> +% endif
> # this is a workaround for
> # http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
> VIRT=$(shell test -x /usr/bin/systemd-detect-virt && /usr/bin/systemd-detect-virt)
> @@ -49,55 +51,43 @@ interpreter_v_minor = int(prj.text('interpreterversion',
> key='interpreterversion').split('.')[1])
> fwd = ""
> if prj.has("portforwarding"):
> - for f in prj.node("portforwarding"):
> - fwd += ",hostfwd=%s::%s-:%s" % (f.text("proto"),
> - f.text("host"),
> - f.text("buildenv"))
> + for f in prj.node("portforwarding"):
> + fwd += ",hostfwd=%s::%s-:%s" % (f.text("proto"),
> + f.text("host"),
> + f.text("buildenv"))
> %>
>
> -all: .stamps/stamp-install-initial-image
> -
> -.elbe-gen/initrd-preseeded.gz: .elbe-in/*
> - rm -rf tmp-tree
> - mkdir tmp-tree
> - cp .elbe-in/*.cfg tmp-tree/
> - -cp .elbe-in/apt.conf tmp-tree/
> - mkdir -p tmp-tree/etc/apt
> - -cp .elbe-in/apt.conf tmp-tree/etc/apt
> - mkdir -p tmp-tree/usr/lib/post-base-installer.d
> - cp .elbe-in/init-elbe.sh tmp-tree/
> - cp .elbe-in/source.xml tmp-tree/
> - cp .elbe-in/initrd-cdrom.gz tmp-tree/
> - cp .elbe-in/vmlinuz tmp-tree/
> -% if opt.devel:
> - cp .elbe-in/elbe-devel.tar.bz2 tmp-tree/
> -% endif
> - mkdir -p tmp-tree/usr/share/keyrings
> - -cp .elbe-in/*.gpg tmp-tree/usr/share/keyrings
> - mkdir -p tmp-tree/usr/lib/base-installer.d
> - echo 'mkdir -p /target/etc/apt/trusted.gpg.d/; cp /usr/share/keyrings/elbe-keyring.gpg /target/etc/apt/trusted.gpg.d/' > tmp-tree/usr/lib/base-installer.d/10copyelbekeyring
> - chmod 755 tmp-tree/usr/lib/base-installer.d/*
> - mkdir -p .elbe-gen
> - gzip -cd .elbe-in/initrd.gz >.elbe-gen/initrd-preseeded
> - cd tmp-tree && find . | cpio -H newc -o --append -F ../.elbe-gen/initrd-preseeded
> - gzip -9f .elbe-gen/initrd-preseeded
> - rm -rf tmp-tree
>
> -.stamps/stamp-create-buildenv-img buildenv.img: .elbe-gen/initrd-preseeded.gz
> - qemu-img create -f ${img} buildenv.img ${imgsize}
> - mkdir -p .stamps
> - touch .stamps/stamp-create-buildenv-img
> +GEN=.elbe-gen
> +IN=.elbe-in
> +
> +INITRD=$(GEN)/initrd-preseeded.gz
> +VMLINUZ=$(IN)/vmlinuz
> +
> +INITRD_FILES=$(shell find $(IN)/initrd-tree -type f)
> +
> +BASE=initvm-base.img
> +INITVM=initvm.img
>
> -.stamps/stamp-install-initial-image: .stamps/stamp-create-buildenv-img
> +CLEAN=$(BASE) $(INITVM) $(GEN)
> +
> +all: $(INITVM)
> +
> +$(INITVM): $(BASE)
> + qemu-img create -f ${img} -F ${img} -b $< $@
> +
> +$(BASE): $(INITRD)
> + qemu-img create -f ${img} $@ ${imgsize}
> @ echo $(INTERPRETER)
> @ $(INTERPRETER) -M $(MACHINE) \
> + $(INTERPRETER-ARGS) \
> -device virtio-rng-pci \
> - -drive file=buildenv.img,if=$(HD_TYPE),bus=1,unit=0 \
> + -drive file=$@,if=$(HD_TYPE),bus=1,unit=0 \
> % if prj.has("mirror/cdrom"):
> -drive file=${prj.text("mirror/cdrom")},if=$(CDROM_TYPE),media=cdrom,bus=1,unit=0 \
> % endif
> - -kernel .elbe-in/vmlinuz \
> - -initrd .elbe-gen/initrd-preseeded.gz \
> + -kernel $(VMLINUZ) \
> + -initrd $(INITRD) \
> -append 'root=/dev/$(HD_NAME) debconf_priority=critical console=$(CONSOLE) DEBIAN_FRONTEND=text' \
> -no-reboot \
> -nographic \
> @@ -106,6 +96,7 @@ all: .stamps/stamp-install-initial-image
> -m $(MEMSIZE) \
> -smp $(SMP) \
> -usb \
> + -cpu host \
> || ( echo; \
> echo "------------------------------------------------------------------"; \
> echo "kvm failed to start"; \
> @@ -117,14 +108,21 @@ all: .stamps/stamp-install-initial-image
> false \
> )
>
> - mkdir -p .stamps
> - touch .stamps/stamp-install-initial-image
> +$(INITRD): $(INITRD_FILES)
> + mkdir -p $(IN)/initrd-tree/usr/lib/base-installer.d
> + echo 'mkdir -p /target/etc/apt/trusted.gpg.d/; cp /usr/share/keyrings/elbe-keyring.gpg /target/etc/apt/trusted.gpg.d/' > $(IN)/initrd-tree/usr/lib/base-installer.d/10copyelbekeyring
> + chmod 755 $(IN)/initrd-tree/usr/lib/base-installer.d/*
> + mkdir -p .elbe-gen
> + gzip -cd $(IN)/initrd.gz > $(GEN)/initrd-preseeded
> + cd $(IN)/initrd-tree && find . | cpio -H newc -o --append -F ../../$(GEN)/initrd-preseeded
> + gzip -9f $(GEN)/initrd-preseeded
>
> run:
> $(INTERPRETER) -M $(MACHINE) \
> + $(INTERPRETER-ARGS) \
> -device virtio-rng-pci \
> -device virtio-net-pci,netdev=user.0 \
> - -drive file=buildenv.img,if=$(HD_TYPE),bus=1,unit=0 \
> + -drive file=$(INITVM),if=$(HD_TYPE),bus=1,unit=0 \
> -no-reboot \
> % if ((interpreter_v_major == 2) and (interpreter_v_minor >= 8)) or (interpreter_v_major > 2):
> -netdev user,ipv4,id=user.0${fwd} \
> @@ -140,9 +138,10 @@ run:
>
> run-con:
> $(INTERPRETER) -M $(MACHINE) \
> + $(INTERPRETER-ARGS)
> -device virtio-rng-pci \
> -device virtio-net-pci,netdev=user.0 \
> - -drive file=buildenv.img,if=$(HD_TYPE),bus=1,unit=0 \
> + -drive file=$(INITVM),if=$(HD_TYPE),bus=1,unit=0 \
> -no-reboot \
> % if ((interpreter_v_major == 2) and (interpreter_v_minor >= 8)) or (interpreter_v_major > 2):
> -netdev user,ipv4,id=user.0${fwd} \
> @@ -158,7 +157,9 @@ run-con:
> -smp $(SMP)
>
> clean:
> - rm -fr .stamps/stamp* buildenv.img .elbe-vm .elbe-gen
> + rm -fr $(CLEAN)
>
> distclean: clean
> - echo clean
> + @echo clean
> +
> +.PHONY: all clean distclean run run-con
> diff --git a/elbepack/init/libvirt.xml.mako b/elbepack/init/libvirt.xml.mako
> index 80159005..b91b849b 100644
> --- a/elbepack/init/libvirt.xml.mako
> +++ b/elbepack/init/libvirt.xml.mako
> @@ -24,7 +24,7 @@ cpus = min(multiprocessing.cpu_count(), cpus)
> memory = size_to_int(prj.text('mem', default=defs, key='mem')) // 1024
>
> imagetype = prj.text('img', default=defs, key='img')
> -img = os.path.join(opt.directory, 'buildenv.img')
> +img = os.path.join(opt.directory, 'initvm.img')
>
> emulator = prj.text('interpreter', default=defs, key='interpreter')
> nicmac = prj.text('buildimage/NIC/MAC', default=defs, key='nicmac')
> --
> 2.27.0
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
More information about the elbe-devel
mailing list