From 09c2b6de63a6ff35348fcb2c2eb57b974f0a8a52 Mon Sep 17 00:00:00 2001 From: Kleidi Bujari Date: Wed, 2 Apr 2025 21:59:20 -0400 Subject: automate project structure --- packages/blog/.gitignore | 2 + packages/blog/config.toml | 8 ++ packages/blog/content/posts/_index.md | 6 ++ .../blog/content/posts/deterministic-hostnames.md | 80 ++++++++++++++++++++ packages/blog/content/posts/nohup.md | 26 +++++++ .../content/posts/stateless-compute-networks.md | 41 ++++++++++ packages/blog/content/posts/vim-compilers.md | 20 +++++ packages/blog/default.nix | 21 ++++++ packages/blog/templates/404.html | 9 +++ packages/blog/templates/base.html | 88 ++++++++++++++++++++++ packages/blog/templates/index.html | 16 ++++ packages/blog/templates/post.html | 14 ++++ 12 files changed, 331 insertions(+) create mode 100644 packages/blog/.gitignore create mode 100644 packages/blog/config.toml create mode 100644 packages/blog/content/posts/_index.md create mode 100644 packages/blog/content/posts/deterministic-hostnames.md create mode 100644 packages/blog/content/posts/nohup.md create mode 100644 packages/blog/content/posts/stateless-compute-networks.md create mode 100644 packages/blog/content/posts/vim-compilers.md create mode 100644 packages/blog/default.nix create mode 100644 packages/blog/templates/404.html create mode 100644 packages/blog/templates/base.html create mode 100644 packages/blog/templates/index.html create mode 100644 packages/blog/templates/post.html (limited to 'packages/blog') diff --git a/packages/blog/.gitignore b/packages/blog/.gitignore new file mode 100644 index 0000000..decc3f8 --- /dev/null +++ b/packages/blog/.gitignore @@ -0,0 +1,2 @@ +public/ +nohup.out diff --git a/packages/blog/config.toml b/packages/blog/config.toml new file mode 100644 index 0000000..69c389b --- /dev/null +++ b/packages/blog/config.toml @@ -0,0 +1,8 @@ +base_url = "https://4kb.net" +build_search_index = false +compile_sass = false +generate_feeds = true +minify_html = true + +[markdown] +highlight_code = false diff --git a/packages/blog/content/posts/_index.md b/packages/blog/content/posts/_index.md new file mode 100644 index 0000000..9efc62e --- /dev/null +++ b/packages/blog/content/posts/_index.md @@ -0,0 +1,6 @@ ++++ +title = "Posts" +sort_by = "date" +page_template = "post.html" +redirect_to="/" ++++ diff --git a/packages/blog/content/posts/deterministic-hostnames.md b/packages/blog/content/posts/deterministic-hostnames.md new file mode 100644 index 0000000..43dc7d8 --- /dev/null +++ b/packages/blog/content/posts/deterministic-hostnames.md @@ -0,0 +1,80 @@ +--- +title: "Deterministic and unique network hostnames" +date: "2024-11-10" +--- + +As part of building out a Kubernetes cluster, I wanted to build and distribute a +single OS image to create stateless worker nodes. Using network booting, and +some clever tricks to differentiate nodes, we can create a scaleable and +efficient farm of workers for a cluster that don't even need disks. + +The idea came from a plan to build a cluster using the +[compute blade](https://computeblade.com/), and a few Raspberry Pi SBCs I +already own. Running the cluster from an SD card is not recommended due to the +not-so-great reliability of the flash used by most manufacturers, so I wanted to +try PXE booting each Pi to save money rather than purchasing an SSD for each +one. The compute blades do support an NVMe disk, but I plan to use those for a +storage cluster later, so they need to remain empty. + +## Base image + +Alpine Linux has been my preferred server OS for a long time. It provides a very +lightweight base system, and bundles an excellent bootstrapping system, +[apkovl](https://wiki.alpinelinux.org/wiki/Alpine_local_backup), that allows the +user to save a set of customisations to an system as an overlay to a stock +Alpine live image. In other words, we can create our image once, save the +changes as an `apkovl.tar.gz` file, and apply the same changes to a base system +on boot. This file can even be provided as a +[kernel parameter](https://wiki.alpinelinux.org/wiki/PXE_boot#Guide_to_options) +and will be fetched from a remote webserver automatically! + +Since the image and configuration will be shipped to the node via the network, +an added benefit of using Alpine is its tiny space consumption. I'm not using +enough nodes for this to really matter, but it's a cool optimization regardless. + +## Differentiating the nodes + +One of the main goals of this project is that there should be no persistent +storage required outside the boot image itself. Since every node will download +and generate the same root file-system on startup, the first problem that arises +is how the nodes will identify themselves both on the network and the cluster, +given that it's not possible to name them ahead of time. In other words, any +given node has to generate a unique hostname that won't collide with other +workers, and that will be the same each time that node boots. + +Since these nodes will not have a predefined name, we have to rely on +characteristics of the hardware to differentiate each one. The hardware MAC +address is perfect for this, since it's unique to to each node and will not be +wiped away after the node reboots. On a system like Linux that exposes its +hardware through a _sysfs_, we can find a file containing the address at +`/sys/class/net/eth0/address`. I don't really like the idea of attaching the +literal MAC address of the node to its network hostname, since it's a security +risk, and a bit too verbose. Instead, we can transform it into something safer +using a `sha1sum`, which is already present on our Alpine base system: + +```console +sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' +``` + +### Applying the new name + +Ideally, the node should apply its generated hostname before reaching out for an +address over DHCP or joining the cluster. We can make sure it happens before any +traffic is sent out by adding a `pre-up` command to the right interface in +`/etc/network/interfaces`: + +``` + +... + +auto eth0 +iface eth0 inet dhcp + pre-up sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' > /etc/hostname + +... +``` + +The VM I tested with looks outputs `worker-e2fae8`. Pretty clean result, and if +you want to know the physical node that maps to each hostname, you can take note +of the MAC address beforehand and generate the same hash on another computer to +match them up. diff --git a/packages/blog/content/posts/nohup.md b/packages/blog/content/posts/nohup.md new file mode 100644 index 0000000..64f7983 --- /dev/null +++ b/packages/blog/content/posts/nohup.md @@ -0,0 +1,26 @@ +--- +title: "Spawning background processes" +date: "2024-11-17" +--- + +Working in a terminal, +I often pair my editor with a background process watching files. +Before reaching for terminal multiplexers, +see if you can get away with simple tty job control. +Spawn the background process, +still attached to the terminal instance: + +``` +program args & +``` + +Also redirect its output to a file, +for when the process writes to the tty from the background: + +``` +nohup program args & +``` + +Extra reading: + +- diff --git a/packages/blog/content/posts/stateless-compute-networks.md b/packages/blog/content/posts/stateless-compute-networks.md new file mode 100644 index 0000000..bac9e5d --- /dev/null +++ b/packages/blog/content/posts/stateless-compute-networks.md @@ -0,0 +1,41 @@ +--- +title: "On stateless compute networks" +date: "2024-11-10" +draft: true +--- + +On the topic of distributed systems and clustering, +I am quite invested in the idea of compute nodes that rely entirely on the network for configuration. +Arbitrary nodes can join a pre-existing cluster, +offering their CPU time and memory for computation without relying on any pre-existing configuration on the node itself. +In other words, any computer could pick up work, +only needing power and a network connection to the cluster. + +Perhaps this eventually leads into a "self-healing" cluster where only one node is manually bootstrapped, +which then serves a _configuration endpoint_ for other stateless nodes to reach out to for their instructions, +which they will then also serve once they are themselves ready. + +Early revisions of these notes mention Kubernetes, +but I am also trying to achieve similar results with NixOS on a custom project. +In any case, these are my ever-updating notes towards a general implementation of a stateless distributed systems architecture. + +## Self healing cluster + +Assuming control of an external DHCP server, +a self healing Kubernetes cluster would be feasible, +with the PXE boot artifacts supplied by the cluster itself. +That is, as long as one node is running the pod hosting the artifacts on a given endpoint, +other nodes can boot those artifacts and join the cluster, +thereby being able to host the artifacts as well. + +## Configuration endpoint + +The nodes shouldn't require a disk installed to be able to join the network. +Rather, the lofty goal of zero-configuration compute nodes passes the job of node initialization to the supporting network. +This is accomplished with PXE boot instructions supplied over DHCP. + +I delegate the following tasks to a single node in the subnet: + +- Gateway: Optional outbound connections if required +- DHCP server: Cluster IPAM +- TFTP and HTTP server: Serves iPXE firmware and kernel/initrd artifacts diff --git a/packages/blog/content/posts/vim-compilers.md b/packages/blog/content/posts/vim-compilers.md new file mode 100644 index 0000000..f735228 --- /dev/null +++ b/packages/blog/content/posts/vim-compilers.md @@ -0,0 +1,20 @@ +--- +title: "Neovim's built-in compilers" +date: "2025-01-17" +--- + +Today I learned that neovim's `:make` comes with many +[backends](https://neovim.io/doc/user/quickfix.html#_6.-selecting-a-compiler) +already configured. +This works for checking c/cpp and python files, among other, but I was +most interested to see [Typst] and [Pandoc] listed as well. + +I was looking to add this functionality with a plugin or implementing it +manually, +where a document can be compiled on the fly from the editor. +After setting `:compiler pandoc`, +generating a pdf is done with `:make pdf`, +with other pandoc options just appended afterwards if needed. + +[typst]: https://typst.app +[pandoc]: https://github.com/jgm/pandoc diff --git a/packages/blog/default.nix b/packages/blog/default.nix new file mode 100644 index 0000000..204388f --- /dev/null +++ b/packages/blog/default.nix @@ -0,0 +1,21 @@ +{ pkgs, perSystem, ... }: +let + inherit (perSystem.self) + cv + ; +in +pkgs.stdenvNoCC.mkDerivation { + name = "4kb.net"; + src = ./.; + + nativeBuildInputs = with pkgs; [ + zola + ]; + + buildPhase = "zola build"; + installPhase = '' + mkdir -p $out/share + cp -r public/* $out/ + cp ${cv} $out/share/cv.pdf + ''; +} diff --git a/packages/blog/templates/404.html b/packages/blog/templates/404.html new file mode 100644 index 0000000..a4669df --- /dev/null +++ b/packages/blog/templates/404.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} + +{% block content %} + + Error 404! + + +404! +{% endblock content %} diff --git a/packages/blog/templates/base.html b/packages/blog/templates/base.html new file mode 100644 index 0000000..5a91df5 --- /dev/null +++ b/packages/blog/templates/base.html @@ -0,0 +1,88 @@ + + + + + + {% block title %}4kb.net{% endblock title %} + + + +
+ Kleidi Bujari <mail@4kb.net> + +
+
{% block content %} {% endblock %}
+ + diff --git a/packages/blog/templates/index.html b/packages/blog/templates/index.html new file mode 100644 index 0000000..6e8a2af --- /dev/null +++ b/packages/blog/templates/index.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} {% block content %} + +
+ {% set section = get_section(path="posts/_index.md") %} + +
    + {% for page in section.pages %} +
  • + {{ page.date }} + {{ page.title }} +
    • + {% endfor %} +
+ + {% endblock content %} +
diff --git a/packages/blog/templates/post.html b/packages/blog/templates/post.html new file mode 100644 index 0000000..b819520 --- /dev/null +++ b/packages/blog/templates/post.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block title%} + {{ page.title }} - {{ super() }} +{% endblock title %} + +{% block content %} + +

{{ page.title }}

+
+ {{ page.content | safe }} +
+ +{% endblock content %} -- cgit v1.3.1