diff options
| author | Kleidi Bujari <mail@4kb.net> | 2024-12-17 23:03:27 -0500 |
|---|---|---|
| committer | Kleidi Bujari <mail@4kb.net> | 2024-12-17 23:03:27 -0500 |
| commit | eb13a9a552f9e50a66faba985521ba8e3483c447 (patch) | |
| tree | 209193a91b0db20fb1bb30007e72e321f0293696 | |
| parent | f5ed731abbc38d765333b005b9815a5f6d3e59c2 (diff) | |
| download | depot-eb13a9a552f9e50a66faba985521ba8e3483c447.tar.gz depot-eb13a9a552f9e50a66faba985521ba8e3483c447.tar.bz2 depot-eb13a9a552f9e50a66faba985521ba8e3483c447.zip | |
Push configuration out of flake
Projects are not necessarily all exported by the system flake, and
should freely support other derivations. The new "mod" directory will be
used for most projects, and it can be imported by machine
configurations.
That said, the tooling around flakes is pretty good and will still be
used until the dependency is lessened.
38 files changed, 1158 insertions, 115 deletions
@@ -21,26 +21,6 @@ "type": "github" } }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1730504689, - "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "506278e768c2a08bec68eb62932193e341f55c90", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1731755305, @@ -60,10 +40,8 @@ "root": { "inputs": { "disko": "disko", - "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", - "sops-nix": "sops-nix", - "sshKeys": "sshKeys" + "sops-nix": "sops-nix" } }, "sops-nix": { @@ -85,18 +63,6 @@ "repo": "sops-nix", "type": "github" } - }, - "sshKeys": { - "flake": false, - "locked": { - "narHash": "sha256-BEKwObEHFoq0UzQcZ0VJ6uVl3Cx2MVylPRkylReyDUk=", - "type": "file", - "url": "https://github.com/kbujari.keys" - }, - "original": { - "type": "file", - "url": "https://github.com/kbujari.keys" - } } }, "root": "root", @@ -4,32 +4,82 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; - flake-parts.url = "github:hercules-ci/flake-parts"; - flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - disko.url = "github:nix-community/disko/v1.6.1"; disko.inputs.nixpkgs.follows = "nixpkgs"; sops-nix.url = "github:Mic92/sops-nix"; sops-nix.inputs.nixpkgs.follows = "nixpkgs"; - - sshKeys = { - url = "https://github.com/kbujari.keys"; - flake = false; - }; }; - outputs = { flake-parts, ... } @ inputs: - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ "x86_64-linux" "aarch64-linux" ]; + outputs = { self, nixpkgs, ... } @ inputs: + let + inherit (builtins) + attrNames + filter + listToAttrs + map + readDir + ; - imports = [ - ./machines/flake-module.nix - ./modules/flake-module.nix - ]; + inherit (nixpkgs.lib) + nixosSystem + ; - perSystem = { pkgs, ... }: { - formatter = pkgs.nixpkgs-fmt; + readTree = import ./mod/nix/readTree { }; + + # Recursively converts subdirectories under ./mod into an attrset + # that can be passed to other parts of the depot. + readDepot = depotArgs: readTree { + args = depotArgs; + path = ./mod; }; + + # Named arguments to be made available to any nix expression in + # the depot module. + depot = readTree.fix (self: readDepot { + depot = self; + + # x86_64-linux is hardcoded as the package arch only for the + # mod directory. This workaround keeps the flake pure, + # at the expense of only supporting one system. This can + # be circumvented by retrieving the system during evaluation, + # but flakes disallow this by design. + # + # This monorepo currently depends on functionality from flakes, + # but this may change in the future, perhaps shifting to an + # impure base with certain packages exposed from the flake in a + # pure way. + # + # In practice, this means that any configuration or package + # exposed from the depot flake that makes use of internal + # derivations is limited to x86_64-linux. For now this is fine, + # but ideally any architecture supported by nix should be + # allowed to evaluate projects hosted here. + pkgs = nixpkgs.legacyPackages."x86_64-linux"; + + # Expose lib attribute to modules. + lib = nixpkgs.lib; + }); + + machines = filter (m: m != "iso" && m != "xnet") + (attrNames (readDir ./machines)); + in + { + inherit depot; + + nixosModules.xnet.imports = + [ ./machines/xnet inputs.disko.nixosModules.disko ]; + + nixosConfigurations = listToAttrs ( + map + (name: { + inherit name; + value = nixosSystem { + system = "x86_64-linux"; + specialArgs = { inherit inputs depot; }; + modules = [ ./machines/${name} self.nixosModules.xnet ]; + }; + }) + machines); }; } diff --git a/machines/flake-module.nix b/machines/flake-module.nix deleted file mode 100644 index 8d9dd2c..0000000 --- a/machines/flake-module.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ lib, self, ... }: -let - inherit (lib) nixosSystem; - inherit (self.nixosModules) xnet xlib; - inherit (self) inputs; -in -{ - flake.nixosConfigurations = { - t480 = nixosSystem { - system = "x86_64-linux"; - specialArgs = { inherit xlib; }; - modules = [ ./t480 xnet ]; - }; - - iso = nixosSystem { - system = "x86_64-linux"; - modules = [ ./iso ]; - }; - - t1 = nixosSystem { - system = "x86_64-linux"; - modules = [ ./t1 xnet ]; - }; - - iridium = nixosSystem { - system = "x86_64-linux"; - specialArgs = { inherit inputs; }; - modules = [ ./iridium xnet ]; - }; - }; - - perSystem = { pkgs, inputs', ... }: { - packages.xinstall = - let - disko = inputs'.disko.packages.disko-install; - - configs = builtins.concatStringsSep " " - (builtins.filter (x: x != "iso") - (builtins.attrNames self.nixosConfigurations)); - in - pkgs.writeShellScriptBin "xinstall" '' - set -euo pipefail - - FLAKE="github:kbujari/depot" - TARGET=$(${pkgs.gum}/bin/gum choose ${configs}) - - # ${disko}/bin/disko-install --help - ''; - }; -} diff --git a/machines/iridium/default.nix b/machines/iridium/default.nix index 8a3edd0..640554b 100644 --- a/machines/iridium/default.nix +++ b/machines/iridium/default.nix @@ -1,4 +1,11 @@ -{ inputs, lib, ... }: { +{ inputs, lib, ... }: +let + gitKeys = builtins.fetchurl { + url = "https://github.com/kbujari.keys"; + sha256 = "0fpa679zkrpx77vangzf3gnidwvmky8ifivn8411xx6albrikaqx"; + }; +in +{ system.stateVersion = "24.11"; users.mutableUsers = false; @@ -18,8 +25,7 @@ gitServer = { enable = true; gitweb.enable = true; - keys = builtins.filter (x: x != "") - (lib.strings.splitString "\n" (builtins.readFile inputs.sshKeys)); + keys = lib.splitString "\n" (builtins.readFile gitKeys); }; }; diff --git a/machines/xnet/README b/machines/xnet/README new file mode 100644 index 0000000..ab1c28c --- /dev/null +++ b/machines/xnet/README @@ -0,0 +1,18 @@ +xnet +==== + +Base configuration for any machines running a standard xnet +configuration. + +Notable features: + + - Root filesystem running in RAM + - Automated ZFS partitioning of disk + - Hardened SSH and web services + +Usage +===== + +The entire module is exposed from the top-level flake as a nixosModule, +but it changes too often to be considered stable. Options are made +available when imported under the 'xnet' attribute set. diff --git a/modules/xnet/default.nix b/machines/xnet/default.nix index a2fb336..5a6c587 100644 --- a/modules/xnet/default.nix +++ b/machines/xnet/default.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, pkgs, ... }: let inherit (lib) mkDefault; in @@ -18,7 +18,6 @@ in nix = { settings = { - allowUnfree = true; auto-optimise-store = true; experimental-features = [ "nix-command" "flakes" ]; warn-dirty = false; diff --git a/modules/xnet/desktop/default.nix b/machines/xnet/desktop/default.nix index 2eeb272..1e37497 100644 --- a/modules/xnet/desktop/default.nix +++ b/machines/xnet/desktop/default.nix @@ -19,7 +19,6 @@ in pulse.enable = true; alsa = { enable = true; - support32Bit = true; }; }; diff --git a/modules/xnet/disk.nix b/machines/xnet/disk.nix index caef61d..caef61d 100644 --- a/modules/xnet/disk.nix +++ b/machines/xnet/disk.nix diff --git a/modules/xnet/gitserver/default.nix b/machines/xnet/gitserver/default.nix index da83f73..da83f73 100644 --- a/modules/xnet/gitserver/default.nix +++ b/machines/xnet/gitserver/default.nix diff --git a/modules/xnet/gitserver/gitweb.nix b/machines/xnet/gitserver/gitweb.nix index bfd593d..bfd593d 100644 --- a/modules/xnet/gitserver/gitweb.nix +++ b/machines/xnet/gitserver/gitweb.nix diff --git a/modules/xnet/monitoring/default.nix b/machines/xnet/monitoring/default.nix index 87ae724..87ae724 100644 --- a/modules/xnet/monitoring/default.nix +++ b/machines/xnet/monitoring/default.nix diff --git a/modules/xnet/monitoring/grafana.nix b/machines/xnet/monitoring/grafana.nix index 6a37ce1..6a37ce1 100644 --- a/modules/xnet/monitoring/grafana.nix +++ b/machines/xnet/monitoring/grafana.nix diff --git a/modules/xnet/monitoring/prometheus.nix b/machines/xnet/monitoring/prometheus.nix index e32e078..e32e078 100644 --- a/modules/xnet/monitoring/prometheus.nix +++ b/machines/xnet/monitoring/prometheus.nix diff --git a/modules/xnet/net/default.nix b/machines/xnet/net/default.nix index 2255f53..2255f53 100644 --- a/modules/xnet/net/default.nix +++ b/machines/xnet/net/default.nix diff --git a/modules/xnet/net/sshd.nix b/machines/xnet/net/sshd.nix index ef225db..ef225db 100644 --- a/modules/xnet/net/sshd.nix +++ b/machines/xnet/net/sshd.nix diff --git a/modules/xnet/nginx.nix b/machines/xnet/nginx.nix index 64c1d65..64c1d65 100644 --- a/modules/xnet/nginx.nix +++ b/machines/xnet/nginx.nix diff --git a/modules/xnet/users.nix b/machines/xnet/users.nix index 53aa15b..53aa15b 100644 --- a/modules/xnet/users.nix +++ b/machines/xnet/users.nix diff --git a/mod/misc/cv/cv.typ b/mod/misc/cv/cv.typ new file mode 100644 index 0000000..e243796 --- /dev/null +++ b/mod/misc/cv/cv.typ @@ -0,0 +1,127 @@ +#import "template.typ": * + +#show: resume.with( + author: "Kleidi Bujari", + email: "mail@4kb.net", + github: "github.com/kbujari", + personal-site: "4kb.net", + paper: "us-letter", +) + +== Education + +#edu( + institution: "Toronto Metropolitan University", + dates: dates-helper(start-date: "Sep 2020", end-date: "Apr 2025"), + location: "Ontario, Canada", + degree: "Bachelor's of Engineering, Computer Engineering", +) + +- *Relevant Coursework*: + Data Structures, Embedded Programming, Compilers, Digital Systems, Computer Networks +- *Extracurriculars*: + Member of student robotics design team, + teaching assistant for micro-processor courses. + +== Experience + +#work( + company: "Toronto Metropolitan University", + title: "Graduate Research Assistant", + dates: dates-helper(start-date: "May 2024", end-date: "Sep 2024"), + location: "Toronto, Canada", +) + +- Implemented transformations for hundreds of media files, + using ffmpeg and unix primitives to parallelize workload. +- Designed frontend with AstroJS to generate only static HTML, + ensuring compatibility with many hosting providers. + +#work( + company: "Canadian Broadcasting Corporation", + title: "Network Engineering Intern", + dates: dates-helper(start-date: "May 2023", end-date: "Apr 2024"), + location: "Toronto, Canada", +) + +- Designed custom PXE-boot implementation for hundreds of devices using NetBox, + eliminating manual configuration. +- Configured Hyper-Converged Proxmox cluster for 2024 Olympics, + saving \$250k+ with reused hardware and open software. +- Deployed vendor-agnostic routing observability from scratch, + with Prometheus metrics and Grafana dashboards. +- Mentored junior application developers in modern C++23 programming, + aiding in performance design and memory safety. + +#work( + company: "WSP Canada", + title: "Student Engineer", + dates: dates-helper(start-date: "May", end-date: "Aug") + ", 2021, 2022", + location: "Toronto, Canada", +) + +- Contributed to subway car control systems with modern C++20, + replacing legacy code with newer STL functions. +- Extended internal distributed filesystem with support for deduplication and compression, + reclaiming 30% of storage. +- Validated new electrical designs for power consumption, + cost efficiency, and viability with existing systems. +- Participated in reviewing and adjusting large scale electrical and structural engineering designs. + +== Projects + +*Custom Linux Distribution* --- +Designed entire Linux distribution used for hosting production servers, +daily desktop use, and embedded programming on a Raspberry Pi. +Using NixOS, it supports enabling only required functionality at build time. +Features systemd, +an in-memory root filesystem, +ZFS persistent storage with backups, +and extremely hardened networking. + +*ICER Compressor* --- +Image compression library written in Rust, +designed for deep-space communication. +Hand tuned for speed and portability by using only integer arithmetic, +no heap allocations, and no standard library by default. +Achieves practically instant compressions, even on microprocessors. + +*Kubernetes Cluster* --- +Bare-metal compute cluster managed with GitOps to be completely reproducible. +Uses Cilium CNI for fast eBPF networking and BGP load balancing, +FluxCD for cluster management, +and CEPH distributed storage for stateful workloads. +Running Prometheus, Loki and AlertManager for complete observability. + +*Mirrorlist Generator* --- +Fetches Arch Linux package mirrors, +filtering them based on user parameters. +Sorts and outputs formatted data compliant with the pacman package manager. +Heavily outperforms default Python implementation. + +*Toronto Metropolitan Robotics* --- +Member of university design team working on space focused automated robotics. +Designed custom STM32 hardware with various interfaces (SPI, I2C, etc.) for controlling motor functions on robot. +Worked alongside various subteams to deliver a competition ready autonomous system. + +== Skills + +- *Languages*: #( + "Rust", + "C++", + "Nix", + "Haskell", + "TypeScript", + "Lua", + "Python", + ).join(", ") + +- *Technologies*: #( + "Linux", + "Compilers", + "Virtualisation", + "Terraform", + "Ansible", + "Computer Networks", + "Frontend (Svelte, Astro)" + ).join(", ") diff --git a/mod/misc/cv/default.nix b/mod/misc/cv/default.nix new file mode 100644 index 0000000..805d880 --- /dev/null +++ b/mod/misc/cv/default.nix @@ -0,0 +1,13 @@ +{ pkgs, ... }: pkgs.stdenvNoCC.mkDerivation { + pname = "cv"; + version = "1.0"; + + src = ./.; + + nativeBuildInputs = with pkgs; [ typst ]; + buildPhase = "typst compile cv.typ"; + installPhase = '' + mkdir -p $out/share + cp cv.pdf $out/share + ''; +} diff --git a/mod/misc/cv/template.typ b/mod/misc/cv/template.typ new file mode 100644 index 0000000..4b07fba --- /dev/null +++ b/mod/misc/cv/template.typ @@ -0,0 +1,211 @@ +#let resume( + author: "", + pronouns: "", + location: "", + email: "", + github: "", + linkedin: "", + phone: "", + personal-site: "", + accent-color: "#000000", + font: "New Computer Modern", + paper: "us-letter", + body, +) = { + + // Sets document metadata + set document(author: author, title: author) + + // Document-wide formatting, including font and margins + set text( + // LaTeX style font + font: font, + size: 10pt, + lang: "en", + // Disable ligatures so ATS systems do not get confused when parsing fonts. + ligatures: false + ) + + // Reccomended to have 0.5in margin on all sides + set page( + margin: (0.5in), + paper: paper, + ) + + // Link styles + show link: underline + + + // Small caps for section titles + show heading.where(level: 2): it => [ + #pad(top: 0pt, bottom: -10pt, [#smallcaps(it.body)]) + #line(length: 100%, stroke: 1pt) + ] + + // Accent Color Styling + show heading: set text( + fill: rgb(accent-color), + ) + + show link: set text( + fill: rgb(accent-color), + ) + + // Name will be aligned left, bold and big + show heading.where(level: 1): it => [ + #set align(left) + #set text( + weight: 700, + size: 20pt, + ) + #pad(it.body) + ] + + // Level 1 Heading + [= #(author)] + + // Personal Info Helper + let contact-item(value, prefix: "", link-type: "") = { + if value != "" { + if link-type != "" { + link(link-type + value)[#(prefix + value)] + } else { + value + } + } + } + + // Personal Info + pad( + top: 0.25em, + align(left)[ + #{ + let items = ( + contact-item(pronouns), + contact-item(phone), + contact-item(location), + contact-item(email, link-type: "mailto:"), + contact-item(github, link-type: "https://"), + contact-item(linkedin, link-type: "https://"), + contact-item(personal-site, link-type: "https://"), + ) + items.filter(x => x != none).join(" | ") + } + ], + ) + + // Main body. + set par(justify: true) + + body +} + +// Generic two by two component for resume +#let generic-two-by-two( + top-left: "", + top-right: "", + bottom-left: "", + bottom-right: "", +) = { + [ + #top-left #h(1fr) #top-right \ + #bottom-left #h(1fr) #bottom-right + ] +} + +// Generic one by two component for resume +#let generic-one-by-two( + left: "", + right: "", +) = { + [ + #left #h(1fr) #right + ] +} + +// Cannot just use normal --- ligature becuase ligatures are disabled for good reasons +#let dates-helper( + start-date: "", + end-date: "", +) = { + start-date + " " + $dash.em$ + " " + end-date +} + +// Section components below +#let edu( + institution: "", + dates: "", + degree: "", + gpa: "", + location: "", +) = { + generic-two-by-two( + top-left: strong(institution), + top-right: location, + bottom-left: emph(degree), + bottom-right: emph(dates), + ) +} + +#let work( + title: "", + dates: "", + company: "", + location: "", +) = { + generic-two-by-two( + top-left: strong(title), + top-right: dates, + bottom-left: company, + bottom-right: emph(location), + ) +} + +#let project( + role: "", + name: "", + url: "", + dates: "", +) = { + generic-one-by-two( + left: { + if role == "" { + [*#name* #if url != "" and dates != "" [ (#link("https://" + url)[#url])]] + } else { + [*#role*, #name #if url != "" and dates != "" [ (#link("https://" + url)[#url])]] + } + }, + right: { + if dates == "" and url != "" { + link("https://" + url)[#url] + } else { + dates + } + }, + ) +} + +#let certificates( + name: "", + issuer: "", + url: "", + date: "", +) = { + [ + *#name*, #issuer + #if url != "" { + [ (#link("https://" + url)[#url])] + } + #h(1fr) #date + ] +} + +#let extracurriculars( + activity: "", + dates: "", +) = { + generic-one-by-two( + left: strong(activity), + right: dates, + ) +} diff --git a/mod/nix/readTree/default.nix b/mod/nix/readTree/default.nix new file mode 100644 index 0000000..4a745ce --- /dev/null +++ b/mod/nix/readTree/default.nix @@ -0,0 +1,326 @@ +# Copyright (c) 2019 Vincent Ambo +# Copyright (c) 2020-2021 The TVL Authors +# SPDX-License-Identifier: MIT +# +# Provides a function to automatically read a filesystem structure +# into a Nix attribute set. +# +# Called with an attribute set taking the following arguments: +# +# path: Path to a directory from which to start reading the tree. +# +# args: Argument set to pass to each imported file. +# +# filter: Function to filter `args` based on the tree location. This should +# be a function of the form `args -> location -> args`, where the +# location is a list of strings representing the path components of +# the current readTree target. Optional. +{ ... }: + +let + inherit (builtins) + attrNames + concatMap + concatStringsSep + elem + elemAt + filter + hasAttr + head + isAttrs + listToAttrs + map + match + readDir + substring; + + argsWithPath = args: parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in + listToAttrs (map + (name: { + inherit name; + value = children.${name}; + }) + names); + + # Create a mark containing the location of this attribute and + # a list of all child attribute names added by readTree. + marker = parts: children: { + __readTree = parts; + __readTreeChildren = builtins.attrNames children; + }; + + # Create a label from a target's tree location. + mkLabel = target: + let label = concatStringsSep "/" target.__readTree; + in if target ? __subtarget + then "${label}:${target.__subtarget}" + else label; + + # Merge two attribute sets, but place attributes in `passthru` via + # `overrideAttrs` for derivation targets that support it. + merge = a: b: + if a ? overrideAttrs + then + a.overrideAttrs + (prev: { + passthru = (prev.passthru or { }) // b; + }) + else a // b; + + # Import a file and enforce our calling convention + importFile = args: scopedArgs: path: parts: filter: + let + importedFile = + if scopedArgs != { } && builtins ? scopedImport # For tvix + then builtins.scopedImport scopedArgs path + else import path; + pathType = builtins.typeOf importedFile; + in + if pathType != "lambda" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + else importedFile (filter parts (argsWithPath args parts)); + + nixFileName = file: + let res = match "(.*)\\.nix" file; + in if res == null then null else head res; + + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + let + dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + + joinChild = c: initPath + ("/" + c); + + self = + if rootDir + then { __readTree = [ ]; } + else importFile args scopedArgs initPath parts argsFilter; + + # Import subdirectories of the current one, unless any skip + # instructions exist. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + filteredChildren = map + (c: { + name = c; + value = readTreeImpl { + inherit argsFilter scopedArgs; + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) + (filter filterDir (attrNames dir)); + + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); + + # Import Nix files + nixFiles = + if skipSubtree then [ ] + else filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map + (c: + let + p = joinChild (c + ".nix"); + childParts = parts ++ [ c ]; + imported = importFile args scopedArgs p childParts argsFilter; + in + { + name = c; + value = + if isAttrs imported + then merge imported (marker childParts { }) + else imported; + }) + nixFiles; + + nodeValue = if dir ? "default.nix" then self else { }; + + allChildren = listToAttrs ( + if dir ? "default.nix" + then children + else nixChildren ++ children + ); + + in + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; + + # Helper function to fetch subtargets from a target. This is a + # temporary helper to warn on the use of the `meta.targets` + # attribute, which is deprecated in favour of `meta.ci.targets`. + subtargets = node: + let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]); + in if node ? meta.targets then + builtins.trace '' + [1;31mWarning: The meta.targets attribute is deprecated. + + Please move the subtargets of //${mkLabel node} to the + meta.ci.targets attribute. + [0m + '' + targets else targets; + + # Function which can be used to find all readTree targets within an + # attribute set. + # + # This function will gather physical targets, that is targets which + # correspond directly to a location in the repository, as well as + # subtargets (specified in the meta.ci.targets attribute of a node). + # + # This can be used to discover targets for inclusion in CI + # pipelines. + # + # Called with the arguments: + # + # eligible: Function to determine whether the given derivation + # should be included in the build. + gather = eligible: node: + if node ? __readTree then + # Include the node itself if it is eligible. + (if eligible node then [ node ] else [ ]) + # Include eligible children of the node + ++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren) + # Include specified sub-targets of the node + ++ filter eligible (map + (k: (node."${k}" or { }) // { + # Keep the same tree location, but explicitly mark this + # node as a subtarget. + __readTree = node.__readTree; + __readTreeChildren = [ ]; + __subtarget = k; + }) + (subtargets node)) + else [ ]; + + # Determine whether a given value is a derivation. + # Copied from nixpkgs/lib for cases where lib is not available yet. + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; +in +{ + inherit gather mkLabel; + + __functor = _: + { path + , args + , filter ? (_parts: x: x) + , scopedArgs ? { } + }: + readTree { + inherit args scopedArgs; + argsFilter = filter; + initPath = path; + rootDir = true; + parts = [ ]; + }; + + # In addition to readTree itself, some functionality is exposed that + # is useful for users of readTree. + + # Create a readTree filter disallowing access to the specified + # top-level folder in the repository, except for specific exceptions + # specified by their (full) paths. + # + # Called with the arguments: + # + # folder: Name of the restricted top-level folder (e.g. 'experimental') + # + # exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]), + # which should be able to access the restricted folder. + # + # reason: Textual explanation for the restriction (included in errors) + restrictFolder = { folder, exceptions ? [ ], reason }: parts: args: + if (elemAt parts 0) == folder || elem parts exceptions + then args + else args // { + depot = args.depot // { + "${folder}" = throw '' + Access to targets under //${folder} is not permitted from + other repository paths. Specific exceptions are configured + at the top-level. + + ${reason} + At location: ${builtins.concatStringsSep "." parts} + ''; + }; + }; + + # This definition of fix is identical to <nixpkgs>.lib.fix, but is + # provided here for cases where readTree is used before nixpkgs can + # be imported. + # + # It is often required to create the args attribute set. + fix = f: let x = f x; in x; + + # Takes an attribute set and adds a meta.ci.targets attribute to it + # which contains all direct children of the attribute set which are + # derivations. + # + # Type: attrs -> attrs + drvTargets = attrs: + attrs // { + # preserve .meta from original attrs + meta = (attrs.meta or { }) // { + # preserve .meta.ci (except .targets) from original attrs + ci = (attrs.meta.ci or { }) // { + targets = builtins.filter + (x: isDerivation attrs."${x}") + (builtins.attrNames attrs); + }; + }; + }; +} diff --git a/mod/web/blog/.gitignore b/mod/web/blog/.gitignore new file mode 100644 index 0000000..decc3f8 --- /dev/null +++ b/mod/web/blog/.gitignore @@ -0,0 +1,2 @@ +public/ +nohup.out diff --git a/mod/web/blog/config.toml b/mod/web/blog/config.toml new file mode 100644 index 0000000..69c389b --- /dev/null +++ b/mod/web/blog/config.toml @@ -0,0 +1,8 @@ +base_url = "https://4kb.net" +build_search_index = false +compile_sass = false +generate_feeds = true +minify_html = true + +[markdown] +highlight_code = false diff --git a/mod/web/blog/content/log/_index.md b/mod/web/blog/content/log/_index.md new file mode 100644 index 0000000..10f2311 --- /dev/null +++ b/mod/web/blog/content/log/_index.md @@ -0,0 +1,7 @@ ++++ +title = "Log" +sort_by = "date" +template = "blog.html" +page_template = "post.html" +description = "notes too short to post" ++++ diff --git a/mod/web/blog/content/log/deterministic-hostnames.md b/mod/web/blog/content/log/deterministic-hostnames.md new file mode 100644 index 0000000..43dc7d8 --- /dev/null +++ b/mod/web/blog/content/log/deterministic-hostnames.md @@ -0,0 +1,80 @@ +--- +title: "Deterministic and unique network hostnames" +date: "2024-11-10" +--- + +As part of building out a Kubernetes cluster, I wanted to build and distribute a +single OS image to create stateless worker nodes. Using network booting, and +some clever tricks to differentiate nodes, we can create a scaleable and +efficient farm of workers for a cluster that don't even need disks. + +The idea came from a plan to build a cluster using the +[compute blade](https://computeblade.com/), and a few Raspberry Pi SBCs I +already own. Running the cluster from an SD card is not recommended due to the +not-so-great reliability of the flash used by most manufacturers, so I wanted to +try PXE booting each Pi to save money rather than purchasing an SSD for each +one. The compute blades do support an NVMe disk, but I plan to use those for a +storage cluster later, so they need to remain empty. + +## Base image + +Alpine Linux has been my preferred server OS for a long time. It provides a very +lightweight base system, and bundles an excellent bootstrapping system, +[apkovl](https://wiki.alpinelinux.org/wiki/Alpine_local_backup), that allows the +user to save a set of customisations to an system as an overlay to a stock +Alpine live image. In other words, we can create our image once, save the +changes as an `apkovl.tar.gz` file, and apply the same changes to a base system +on boot. This file can even be provided as a +[kernel parameter](https://wiki.alpinelinux.org/wiki/PXE_boot#Guide_to_options) +and will be fetched from a remote webserver automatically! + +Since the image and configuration will be shipped to the node via the network, +an added benefit of using Alpine is its tiny space consumption. I'm not using +enough nodes for this to really matter, but it's a cool optimization regardless. + +## Differentiating the nodes + +One of the main goals of this project is that there should be no persistent +storage required outside the boot image itself. Since every node will download +and generate the same root file-system on startup, the first problem that arises +is how the nodes will identify themselves both on the network and the cluster, +given that it's not possible to name them ahead of time. In other words, any +given node has to generate a unique hostname that won't collide with other +workers, and that will be the same each time that node boots. + +Since these nodes will not have a predefined name, we have to rely on +characteristics of the hardware to differentiate each one. The hardware MAC +address is perfect for this, since it's unique to to each node and will not be +wiped away after the node reboots. On a system like Linux that exposes its +hardware through a _sysfs_, we can find a file containing the address at +`/sys/class/net/eth0/address`. I don't really like the idea of attaching the +literal MAC address of the node to its network hostname, since it's a security +risk, and a bit too verbose. Instead, we can transform it into something safer +using a `sha1sum`, which is already present on our Alpine base system: + +```console +sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' +``` + +### Applying the new name + +Ideally, the node should apply its generated hostname before reaching out for an +address over DHCP or joining the cluster. We can make sure it happens before any +traffic is sent out by adding a `pre-up` command to the right interface in +`/etc/network/interfaces`: + +``` + +... + +auto eth0 +iface eth0 inet dhcp + pre-up sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' > /etc/hostname + +... +``` + +The VM I tested with looks outputs `worker-e2fae8`. Pretty clean result, and if +you want to know the physical node that maps to each hostname, you can take note +of the MAC address beforehand and generate the same hash on another computer to +match them up. diff --git a/mod/web/blog/content/log/nohup.md b/mod/web/blog/content/log/nohup.md new file mode 100644 index 0000000..64f7983 --- /dev/null +++ b/mod/web/blog/content/log/nohup.md @@ -0,0 +1,26 @@ +--- +title: "Spawning background processes" +date: "2024-11-17" +--- + +Working in a terminal, +I often pair my editor with a background process watching files. +Before reaching for terminal multiplexers, +see if you can get away with simple tty job control. +Spawn the background process, +still attached to the terminal instance: + +``` +program args & +``` + +Also redirect its output to a file, +for when the process writes to the tty from the background: + +``` +nohup program args & +``` + +Extra reading: + +- <https://jvns.ca/blog/2024/07/03/reasons-to-use-job-control/> diff --git a/mod/web/blog/content/posts/_index.md b/mod/web/blog/content/posts/_index.md new file mode 100644 index 0000000..1f4024a --- /dev/null +++ b/mod/web/blog/content/posts/_index.md @@ -0,0 +1,8 @@ ++++ +title = "Posts" +sort_by = "date" +template = "blog.html" +page_template = "post.html" ++++ + +Longer form writing. diff --git a/mod/web/blog/content/posts/stateless-compute-networks.md b/mod/web/blog/content/posts/stateless-compute-networks.md new file mode 100644 index 0000000..bac9e5d --- /dev/null +++ b/mod/web/blog/content/posts/stateless-compute-networks.md @@ -0,0 +1,41 @@ +--- +title: "On stateless compute networks" +date: "2024-11-10" +draft: true +--- + +On the topic of distributed systems and clustering, +I am quite invested in the idea of compute nodes that rely entirely on the network for configuration. +Arbitrary nodes can join a pre-existing cluster, +offering their CPU time and memory for computation without relying on any pre-existing configuration on the node itself. +In other words, any computer could pick up work, +only needing power and a network connection to the cluster. + +Perhaps this eventually leads into a "self-healing" cluster where only one node is manually bootstrapped, +which then serves a _configuration endpoint_ for other stateless nodes to reach out to for their instructions, +which they will then also serve once they are themselves ready. + +Early revisions of these notes mention Kubernetes, +but I am also trying to achieve similar results with NixOS on a custom project. +In any case, these are my ever-updating notes towards a general implementation of a stateless distributed systems architecture. + +## Self healing cluster + +Assuming control of an external DHCP server, +a self healing Kubernetes cluster would be feasible, +with the PXE boot artifacts supplied by the cluster itself. +That is, as long as one node is running the pod hosting the artifacts on a given endpoint, +other nodes can boot those artifacts and join the cluster, +thereby being able to host the artifacts as well. + +## Configuration endpoint + +The nodes shouldn't require a disk installed to be able to join the network. +Rather, the lofty goal of zero-configuration compute nodes passes the job of node initialization to the supporting network. +This is accomplished with PXE boot instructions supplied over DHCP. + +I delegate the following tasks to a single node in the subnet: + +- Gateway: Optional outbound connections if required +- DHCP server: Cluster IPAM +- TFTP and HTTP server: Serves iPXE firmware and kernel/initrd artifacts diff --git a/mod/web/blog/default.nix b/mod/web/blog/default.nix new file mode 100644 index 0000000..5d76e7a --- /dev/null +++ b/mod/web/blog/default.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: pkgs.stdenvNoCC.mkDerivation { + pname = "4kb.net"; + version = "1.0"; + src = ./.; + + nativeBuildInputs = with pkgs; [ + zola + ]; + + buildPhase = "zola build"; + installPhase = '' + mkdir -p $out + cp -r public/* $out/ + ''; +} diff --git a/mod/web/blog/static/avatar.png b/mod/web/blog/static/avatar.png Binary files differnew file mode 100644 index 0000000..91357dc --- /dev/null +++ b/mod/web/blog/static/avatar.png diff --git a/mod/web/blog/static/style.css b/mod/web/blog/static/style.css new file mode 100644 index 0000000..fdfd692 --- /dev/null +++ b/mod/web/blog/static/style.css @@ -0,0 +1,85 @@ +:root { + --bg: #181616; + --fg: #c5c9c5; + --link: #76946a; + --linkhover: #98bb6c; +} + +html { + scroll-behavior: smooth; + color-scheme: dark; +} + +body { + font-family: serif; + padding: 0 0.75em; + max-width: 42em; + margin: auto; + background: var(--bg); + color: var(--fg); +} + +p { + line-height: 1.3em; +} + +th, +td { + padding: 0.2em 0.4em; + border: thin solid; +} + +table { + border: thin solid; + border-collapse: collapse; +} + +article { + padding-top: 1em; +} + +footer { + margin: 3em auto; + text-align: center; +} + +a, +a:link { + color: var(--link); +} + +a:hover { + color: var(--linkhover); +} + +a[href^="http"]:where(:not([href*="4kb.net/"]))::after { + content: " \21e2"; +} + +article img { + display: block; + margin: 0 auto; + max-width: 80%; + height: auto; + object-fit: contain; +} + +nav { + margin-top: 1em; + display: flex; + justify-content: space-between; + align-items: center; + font-family: monospace; + font-weight: bold; + gap: 8px; +} + +nav div { + display: flex; + gap: 10px; +} + +pre { + padding: 1em; + overflow-x: scroll; +} diff --git a/mod/web/blog/templates/404.html b/mod/web/blog/templates/404.html new file mode 100644 index 0000000..a4669df --- /dev/null +++ b/mod/web/blog/templates/404.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} + +{% block content %} +<head> + <title>Error 404!</title> +</head> + +404! +{% endblock content %} diff --git a/mod/web/blog/templates/base.html b/mod/web/blog/templates/base.html new file mode 100644 index 0000000..6757d02 --- /dev/null +++ b/mod/web/blog/templates/base.html @@ -0,0 +1,26 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <link href="/style.css" rel="stylesheet" /> + <link rel="preload" href="style.css" as="style" /> + <title>{% block title %}4kb.net{% endblock title %}</title> + </head> + <body> + <nav> + <span>4kb.net</span> + <div> + <a href="/">Home</a> + <a href="/posts">Posts</a> + <a href="/log">Log</a> + </div> + </nav> + <hr /> + <main>{% block content %} {% endblock %}</main> + <footer> + <hr /> + <span>© 2024 Kleidi Bujari</span> + </footer> + </body> +</html> diff --git a/mod/web/blog/templates/blog.html b/mod/web/blog/templates/blog.html new file mode 100644 index 0000000..6c92a07 --- /dev/null +++ b/mod/web/blog/templates/blog.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} + +{% block content %} + +<head> + <title>4kb.net</title> +</head> + +<h2>{{ section.title }}</h2> + +{{ section.content | safe }} + +<ul> + {% for page in section.pages %} + <li> + <span>[{{ page.date }}]</span> + <a href="{{ page.permalink | safe }}">{{ page.title }}</a> + </li> + {% endfor %} +</ul> + +{% endblock content %} diff --git a/mod/web/blog/templates/index.html b/mod/web/blog/templates/index.html new file mode 100644 index 0000000..74e9a0f --- /dev/null +++ b/mod/web/blog/templates/index.html @@ -0,0 +1,44 @@ +{% extends "base.html" %} {% block content %} + +<h2>welcome!</h2> + +<p> + I'm a computer engineering undergrad studying in Toronto. This is my personal + site where I sometimes publish long form posts on topics I'm interested in, or + short "logs" for information I want to remember. +</p> + +<span>Find me on:</span> +<ul> + <li>email: <a href="mailto:mail@4kb.net">mail@4kb.net</a></li> + <li>github: <a href="https://github.com/kbujari">kbujari</a></li> +</ul> + +<!--<h3>Recent Posts</h3>--> +<!----> +<!--{% set section = get_section(path="posts/_index.md") %}--> +<!----> +<!--<ul>--> +<!-- {% for page in section.pages | slice(end=4) %}--> +<!-- <li>--> +<!-- <span>[{{ page.date }}]</span>--> +<!-- <a href="{{ page.permalink | safe }}">{{ page.title }}</a>--> +<!-- </li>--> +<!-- {% endfor %}--> +<!--</ul>--> + +<h3>Recent Logs</h3> + +{% set section = get_section(path="log/_index.md") %} + +<ul> + {% for page in section.pages | slice(end=4) %} + <li> + <span>[{{ page.date }}]</span> + <a href="{{ page.permalink | safe }}">{{ page.title }}</a> + </li> + {% endfor %} +</ul> + + +{% endblock content %} diff --git a/mod/web/blog/templates/post.html b/mod/web/blog/templates/post.html new file mode 100644 index 0000000..b1981e4 --- /dev/null +++ b/mod/web/blog/templates/post.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block title%} + {{ page.title }} - {{ super() }} +{% endblock title %} + +{% block content %} + +<h1 style="text-align: center; margin-top: 1em;">{{ page.title }}</h1> +<article> + {{ page.content | safe }} +</article> +{% endblock content %} diff --git a/modules/flake-module.nix b/modules/flake-module.nix deleted file mode 100644 index 40ec4b4..0000000 --- a/modules/flake-module.nix +++ /dev/null @@ -1,6 +0,0 @@ -{ inputs, ... }: { - flake.nixosModules = { - xnet.imports = [ ./xnet inputs.disko.nixosModules.disko ]; - xlib.imports = [ ./xlib ]; - }; -} diff --git a/modules/xlib/default.nix b/modules/xlib/default.nix deleted file mode 100644 index c915eb0..0000000 --- a/modules/xlib/default.nix +++ /dev/null @@ -1 +0,0 @@ -{ ... }: { } |
