diff options
38 files changed, 1158 insertions, 115 deletions
@@ -21,26 +21,6 @@ "type": "github" } }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1730504689, - "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "506278e768c2a08bec68eb62932193e341f55c90", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1731755305, @@ -60,10 +40,8 @@ "root": { "inputs": { "disko": "disko", - "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", - "sops-nix": "sops-nix", - "sshKeys": "sshKeys" + "sops-nix": "sops-nix" } }, "sops-nix": { @@ -85,18 +63,6 @@ "repo": "sops-nix", "type": "github" } - }, - "sshKeys": { - "flake": false, - "locked": { - "narHash": "sha256-BEKwObEHFoq0UzQcZ0VJ6uVl3Cx2MVylPRkylReyDUk=", - "type": "file", - "url": "https://github.com/kbujari.keys" - }, - "original": { - "type": "file", - "url": "https://github.com/kbujari.keys" - } } }, "root": "root", @@ -4,32 +4,82 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; - flake-parts.url = "github:hercules-ci/flake-parts"; - flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - disko.url = "github:nix-community/disko/v1.6.1"; disko.inputs.nixpkgs.follows = "nixpkgs"; sops-nix.url = "github:Mic92/sops-nix"; sops-nix.inputs.nixpkgs.follows = "nixpkgs"; - - sshKeys = { - url = "https://github.com/kbujari.keys"; - flake = false; - }; }; - outputs = { flake-parts, ... } @ inputs: - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ "x86_64-linux" "aarch64-linux" ]; + outputs = { self, nixpkgs, ... } @ inputs: + let + inherit (builtins) + attrNames + filter + listToAttrs + map + readDir + ; - imports = [ - ./machines/flake-module.nix - ./modules/flake-module.nix - ]; + inherit (nixpkgs.lib) + nixosSystem + ; - perSystem = { pkgs, ... }: { - formatter = pkgs.nixpkgs-fmt; + readTree = import ./mod/nix/readTree { }; + + # Recursively converts subdirectories under ./mod into an attrset + # that can be passed to other parts of the depot. + readDepot = depotArgs: readTree { + args = depotArgs; + path = ./mod; }; + + # Named arguments to be made available to any nix expression in + # the depot module. + depot = readTree.fix (self: readDepot { + depot = self; + + # x86_64-linux is hardcoded as the package arch only for the + # mod directory. This workaround keeps the flake pure, + # at the expense of only supporting one system. This can + # be circumvented by retrieving the system during evaluation, + # but flakes disallow this by design. + # + # This monorepo currently depends on functionality from flakes, + # but this may change in the future, perhaps shifting to an + # impure base with certain packages exposed from the flake in a + # pure way. + # + # In practice, this means that any configuration or package + # exposed from the depot flake that makes use of internal + # derivations is limited to x86_64-linux. For now this is fine, + # but ideally any architecture supported by nix should be + # allowed to evaluate projects hosted here. + pkgs = nixpkgs.legacyPackages."x86_64-linux"; + + # Expose lib attribute to modules. + lib = nixpkgs.lib; + }); + + machines = filter (m: m != "iso" && m != "xnet") + (attrNames (readDir ./machines)); + in + { + inherit depot; + + nixosModules.xnet.imports = + [ ./machines/xnet inputs.disko.nixosModules.disko ]; + + nixosConfigurations = listToAttrs ( + map + (name: { + inherit name; + value = nixosSystem { + system = "x86_64-linux"; + specialArgs = { inherit inputs depot; }; + modules = [ ./machines/${name} self.nixosModules.xnet ]; + }; + }) + machines); }; } diff --git a/machines/flake-module.nix b/machines/flake-module.nix deleted file mode 100644 index 8d9dd2c..0000000 --- a/machines/flake-module.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ lib, self, ... }: -let - inherit (lib) nixosSystem; - inherit (self.nixosModules) xnet xlib; - inherit (self) inputs; -in -{ - flake.nixosConfigurations = { - t480 = nixosSystem { - system = "x86_64-linux"; - specialArgs = { inherit xlib; }; - modules = [ ./t480 xnet ]; - }; - - iso = nixosSystem { - system = "x86_64-linux"; - modules = [ ./iso ]; - }; - - t1 = nixosSystem { - system = "x86_64-linux"; - modules = [ ./t1 xnet ]; - }; - - iridium = nixosSystem { - system = "x86_64-linux"; - specialArgs = { inherit inputs; }; - modules = [ ./iridium xnet ]; - }; - }; - - perSystem = { pkgs, inputs', ... }: { - packages.xinstall = - let - disko = inputs'.disko.packages.disko-install; - - configs = builtins.concatStringsSep " " - (builtins.filter (x: x != "iso") - (builtins.attrNames self.nixosConfigurations)); - in - pkgs.writeShellScriptBin "xinstall" '' - set -euo pipefail - - FLAKE="github:kbujari/depot" - TARGET=$(${pkgs.gum}/bin/gum choose ${configs}) - - # ${disko}/bin/disko-install --help - ''; - }; -} diff --git a/machines/iridium/default.nix b/machines/iridium/default.nix index 8a3edd0..640554b 100644 --- a/machines/iridium/default.nix +++ b/machines/iridium/default.nix @@ -1,4 +1,11 @@ -{ inputs, lib, ... }: { +{ inputs, lib, ... }: +let + gitKeys = builtins.fetchurl { + url = "https://github.com/kbujari.keys"; + sha256 = "0fpa679zkrpx77vangzf3gnidwvmky8ifivn8411xx6albrikaqx"; + }; +in +{ system.stateVersion = "24.11"; users.mutableUsers = false; @@ -18,8 +25,7 @@ gitServer = { enable = true; gitweb.enable = true; - keys = builtins.filter (x: x != "") - (lib.strings.splitString "\n" (builtins.readFile inputs.sshKeys)); + keys = lib.splitString "\n" (builtins.readFile gitKeys); }; }; diff --git a/machines/xnet/README b/machines/xnet/README new file mode 100644 index 0000000..ab1c28c --- /dev/null +++ b/machines/xnet/README @@ -0,0 +1,18 @@ +xnet +==== + +Base configuration for any machines running a standard xnet +configuration. + +Notable features: + + - Root filesystem running in RAM + - Automated ZFS partitioning of disk + - Hardened SSH and web services + +Usage +===== + +The entire module is exposed from the top-level flake as a nixosModule, +but it changes too often to be considered stable. Options are made +available when imported under the 'xnet' attribute set. diff --git a/modules/xnet/default.nix b/machines/xnet/default.nix index a2fb336..5a6c587 100644 --- a/modules/xnet/default.nix +++ b/machines/xnet/default.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, pkgs, ... }: let inherit (lib) mkDefault; in @@ -18,7 +18,6 @@ in nix = { settings = { - allowUnfree = true; auto-optimise-store = true; experimental-features = [ "nix-command" "flakes" ]; warn-dirty = false; diff --git a/modules/xnet/desktop/default.nix b/machines/xnet/desktop/default.nix index 2eeb272..1e37497 100644 --- a/modules/xnet/desktop/default.nix +++ b/machines/xnet/desktop/default.nix @@ -19,7 +19,6 @@ in pulse.enable = true; alsa = { enable = true; - support32Bit = true; }; }; diff --git a/modules/xnet/disk.nix b/machines/xnet/disk.nix index caef61d..caef61d 100644 --- a/modules/xnet/disk.nix +++ b/machines/xnet/disk.nix diff --git a/modules/xnet/gitserver/default.nix b/machines/xnet/gitserver/default.nix index da83f73..da83f73 100644 --- a/modules/xnet/gitserver/default.nix +++ b/machines/xnet/gitserver/default.nix diff --git a/modules/xnet/gitserver/gitweb.nix b/machines/xnet/gitserver/gitweb.nix index bfd593d..bfd593d 100644 --- a/modules/xnet/gitserver/gitweb.nix +++ b/machines/xnet/gitserver/gitweb.nix diff --git a/modules/xnet/monitoring/default.nix b/machines/xnet/monitoring/default.nix index 87ae724..87ae724 100644 --- a/modules/xnet/monitoring/default.nix +++ b/machines/xnet/monitoring/default.nix diff --git a/modules/xnet/monitoring/grafana.nix b/machines/xnet/monitoring/grafana.nix index 6a37ce1..6a37ce1 100644 --- a/modules/xnet/monitoring/grafana.nix +++ b/machines/xnet/monitoring/grafana.nix diff --git a/modules/xnet/monitoring/prometheus.nix b/machines/xnet/monitoring/prometheus.nix index e32e078..e32e078 100644 --- a/modules/xnet/monitoring/prometheus.nix +++ b/machines/xnet/monitoring/prometheus.nix diff --git a/modules/xnet/net/default.nix b/machines/xnet/net/default.nix index 2255f53..2255f53 100644 --- a/modules/xnet/net/default.nix +++ b/machines/xnet/net/default.nix diff --git a/modules/xnet/net/sshd.nix b/machines/xnet/net/sshd.nix index ef225db..ef225db 100644 --- a/modules/xnet/net/sshd.nix +++ b/machines/xnet/net/sshd.nix diff --git a/modules/xnet/nginx.nix b/machines/xnet/nginx.nix index 64c1d65..64c1d65 100644 --- a/modules/xnet/nginx.nix +++ b/machines/xnet/nginx.nix diff --git a/modules/xnet/users.nix b/machines/xnet/users.nix index 53aa15b..53aa15b 100644 --- a/modules/xnet/users.nix +++ b/machines/xnet/users.nix diff --git a/mod/misc/cv/cv.typ b/mod/misc/cv/cv.typ new file mode 100644 index 0000000..e243796 --- /dev/null +++ b/mod/misc/cv/cv.typ @@ -0,0 +1,127 @@ +#import "template.typ": * + +#show: resume.with( + author: "Kleidi Bujari", + email: "mail@4kb.net", + github: "github.com/kbujari", + personal-site: "4kb.net", + paper: "us-letter", +) + +== Education + +#edu( + institution: "Toronto Metropolitan University", + dates: dates-helper(start-date: "Sep 2020", end-date: "Apr 2025"), + location: "Ontario, Canada", + degree: "Bachelor's of Engineering, Computer Engineering", +) + +- *Relevant Coursework*: + Data Structures, Embedded Programming, Compilers, Digital Systems, Computer Networks +- *Extracurriculars*: + Member of student robotics design team, + teaching assistant for micro-processor courses. + +== Experience + +#work( + company: "Toronto Metropolitan University", + title: "Graduate Research Assistant", + dates: dates-helper(start-date: "May 2024", end-date: "Sep 2024"), + location: "Toronto, Canada", +) + +- Implemented transformations for hundreds of media files, + using ffmpeg and unix primitives to parallelize workload. +- Designed frontend with AstroJS to generate only static HTML, + ensuring compatibility with many hosting providers. + +#work( + company: "Canadian Broadcasting Corporation", + title: "Network Engineering Intern", + dates: dates-helper(start-date: "May 2023", end-date: "Apr 2024"), + location: "Toronto, Canada", +) + +- Designed custom PXE-boot implementation for hundreds of devices using NetBox, + eliminating manual configuration. +- Configured Hyper-Converged Proxmox cluster for 2024 Olympics, + saving \$250k+ with reused hardware and open software. +- Deployed vendor-agnostic routing observability from scratch, + with Prometheus metrics and Grafana dashboards. +- Mentored junior application developers in modern C++23 programming, + aiding in performance design and memory safety. + +#work( + company: "WSP Canada", + title: "Student Engineer", + dates: dates-helper(start-date: "May", end-date: "Aug") + ", 2021, 2022", + location: "Toronto, Canada", +) + +- Contributed to subway car control systems with modern C++20, + replacing legacy code with newer STL functions. +- Extended internal distributed filesystem with support for deduplication and compression, + reclaiming 30% of storage. +- Validated new electrical designs for power consumption, + cost efficiency, and viability with existing systems. +- Participated in reviewing and adjusting large scale electrical and structural engineering designs. + +== Projects + +*Custom Linux Distribution* --- +Designed entire Linux distribution used for hosting production servers, +daily desktop use, and embedded programming on a Raspberry Pi. +Using NixOS, it supports enabling only required functionality at build time. +Features systemd, +an in-memory root filesystem, +ZFS persistent storage with backups, +and extremely hardened networking. + +*ICER Compressor* --- +Image compression library written in Rust, +designed for deep-space communication. +Hand tuned for speed and portability by using only integer arithmetic, +no heap allocations, and no standard library by default. +Achieves practically instant compressions, even on microprocessors. + +*Kubernetes Cluster* --- +Bare-metal compute cluster managed with GitOps to be completely reproducible. +Uses Cilium CNI for fast eBPF networking and BGP load balancing, +FluxCD for cluster management, +and CEPH distributed storage for stateful workloads. +Running Prometheus, Loki and AlertManager for complete observability. + +*Mirrorlist Generator* --- +Fetches Arch Linux package mirrors, +filtering them based on user parameters. +Sorts and outputs formatted data compliant with the pacman package manager. +Heavily outperforms default Python implementation. + +*Toronto Metropolitan Robotics* --- +Member of university design team working on space focused automated robotics. +Designed custom STM32 hardware with various interfaces (SPI, I2C, etc.) for controlling motor functions on robot. +Worked alongside various subteams to deliver a competition ready autonomous system. + +== Skills + +- *Languages*: #( + "Rust", + "C++", + "Nix", + "Haskell", + "TypeScript", + "Lua", + "Python", + ).join(", ") + +- *Technologies*: #( + "Linux", + "Compilers", + "Virtualisation", + "Terraform", + "Ansible", + "Computer Networks", + "Frontend (Svelte, Astro)" + ).join(", ") diff --git a/mod/misc/cv/default.nix b/mod/misc/cv/default.nix new file mode 100644 index 0000000..805d880 --- /dev/null +++ b/mod/misc/cv/default.nix @@ -0,0 +1,13 @@ +{ pkgs, ... }: pkgs.stdenvNoCC.mkDerivation { + pname = "cv"; + version = "1.0"; + + src = ./.; + + nativeBuildInputs = with pkgs; [ typst ]; + buildPhase = "typst compile cv.typ"; + installPhase = '' + mkdir -p $out/share + cp cv.pdf $out/share + ''; +} diff --git a/mod/misc/cv/template.typ b/mod/misc/cv/template.typ new file mode 100644 index 0000000..4b07fba --- /dev/null +++ b/mod/misc/cv/template.typ @@ -0,0 +1,211 @@ +#let resume( + author: "", + pronouns: "", + location: "", + email: "", + github: "", + linkedin: "", + phone: "", + personal-site: "", + accent-color: "#000000", + font: "New Computer Modern", + paper: "us-letter", + body, +) = { + + // Sets document metadata + set document(author: author, title: author) + + // Document-wide formatting, including font and margins + set text( + // LaTeX style font + font: font, + size: 10pt, + lang: "en", + // Disable ligatures so ATS systems do not get confused when parsing fonts. + ligatures: false + ) + + // Reccomended to have 0.5in margin on all sides + set page( + margin: (0.5in), + paper: paper, + ) + + // Link styles + show link: underline + + + // Small caps for section titles + show heading.where(level: 2): it => [ + #pad(top: 0pt, bottom: -10pt, [#smallcaps(it.body)]) + #line(length: 100%, stroke: 1pt) + ] + + // Accent Color Styling + show heading: set text( + fill: rgb(accent-color), + ) + + show link: set text( + fill: rgb(accent-color), + ) + + // Name will be aligned left, bold and big + show heading.where(level: 1): it => [ + #set align(left) + #set text( + weight: 700, + size: 20pt, + ) + #pad(it.body) + ] + + // Level 1 Heading + [= #(author)] + + // Personal Info Helper + let contact-item(value, prefix: "", link-type: "") = { + if value != "" { + if link-type != "" { + link(link-type + value)[#(prefix + value)] + } else { + value + } + } + } + + // Personal Info + pad( + top: 0.25em, + align(left)[ + #{ + let items = ( + contact-item(pronouns), + contact-item(phone), + contact-item(location), + contact-item(email, link-type: "mailto:"), + contact-item(github, link-type: "https://"), + contact-item(linkedin, link-type: "https://"), + contact-item(personal-site, link-type: "https://"), + ) + items.filter(x => x != none).join(" | ") + } + ], + ) + + // Main body. + set par(justify: true) + + body +} + +// Generic two by two component for resume +#let generic-two-by-two( + top-left: "", + top-right: "", + bottom-left: "", + bottom-right: "", +) = { + [ + #top-left #h(1fr) #top-right \ + #bottom-left #h(1fr) #bottom-right + ] +} + +// Generic one by two component for resume +#let generic-one-by-two( + left: "", + right: "", +) = { + [ + #left #h(1fr) #right + ] +} + +// Cannot just use normal --- ligature becuase ligatures are disabled for good reasons +#let dates-helper( + start-date: "", + end-date: "", +) = { + start-date + " " + $dash.em$ + " " + end-date +} + +// Section components below +#let edu( + institution: "", + dates: "", + degree: "", + gpa: "", + location: "", +) = { + generic-two-by-two( + top-left: strong(institution), + top-right: location, + bottom-left: emph(degree), + bottom-right: emph(dates), + ) +} + +#let work( + title: "", + dates: "", + company: "", + location: "", +) = { + generic-two-by-two( + top-left: strong(title), + top-right: dates, + bottom-left: company, + bottom-right: emph(location), + ) +} + +#let project( + role: "", + name: "", + url: "", + dates: "", +) = { + generic-one-by-two( + left: { + if role == "" { + [*#name* #if url != "" and dates != "" [ (#link("https://" + url)[#url])]] + } else { + [*#role*, #name #if url != "" and dates != "" [ (#link("https://" + url)[#url])]] + } + }, + right: { + if dates == "" and url != "" { + link("https://" + url)[#url] + } else { + dates + } + }, + ) +} + +#let certificates( + name: "", + issuer: "", + url: "", + date: "", +) = { + [ + *#name*, #issuer + #if url != "" { + [ (#link("https://" + url)[#url])] + } + #h(1fr) #date + ] +} + +#let extracurriculars( + activity: "", + dates: "", +) = { + generic-one-by-two( + left: strong(activity), + right: dates, + ) +} diff --git a/mod/nix/readTree/default.nix b/mod/nix/readTree/default.nix new file mode 100644 index 0000000..4a745ce --- /dev/null +++ b/mod/nix/readTree/default.nix @@ -0,0 +1,326 @@ +# Copyright (c) 2019 Vincent Ambo +# Copyright (c) 2020-2021 The TVL Authors +# SPDX-License-Identifier: MIT +# +# Provides a function to automatically read a filesystem structure +# into a Nix attribute set. +# +# Called with an attribute set taking the following arguments: +# +# path: Path to a directory from which to start reading the tree. +# +# args: Argument set to pass to each imported file. +# +# filter: Function to filter `args` based on the tree location. This should +# be a function of the form `args -> location -> args`, where the +# location is a list of strings representing the path components of +# the current readTree target. Optional. +{ ... }: + +let + inherit (builtins) + attrNames + concatMap + concatStringsSep + elem + elemAt + filter + hasAttr + head + isAttrs + listToAttrs + map + match + readDir + substring; + + argsWithPath = args: parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in + listToAttrs (map + (name: { + inherit name; + value = children.${name}; + }) + names); + + # Create a mark containing the location of this attribute and + # a list of all child attribute names added by readTree. + marker = parts: children: { + __readTree = parts; + __readTreeChildren = builtins.attrNames children; + }; + + # Create a label from a target's tree location. + mkLabel = target: + let label = concatStringsSep "/" target.__readTree; + in if target ? __subtarget + then "${label}:${target.__subtarget}" + else label; + + # Merge two attribute sets, but place attributes in `passthru` via + # `overrideAttrs` for derivation targets that support it. + merge = a: b: + if a ? overrideAttrs + then + a.overrideAttrs + (prev: { + passthru = (prev.passthru or { }) // b; + }) + else a // b; + + # Import a file and enforce our calling convention + importFile = args: scopedArgs: path: parts: filter: + let + importedFile = + if scopedArgs != { } && builtins ? scopedImport # For tvix + then builtins.scopedImport scopedArgs path + else import path; + pathType = builtins.typeOf importedFile; + in + if pathType != "lambda" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + else importedFile (filter parts (argsWithPath args parts)); + + nixFileName = file: + let res = match "(.*)\\.nix" file; + in if res == null then null else head res; + + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + let + dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + + joinChild = c: initPath + ("/" + c); + + self = + if rootDir + then { __readTree = [ ]; } + else importFile args scopedArgs initPath parts argsFilter; + + # Import subdirectories of the current one, unless any skip + # instructions exist. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + filteredChildren = map + (c: { + name = c; + value = readTreeImpl { + inherit argsFilter scopedArgs; + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) + (filter filterDir (attrNames dir)); + + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); + + # Import Nix files + nixFiles = + if skipSubtree then [ ] + else filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map + (c: + let + p = joinChild (c + ".nix"); + childParts = parts ++ [ c ]; + imported = importFile args scopedArgs p childParts argsFilter; + in + { + name = c; + value = + if isAttrs imported + then merge imported (marker childParts { }) + else imported; + }) + nixFiles; + + nodeValue = if dir ? "default.nix" then self else { }; + + allChildren = listToAttrs ( + if dir ? "default.nix" + then children + else nixChildren ++ children + ); + + in + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; + + # Helper function to fetch subtargets from a target. This is a + # temporary helper to warn on the use of the `meta.targets` + # attribute, which is deprecated in favour of `meta.ci.targets`. + subtargets = node: + let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]); + in if node ? meta.targets then + builtins.trace '' + [1;31mWarning: The meta.targets attribute is deprecated. + + Please move the subtargets of //${mkLabel node} to the + meta.ci.targets attribute. + [0m + '' + targets else targets; + + # Function which can be used to find all readTree targets within an + # attribute set. + # + # This function will gather physical targets, that is targets which + # correspond directly to a location in the repository, as well as + # subtargets (specified in the meta.ci.targets attribute of a node). + # + # This can be used to discover targets for inclusion in CI + # pipelines. + # + # Called with the arguments: + # + # eligible: Function to determine whether the given derivation + # should be included in the build. + gather = eligible: node: + if node ? __readTree then + # Include the node itself if it is eligible. + (if eligible node then [ node ] else [ ]) + # Include eligible children of the node + ++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren) + # Include specified sub-targets of the node + ++ filter eligible (map + (k: (node."${k}" or { }) // { + # Keep the same tree location, but explicitly mark this + # node as a subtarget. + __readTree = node.__readTree; + __readTreeChildren = [ ]; + __subtarget = k; + }) + (subtargets node)) + else [ ]; + + # Determine whether a given value is a derivation. + # Copied from nixpkgs/lib for cases where lib is not available yet. + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; +in +{ + inherit gather mkLabel; + + __functor = _: + { path + , args + , filter ? (_parts: x: x) + , scopedArgs ? { } + }: + readTree { + inherit args scopedArgs; + argsFilter = filter; + initPath = path; + rootDir = true; + parts = [ ]; + }; + + # In addition to readTree itself, some functionality is exposed that + # is useful for users of readTree. + + # Create a readTree filter disallowing access to the specified + # top-level folder in the repository, except for specific exceptions + # specified by their (full) paths. + # + # Called with the arguments: + # + # folder: Name of the restricted top-level folder (e.g. 'experimental') + # + # exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]), + # which should be able to access the restricted folder. + # + # reason: Textual explanation for the restriction (included in errors) + restrictFolder = { folder, exceptions ? [ ], reason }: parts: args: + if (elemAt parts 0) == folder || elem parts exceptions + then args + else args // { + depot = args.depot // { + "${folder}" = throw '' + Access to targets under //${folder} is not permitted from + other repository paths. Specific exceptions are configured + at the top-level. + + ${reason} + At location: ${builtins.concatStringsSep "." parts} + ''; + }; + }; + + # This definition of fix is identical to <nixpkgs>.lib.fix, but is + # provided here for cases where readTree is used before nixpkgs can + # be imported. + # + # It is often required to create the args attribute set. + fix = f: let x = f x; in x; + + # Takes an attribute set and adds a meta.ci.targets attribute to it + # which contains all direct children of the attribute set which are + # derivations. + # + # Type: attrs -> attrs + drvTargets = attrs: + attrs // { + # preserve .meta from original attrs + meta = (attrs.meta or { }) // { + # preserve .meta.ci (except .targets) from original attrs + ci = (attrs.meta.ci or { }) // { + targets = builtins.filter + (x: isDerivation attrs."${x}") + (builtins.attrNames attrs); + }; + }; + }; +} diff --git a/mod/web/blog/.gitignore b/mod/web/blog/.gitignore new file mode 100644 index 0000000..decc3f8 --- /dev/null +++ b/mod/web/blog/.gitignore @@ -0,0 +1,2 @@ +public/ +nohup.out diff --git a/mod/web/blog/config.toml b/mod/web/blog/config.toml new file mode 100644 index 0000000..69c389b --- /dev/null +++ b/mod/web/blog/config.toml @@ -0,0 +1,8 @@ +base_url = "https://4kb.net" +build_search_index = false +compile_sass = false +generate_feeds = true +minify_html = true + +[markdown] +highlight_code = false diff --git a/mod/web/blog/content/log/_index.md b/mod/web/blog/content/log/_index.md new file mode 100644 index 0000000..10f2311 --- /dev/null +++ b/mod/web/blog/content/log/_index.md @@ -0,0 +1,7 @@ ++++ +title = "Log" +sort_by = "date" +template = "blog.html" +page_template = "post.html" +description = "notes too short to post" ++++ diff --git a/mod/web/blog/content/log/deterministic-hostnames.md b/mod/web/blog/content/log/deterministic-hostnames.md new file mode 100644 index 0000000..43dc7d8 --- /dev/null +++ b/mod/web/blog/content/log/deterministic-hostnames.md @@ -0,0 +1,80 @@ +--- +title: "Deterministic and unique network hostnames" +date: "2024-11-10" +--- + +As part of building out a Kubernetes cluster, I wanted to build and distribute a +single OS image to create stateless worker nodes. Using network booting, and +some clever tricks to differentiate nodes, we can create a scaleable and +efficient farm of workers for a cluster that don't even need disks. + +The idea came from a plan to build a cluster using the +[compute blade](https://computeblade.com/), and a few Raspberry Pi SBCs I +already own. Running the cluster from an SD card is not recommended due to the +not-so-great reliability of the flash used by most manufacturers, so I wanted to +try PXE booting each Pi to save money rather than purchasing an SSD for each +one. The compute blades do support an NVMe disk, but I plan to use those for a +storage cluster later, so they need to remain empty. + +## Base image + +Alpine Linux has been my preferred server OS for a long time. It provides a very +lightweight base system, and bundles an excellent bootstrapping system, +[apkovl](https://wiki.alpinelinux.org/wiki/Alpine_local_backup), that allows the +user to save a set of customisations to an system as an overlay to a stock +Alpine live image. In other words, we can create our image once, save the +changes as an `apkovl.tar.gz` file, and apply the same changes to a base system +on boot. This file can even be provided as a +[kernel parameter](https://wiki.alpinelinux.org/wiki/PXE_boot#Guide_to_options) +and will be fetched from a remote webserver automatically! + +Since the image and configuration will be shipped to the node via the network, +an added benefit of using Alpine is its tiny space consumption. I'm not using +enough nodes for this to really matter, but it's a cool optimization regardless. + +## Differentiating the nodes + +One of the main goals of this project is that there should be no persistent +storage required outside the boot image itself. Since every node will download +and generate the same root file-system on startup, the first problem that arises +is how the nodes will identify themselves both on the network and the cluster, +given that it's not possible to name them ahead of time. In other words, any +given node has to generate a unique hostname that won't collide with other +workers, and that will be the same each time that node boots. + +Since these nodes will not have a predefined name, we have to rely on +characteristics of the hardware to differentiate each one. The hardware MAC +address is perfect for this, since it's unique to to each node and will not be +wiped away after the node reboots. On a system like Linux that exposes its +hardware through a _sysfs_, we can find a file containing the address at +`/sys/class/net/eth0/address`. I don't really like the idea of attaching the +literal MAC address of the node to its network hostname, since it's a security +risk, and a bit too verbose. Instead, we can transform it into something safer +using a `sha1sum`, which is already present on our Alpine base system: + +```console +sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' +``` + +### Applying the new name + +Ideally, the node should apply its generated hostname before reaching out for an +address over DHCP or joining the cluster. We can make sure it happens before any +traffic is sent out by adding a `pre-up` command to the right interface in +`/etc/network/interfaces`: + +``` + +... + +auto eth0 +iface eth0 inet dhcp + pre-up sha1sum /sys/class/net/eth0/address | head -c 6 | awk '{print "worker-" $0}' > /etc/hostname + +... +``` + +The VM I tested with looks outputs `worker-e2fae8`. Pretty clean result, and if +you want to know the physical node that maps to each hostname, you can take note +of the MAC address beforehand and generate the same hash on another computer to +match them up. diff --git a/mod/web/blog/content/log/nohup.md b/mod/web/blog/content/log/nohup.md new file mode 100644 index 0000000..64f7983 --- /dev/null +++ b/mod/web/blog/content/log/nohup.md @@ -0,0 +1,26 @@ +--- +title: "Spawning background processes" +date: "2024-11-17" +--- + +Working in a terminal, +I often pair my editor with a background process watching files. +Before reaching for terminal multiplexers, +see if you can get away with simple tty job control. +Spawn the background process, +still attached to the terminal instance: + +``` +program args & +``` + +Also redirect its output to a file, +for when the process writes to the tty from the background: + +``` +nohup program args & +``` + +Extra reading: + +- <https://jvns.ca/blog/2024/07/03/reasons-to-use-job-control/> diff --git a/mod/web/blog/content/posts/_index.md b/mod/web/blog/content/posts/_index.md new file mode 100644 index 0000000..1f4024a --- /dev/null +++ b/mod/web/blog/content/posts/_index.md @@ -0,0 +1,8 @@ ++++ +title = "Posts" +sort_by = "date" +template = "blog.html" +page_template = "post.html" ++++ + +Longer form writing. diff --git a/mod/web/blog/content/posts/stateless-compute-networks.md b/mod/web/blog/content/posts/stateless-compute-networks.md new file mode 100644 index 0000000..bac9e5d --- /dev/null +++ b/mod/web/blog/content/posts/stateless-compute-networks.md @@ -0,0 +1,41 @@ +--- +title: "On stateless compute networks" +date: "2024-11-10" +draft: true +--- + +On the topic of distributed systems and clustering, +I am quite invested in the idea of compute nodes that rely entirely on the network for configuration. +Arbitrary nodes can join a pre-existing cluster, +offering their CPU time and memory for computation without relying on any pre-existing configuration on the node itself. +In other words, any computer could pick up work, +only needing power and a network connection to the cluster. + +Perhaps this eventually leads into a "self-healing" cluster where only one node is manually bootstrapped, +which then serves a _configuration endpoint_ for other stateless nodes to reach out to for their instructions, +which they will then also serve once they are themselves ready. + +Early revisions of these notes mention Kubernetes, +but I am also trying to achieve similar results with NixOS on a custom project. +In any case, these are my ever-updating notes towards a general implementation of a stateless distributed systems architecture. + +## Self healing cluster + +Assuming control of an external DHCP server, +a self healing Kubernetes cluster would be feasible, +with the PXE boot artifacts supplied by the cluster itself. +That is, as long as one node is running the pod hosting the artifacts on a given endpoint, +other nodes can boot those artifacts and join the cluster, +thereby being able to host the artifacts as well. + +## Configuration endpoint + +The nodes shouldn't require a disk installed to be able to join the network. +Rather, the lofty goal of zero-configuration compute nodes passes the job of node initialization to the supporting network. +This is accomplished with PXE boot instructions supplied over DHCP. + +I delegate the following tasks to a single node in the subnet: + +- Gateway: Optional outbound connections if required +- DHCP server: Cluster IPAM +- TFTP and HTTP server: Serves iPXE firmware and kernel/initrd artifacts diff --git a/mod/web/blog/default.nix b/mod/web/blog/default.nix new file mode 100644 index 0000000..5d76e7a --- /dev/null +++ b/mod/web/blog/default.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: pkgs.stdenvNoCC.mkDerivation { + pname = "4kb.net"; + version = "1.0"; + src = ./.; + + nativeBuildInputs = with pkgs; [ + zola + ]; + + buildPhase = "zola build"; + installPhase = '' + mkdir -p $out + cp -r public/* $out/ + ''; +} diff --git a/mod/web/blog/static/avatar.png b/mod/web/blog/static/avatar.png Binary files differnew file mode 100644 index 0000000..91357dc --- /dev/null +++ b/mod/web/blog/static/avatar.png diff --git a/mod/web/blog/static/style.css b/mod/web/blog/static/style.css new file mode 100644 index 0000000..fdfd692 --- /dev/null +++ b/mod/web/blog/static/style.css @@ -0,0 +1,85 @@ +:root { + --bg: #181616; + --fg: #c5c9c5; + --link: #76946a; + --linkhover: #98bb6c; +} + +html { + scroll-behavior: smooth; + color-scheme: dark; +} + +body { + font-family: serif; + padding: 0 0.75em; + max-width: 42em; + margin: auto; + background: var(--bg); + color: var(--fg); +} + +p { + line-height: 1.3em; +} + +th, +td { + padding: 0.2em 0.4em; + border: thin solid; +} + +table { + border: thin solid; + border-collapse: collapse; +} + +article { + padding-top: 1em; +} + +footer { + margin: 3em auto; + text-align: center; +} + +a, +a:link { + color: var(--link); +} + +a:hover { + color: var(--linkhover); +} + +a[href^="http"]:where(:not([href*="4kb.net/"]))::after { + content: " \21e2"; +} + +article img { + display: block; + margin: 0 auto; + max-width: 80%; + height: auto; + object-fit: contain; +} + +nav { + margin-top: 1em; + display: flex; + justify-content: space-between; + align-items: center; + font-family: monospace; + font-weight: bold; + gap: 8px; +} + +nav div { + display: flex; + gap: 10px; +} + +pre { + padding: 1em; + overflow-x: scroll; +} diff --git a/mod/web/blog/templates/404.html b/mod/web/blog/templates/404.html new file mode 100644 index 0000000..a4669df --- /dev/null +++ b/mod/web/blog/templates/404.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} + +{% block content %} +<head> + <title>Error 404!</title> +</head> + +404! +{% endblock content %} diff --git a/mod/web/blog/templates/base.html b/mod/web/blog/templates/base.html new file mode 100644 index 0000000..6757d02 --- /dev/null +++ b/mod/web/blog/templates/base.html @@ -0,0 +1,26 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <link href="/style.css" rel="stylesheet" /> + <link rel="preload" href="style.css" as="style" /> + <title>{% block title %}4kb.net{% endblock title %}</title> + </head> + <body> + <nav> + <span>4kb.net</span> + <div> + <a href="/">Home</a> + <a href="/posts">Posts</a> + <a href="/log">Log</a> + </div> + </nav> + <hr /> + <main>{% block content %} {% endblock %}</main> + <footer> + <hr /> + <span>© 2024 Kleidi Bujari</span> + </footer> + </body> +</html> diff --git a/mod/web/blog/templates/blog.html b/mod/web/blog/templates/blog.html new file mode 100644 index 0000000..6c92a07 --- /dev/null +++ b/mod/web/blog/templates/blog.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} + +{% block content %} + +<head> + <title>4kb.net</title> +</head> + +<h2>{{ section.title }}</h2> + +{{ section.content | safe }} + +<ul> + {% for page in section.pages %} + <li> + <span>[{{ page.date }}]</span> + <a href="{{ page.permalink | safe }}">{{ page.title }}</a> + </li> + {% endfor %} +</ul> + +{% endblock content %} diff --git a/mod/web/blog/templates/index.html b/mod/web/blog/templates/index.html new file mode 100644 index 0000000..74e9a0f --- /dev/null +++ b/mod/web/blog/templates/index.html @@ -0,0 +1,44 @@ +{% extends "base.html" %} {% block content %} + +<h2>welcome!</h2> + +<p> + I'm a computer engineering undergrad studying in Toronto. This is my personal + site where I sometimes publish long form posts on topics I'm interested in, or + short "logs" for information I want to remember. +</p> + +<span>Find me on:</span> +<ul> + <li>email: <a href="mailto:mail@4kb.net">mail@4kb.net</a></li> + <li>github: <a href="https://github.com/kbujari">kbujari</a></li> +</ul> + +<!--<h3>Recent Posts</h3>--> +<!----> +<!--{% set section = get_section(path="posts/_index.md") %}--> +<!----> +<!--<ul>--> +<!-- {% for page in section.pages | slice(end=4) %}--> +<!-- <li>--> +<!-- <span>[{{ page.date }}]</span>--> +<!-- <a href="{{ page.permalink | safe }}">{{ page.title }}</a>--> +<!-- </li>--> +<!-- {% endfor %}--> +<!--</ul>--> + +<h3>Recent Logs</h3> + +{% set section = get_section(path="log/_index.md") %} + +<ul> + {% for page in section.pages | slice(end=4) %} + <li> + <span>[{{ page.date }}]</span> + <a href="{{ page.permalink | safe }}">{{ page.title }}</a> + </li> + {% endfor %} +</ul> + + +{% endblock content %} diff --git a/mod/web/blog/templates/post.html b/mod/web/blog/templates/post.html new file mode 100644 index 0000000..b1981e4 --- /dev/null +++ b/mod/web/blog/templates/post.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block title%} + {{ page.title }} - {{ super() }} +{% endblock title %} + +{% block content %} + +<h1 style="text-align: center; margin-top: 1em;">{{ page.title }}</h1> +<article> + {{ page.content | safe }} +</article> +{% endblock content %} diff --git a/modules/flake-module.nix b/modules/flake-module.nix deleted file mode 100644 index 40ec4b4..0000000 --- a/modules/flake-module.nix +++ /dev/null @@ -1,6 +0,0 @@ -{ inputs, ... }: { - flake.nixosModules = { - xnet.imports = [ ./xnet inputs.disko.nixosModules.disko ]; - xlib.imports = [ ./xlib ]; - }; -} diff --git a/modules/xlib/default.nix b/modules/xlib/default.nix deleted file mode 100644 index c915eb0..0000000 --- a/modules/xlib/default.nix +++ /dev/null @@ -1 +0,0 @@ -{ ... }: { } |
