From 47c29610330b551ed78d0ebe048ed703e81e65ca Mon Sep 17 00:00:00 2001 From: "Geir O. Jerstad" Date: Thu, 3 Jul 2025 17:45:34 +0200 Subject: [PATCH] Refactor emacs configuration and clean up lab-tool project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reorganized emacs configuration with profiles in modules/development/emacs.nix - Updated machine configurations to use new emacs module structure - Cleaned up lab-tool project by removing archive, research, testing, and utils directories - Streamlined lab-tool to focus on core deployment functionality with deploy-rs - Added DEVELOPMENT.md documentation for lab-tool ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- dotfiles/geir/emacs-config/init-nix.el | 20 +- .../geir/emacs-config/modules/claude-code.el | 4 +- .../congenital-optimist/configuration.nix | 2 +- machines/grey-area/configuration.nix | 2 +- machines/little-rascal/configuration.nix | 4 +- machines/reverse-proxy/configuration.nix | 2 +- machines/sleeper-service/configuration.nix | 2 +- modules/common/emacs.nix | 20 - modules/development/emacs.nix | 72 +- modules/users/geir.nix | 2 - packages/lab-tool/DEVELOPMENT.md | 148 +++ packages/lab-tool/PROJECT_STATUS.md | 60 -- packages/lab-tool/REFACTORING_SUMMARY.md | 119 --- packages/lab-tool/TODO.md | 35 - packages/lab-tool/archive/core/health.scm | 75 -- packages/lab-tool/archive/core/logging.scm | 29 - packages/lab-tool/archive/core/ssh.scm | 24 - packages/lab-tool/archive/core/status.scm | 84 -- packages/lab-tool/archive/core/utils.scm | 12 - .../archive/deployment/deployment.scm | 109 --- packages/lab-tool/default.nix | 2 +- packages/lab-tool/lab/deployment.scm | 8 +- packages/lab-tool/research/core.scm | 326 ------- packages/lab-tool/research/deployment.scm | 329 ------- .../lab-tool/research/guile-mcp-server.scm | 348 ------- packages/lab-tool/research/guile.md | 846 ------------------ packages/lab-tool/research/guile_ecosystem.md | 394 -------- .../research/guile_scripting_solution.md | 334 ------- packages/lab-tool/research/home-lab-tool.scm | 74 -- packages/lab-tool/research/machines.scm | 258 ------ packages/lab-tool/research/monitoring.scm | 337 ------- packages/lab-tool/testing/README.md | 48 - .../lab-tool/testing/final-verification.scm | 45 - packages/lab-tool/testing/tdd-summary.scm | 36 - packages/lab-tool/testing/test-deployment.scm | 67 -- .../testing/test-final-validation.scm | 77 -- .../lab-tool/testing/test-functionality.scm | 24 - .../lab-tool/testing/test-implementation.scm | 72 -- .../lab-tool/testing/test-integration.scm | 121 --- packages/lab-tool/testing/test-main.scm | 59 -- .../testing/test-missing-functions.scm | 73 -- packages/lab-tool/testing/test-modular.scm | 43 - .../lab-tool/testing/test-modules-simple.scm | 63 -- packages/lab-tool/utils/config-new.scm | 43 - packages/lab-tool/utils/config/accessor.scm | 74 -- packages/lab-tool/utils/config/defaults.scm | 35 - packages/lab-tool/utils/config/loader.scm | 60 -- packages/lab-tool/utils/config/state.scm | 69 -- packages/lab-tool/utils/json-new.scm | 48 - packages/lab-tool/utils/json/file-io.scm | 57 -- packages/lab-tool/utils/json/manipulation.scm | 63 -- packages/lab-tool/utils/json/parse.scm | 21 - packages/lab-tool/utils/json/pretty-print.scm | 13 - packages/lab-tool/utils/json/serialize.scm | 27 - packages/lab-tool/utils/json/validation.scm | 67 -- packages/lab-tool/utils/logging-new.scm | 42 - packages/lab-tool/utils/logging/core.scm | 38 - packages/lab-tool/utils/logging/format.scm | 42 - packages/lab-tool/utils/logging/level.scm | 30 - packages/lab-tool/utils/logging/output.scm | 23 - packages/lab-tool/utils/logging/spinner.scm | 27 - packages/lab-tool/utils/logging/state.scm | 27 - packages/lab-tool/utils/ssh-new.scm | 27 - .../lab-tool/utils/ssh/connection-test.scm | 41 - packages/lab-tool/utils/ssh/context.scm | 33 - packages/lab-tool/utils/ssh/file-copy.scm | 50 -- .../lab-tool/utils/ssh/remote-command.scm | 58 -- packages/lab-tool/utils/ssh/retry.scm | 45 - packages/lab-tools.nix | 2 +- shell.nix | 12 +- 70 files changed, 195 insertions(+), 5688 deletions(-) delete mode 100644 modules/common/emacs.nix create mode 100644 packages/lab-tool/DEVELOPMENT.md delete mode 100644 packages/lab-tool/PROJECT_STATUS.md delete mode 100644 packages/lab-tool/REFACTORING_SUMMARY.md delete mode 100644 packages/lab-tool/TODO.md delete mode 100644 packages/lab-tool/archive/core/health.scm delete mode 100644 packages/lab-tool/archive/core/logging.scm delete mode 100644 packages/lab-tool/archive/core/ssh.scm delete mode 100644 packages/lab-tool/archive/core/status.scm delete mode 100644 packages/lab-tool/archive/core/utils.scm delete mode 100644 packages/lab-tool/archive/deployment/deployment.scm delete mode 100644 packages/lab-tool/research/core.scm delete mode 100644 packages/lab-tool/research/deployment.scm delete mode 100644 packages/lab-tool/research/guile-mcp-server.scm delete mode 100644 packages/lab-tool/research/guile.md delete mode 100644 packages/lab-tool/research/guile_ecosystem.md delete mode 100644 packages/lab-tool/research/guile_scripting_solution.md delete mode 100755 packages/lab-tool/research/home-lab-tool.scm delete mode 100644 packages/lab-tool/research/machines.scm delete mode 100644 packages/lab-tool/research/monitoring.scm delete mode 100644 packages/lab-tool/testing/README.md delete mode 100644 packages/lab-tool/testing/final-verification.scm delete mode 100644 packages/lab-tool/testing/tdd-summary.scm delete mode 100755 packages/lab-tool/testing/test-deployment.scm delete mode 100755 packages/lab-tool/testing/test-final-validation.scm delete mode 100755 packages/lab-tool/testing/test-functionality.scm delete mode 100644 packages/lab-tool/testing/test-implementation.scm delete mode 100755 packages/lab-tool/testing/test-integration.scm delete mode 100755 packages/lab-tool/testing/test-main.scm delete mode 100755 packages/lab-tool/testing/test-missing-functions.scm delete mode 100755 packages/lab-tool/testing/test-modular.scm delete mode 100755 packages/lab-tool/testing/test-modules-simple.scm delete mode 100644 packages/lab-tool/utils/config-new.scm delete mode 100644 packages/lab-tool/utils/config/accessor.scm delete mode 100644 packages/lab-tool/utils/config/defaults.scm delete mode 100644 packages/lab-tool/utils/config/loader.scm delete mode 100644 packages/lab-tool/utils/config/state.scm delete mode 100644 packages/lab-tool/utils/json-new.scm delete mode 100644 packages/lab-tool/utils/json/file-io.scm delete mode 100644 packages/lab-tool/utils/json/manipulation.scm delete mode 100644 packages/lab-tool/utils/json/parse.scm delete mode 100644 packages/lab-tool/utils/json/pretty-print.scm delete mode 100644 packages/lab-tool/utils/json/serialize.scm delete mode 100644 packages/lab-tool/utils/json/validation.scm delete mode 100644 packages/lab-tool/utils/logging-new.scm delete mode 100644 packages/lab-tool/utils/logging/core.scm delete mode 100644 packages/lab-tool/utils/logging/format.scm delete mode 100644 packages/lab-tool/utils/logging/level.scm delete mode 100644 packages/lab-tool/utils/logging/output.scm delete mode 100644 packages/lab-tool/utils/logging/spinner.scm delete mode 100644 packages/lab-tool/utils/logging/state.scm delete mode 100644 packages/lab-tool/utils/ssh-new.scm delete mode 100644 packages/lab-tool/utils/ssh/connection-test.scm delete mode 100644 packages/lab-tool/utils/ssh/context.scm delete mode 100644 packages/lab-tool/utils/ssh/file-copy.scm delete mode 100644 packages/lab-tool/utils/ssh/remote-command.scm delete mode 100644 packages/lab-tool/utils/ssh/retry.scm diff --git a/dotfiles/geir/emacs-config/init-nix.el b/dotfiles/geir/emacs-config/init-nix.el index e17d5fc..f86374e 100644 --- a/dotfiles/geir/emacs-config/init-nix.el +++ b/dotfiles/geir/emacs-config/init-nix.el @@ -155,7 +155,9 @@ ;; Module loading system ;; Load modules based on availability and profile (defvar my-modules-dir - (expand-file-name "modules/" user-emacs-directory) + (if (getenv "EMACS_PROFILE") + "/etc/emacs/modules/" ; System modules for Nix environment + (expand-file-name "modules/" user-emacs-directory)) ; User modules for non-Nix "Directory containing modular configuration files.") (defun load-module (module-name) @@ -168,19 +170,21 @@ ;; Load modules based on profile (let ((profile (getenv "EMACS_PROFILE"))) (pcase profile - ("server" - ;; Minimal modules for server - (load-module "ui")) + ("nox" + ;; Minimal modules for terminal use + (load-module "completion") + (load-module "navigation") + (load-module "development") + (load-module "elisp-development")) - ((or "laptop" "workstation") - ;; Full module set for development machines + ("gui" + ;; Full module set for GUI development (load-module "ui") (load-module "completion") (load-module "navigation") (load-module "development") (load-module "elisp-development") - (when (string= profile "workstation") - (load-module "claude-code"))) + (load-module "claude-code")) (_ ;; Default module loading (non-Nix environment) diff --git a/dotfiles/geir/emacs-config/modules/claude-code.el b/dotfiles/geir/emacs-config/modules/claude-code.el index 4ac5b44..393f078 100644 --- a/dotfiles/geir/emacs-config/modules/claude-code.el +++ b/dotfiles/geir/emacs-config/modules/claude-code.el @@ -73,7 +73,7 @@ (use-package eat :ensure nil ; Already installed via quelpa :custom - (eat-term-name "xterm-256color") + (eat-term-name "xterm-256color")OB (eat-kill-buffer-on-exit t)) ;; Alternative terminal emulator (if eat fails or user prefers vterm) @@ -123,4 +123,4 @@ (global-set-key (kbd "C-c C-c p") #'claude-code-project-instance) (provide 'claude-code) -;;; claude-code.el ends here \ No newline at end of file +;;; claude-code.el ends here diff --git a/machines/congenital-optimist/configuration.nix b/machines/congenital-optimist/configuration.nix index 8b472e1..709d234 100644 --- a/machines/congenital-optimist/configuration.nix +++ b/machines/congenital-optimist/configuration.nix @@ -68,7 +68,7 @@ # Emacs workstation configuration services.emacs-profiles = { enable = true; - profile = "workstation"; + profile = "gui"; enableDaemon = true; user = "geir"; }; diff --git a/machines/grey-area/configuration.nix b/machines/grey-area/configuration.nix index 17f184e..04daf45 100644 --- a/machines/grey-area/configuration.nix +++ b/machines/grey-area/configuration.nix @@ -49,7 +49,7 @@ # Emacs server configuration (minimal for services host) services.emacs-profiles = { enable = true; - profile = "server"; + profile = "nox"; enableDaemon = false; user = "sma"; }; diff --git a/machines/little-rascal/configuration.nix b/machines/little-rascal/configuration.nix index b92b075..88ee61f 100644 --- a/machines/little-rascal/configuration.nix +++ b/machines/little-rascal/configuration.nix @@ -79,10 +79,10 @@ kernel.sysctl."vm.swappiness" = 180; }; - # Emacs laptop configuration + # Emacs GUI configuration services.emacs-profiles = { enable = true; - profile = "laptop"; + profile = "gui"; enableDaemon = true; user = "geir"; }; diff --git a/machines/reverse-proxy/configuration.nix b/machines/reverse-proxy/configuration.nix index 0119103..0ad7b48 100644 --- a/machines/reverse-proxy/configuration.nix +++ b/machines/reverse-proxy/configuration.nix @@ -49,7 +49,7 @@ # Emacs server configuration (minimal for edge server) services.emacs-profiles = { enable = true; - profile = "server"; + profile = "nox"; enableDaemon = false; user = "sma"; }; diff --git a/machines/sleeper-service/configuration.nix b/machines/sleeper-service/configuration.nix index 1b8bd71..b3dcffc 100644 --- a/machines/sleeper-service/configuration.nix +++ b/machines/sleeper-service/configuration.nix @@ -53,7 +53,7 @@ # Emacs server configuration (minimal) services.emacs-profiles = { enable = true; - profile = "server"; + profile = "nox"; enableDaemon = false; # Don't run daemon on server user = "sma"; }; diff --git a/modules/common/emacs.nix b/modules/common/emacs.nix deleted file mode 100644 index 5b07da6..0000000 --- a/modules/common/emacs.nix +++ /dev/null @@ -1,20 +0,0 @@ -# Common Emacs Configuration -# Shared Emacs setup for all machines -{ - config, - pkgs, - ... -}: { - # System-wide Emacs installation - environment.systemPackages = with pkgs; [ - emacs - # Basic Emacs utilities - emacsPackages.use-package - ]; - - # Set Emacs as default editor - environment.sessionVariables = { - EDITOR = "emacs"; - VISUAL = "emacs"; - }; -} diff --git a/modules/development/emacs.nix b/modules/development/emacs.nix index 9159622..1bc2c29 100644 --- a/modules/development/emacs.nix +++ b/modules/development/emacs.nix @@ -80,7 +80,6 @@ with lib; let org org-roam org-roam-ui - org-agenda # UI enhancements doom-themes @@ -105,7 +104,6 @@ with lib; let workstation = epkgs: with epkgs; [ # All development packages plus extras - claude-code # AI assistance (when available) pdf-tools nov # EPUB reader elfeed # RSS reader @@ -118,76 +116,33 @@ with lib; let }; # Generate Emacs configuration based on profile - # Uses emacs-gtk to track upstream with GTK3 support for desktop profiles + # Uses emacs-pgtk for native Wayland support on desktop profiles # Uses emacs-nox for server profiles (no X11/GUI dependencies) emacsWithProfile = profile: let # Choose Emacs package based on profile emacsPackage = - if profile == "server" - then pkgs.emacs-nox # No GUI for servers - else pkgs.emacs-gtk; # GTK3 for desktops + if profile == "nox" + then pkgs.emacs-nox # Terminal only + else pkgs.emacs-pgtk; # Pure GTK for native Wayland support # Combine package sets based on profile selectedPackages = epkgs: (packageSets.essential epkgs) ++ ( - if profile == "server" + if profile == "nox" then packageSets.minimal epkgs - else if profile == "laptop" - then packageSets.development epkgs - else if profile == "workstation" - then (packageSets.development epkgs) ++ (packageSets.workstation epkgs) - else packageSets.minimal epkgs + else (packageSets.development epkgs) ++ (packageSets.workstation epkgs) ); in - pkgs.emacsWithPackagesFromUsePackage { - config = builtins.readFile ../../dotfiles/geir/emacs-config/init-nix.el; - package = emacsPackage; - extraEmacsPackages = selectedPackages; - - # Provide external tools that Emacs will use - # These will be available via environment variables - override = epkgs: - epkgs - // { - # External tools for Emacs integration - external-tools = - [ - pkgs.ripgrep # for fast searching - pkgs.fd # for file finding - pkgs.sqlite # for org-roam database - pkgs.ag # the silver searcher - pkgs.git # version control - pkgs.direnv # environment management - - # Language servers (when available) - pkgs.nixd # Nix language server - pkgs.nodePackages.bash-language-server - pkgs.nodePackages.yaml-language-server - pkgs.marksman # Markdown language server - - # Formatters - pkgs.alejandra # Nix formatter - pkgs.shellcheck # Shell script analysis - pkgs.shfmt # Shell script formatter - ] - ++ lib.optionals (profile != "server") [ - # Additional tools for development profiles - pkgs.nodejs # for various language servers - pkgs.python3 # for Python development - pkgs.rustup # Rust toolchain - pkgs.go # Go language - ]; - }; - }; + emacsPackage.pkgs.withPackages (epkgs: selectedPackages epkgs); in { options.services.emacs-profiles = { enable = mkEnableOption "Emacs with machine-specific profiles"; profile = mkOption { - type = types.enum ["server" "laptop" "workstation"]; - default = "laptop"; - description = "Emacs profile to use based on machine type"; + type = types.enum ["gui" "nox"]; + default = "gui"; + description = "Emacs profile: gui (with UI) or nox (terminal only)"; }; enableDaemon = mkOption { @@ -207,6 +162,7 @@ in { # Install Emacs with the selected profile environment.systemPackages = [ (emacsWithProfile cfg.profile) + pkgs.silver-searcher ]; # System-wide Emacs daemon (optional) @@ -248,7 +204,7 @@ in { mode = "0644"; }; - "emacs/modules/claude-code.el" = mkIf (cfg.profile == "workstation") { + "emacs/modules/claude-code.el" = mkIf (cfg.profile == "gui") { source = ../../dotfiles/geir/emacs-config/modules/claude-code.el; mode = "0644"; }; @@ -257,12 +213,10 @@ in { # Environment variables for Nix integration environment.variables = { EMACS_PROFILE = cfg.profile; - - # Tool paths for Emacs integration RG_PATH = "${pkgs.ripgrep}/bin/rg"; FD_PATH = "${pkgs.fd}/bin/fd"; SQLITE_PATH = "${pkgs.sqlite}/bin/sqlite3"; - AG_PATH = "${pkgs.ag}/bin/ag"; + AG_PATH = "${pkgs.silver-searcher}/bin/ag"; # Language servers NIL_LSP_PATH = "${pkgs.nixd}/bin/nixd"; diff --git a/modules/users/geir.nix b/modules/users/geir.nix index 0d60457..eb147be 100644 --- a/modules/users/geir.nix +++ b/modules/users/geir.nix @@ -95,8 +95,6 @@ in { celluloid ytmdesktop - # Emacs Integration - emacsPackages.vterm # Gaming steam # Desktop integration (moved from system) diff --git a/packages/lab-tool/DEVELOPMENT.md b/packages/lab-tool/DEVELOPMENT.md new file mode 100644 index 0000000..f16eefd --- /dev/null +++ b/packages/lab-tool/DEVELOPMENT.md @@ -0,0 +1,148 @@ +# Lab Tool Development Guide + +## Build Commands + +### Build the Lab Tool Package +```bash +# Build the lab tool from project root +nix build .#packages.x86_64-linux.lab + +# The binary will be available at ./result/bin/lab +``` + +### Quick Development Build +```bash +# From the lab-tool directory +cd packages/lab-tool +nix build .#lab-tool # if available, otherwise use full path above +``` + +## Testing Commands + +### Test Lab Tool Functionality +```bash +# Test help command +./result/bin/lab help + +# Test machine listing +./result/bin/lab machines + +# Test status check +./result/bin/lab status + +# Test dry-run deployment +./result/bin/lab deploy little-rascal --dry-run + +# Test actual deployment +./result/bin/lab deploy little-rascal +``` + +### Test System Integration +```bash +# Deploy configuration using nixos-rebuild (requires sudo access) +sudo nixos-rebuild switch --flake .#little-rascal --show-trace + +# Or using lab tool (recommended) +lab deploy little-rascal +``` + +## Development Workflow + +### 1. Make Changes +Edit source files in: +- `main.scm` - CLI interface +- `lab/deployment.scm` - Deployment logic +- `lab/machines.scm` - Machine management +- `utils/*.scm` - Utility functions + +### 2. Build and Test +```bash +# Rebuild after changes +nix build .#packages.x86_64-linux.lab + +# Test basic functionality +./result/bin/lab help +./result/bin/lab machines + +# Test deployment (dry-run first) +./result/bin/lab deploy little-rascal --dry-run +``` + +### 3. Debug Issues +```bash +# Enable Guile debugging +export GUILE_AUTO_COMPILE=0 + +# Run with verbose output +./result/bin/lab deploy little-rascal --dry-run + +# Check deploy-rs command directly +deploy --help +``` + +## Common Development Tasks + +### Update Deploy-rs Command Format +Edit `lab/deployment.scm` in the `build-deploy-command` function: +```scheme +;; Example: Add new flags +(when new-option + (set! flags (cons "--new-flag=value" flags))) +``` + +### Add New Machine +Add to the machine list in `lab/machines.scm` or config files. + +### Debug Deployment Issues +1. Check the generated command with dry-run +2. Test deploy-rs directly: `deploy '.#little-rascal' --dry-activate` +3. Check flake structure: `nix flake show` + +### Module Structure +- `main.scm` - Entry point and CLI parsing +- `lab/core.scm` - Core lab functionality +- `lab/deployment.scm` - Deploy-rs integration +- `lab/machines.scm` - Machine management +- `lab/monitoring.scm` - Health checks and monitoring +- `lab/auto-update.scm` - Automatic update system +- `utils/logging.scm` - Logging system with colors +- `utils/config.scm` - Configuration management +- `utils/ssh.scm` - SSH utilities +- `utils/json.scm` - JSON handling + +## Troubleshooting + +### Build Failures +```bash +# Check flake structure +nix flake show + +# Verify Guile syntax +guile --no-auto-compile -c "(load \"main.scm\")" +``` + +### Runtime Errors +```bash +# Check module exports +guile -c "(use-modules (lab deployment)) (display 'loaded)" + +# Test individual functions +guile -c "(use-modules (lab deployment)) (deploy-machine \"little-rascal\" \"default\" '((dry-run . #t)))" +``` + +### Deploy-rs Issues +```bash +# Test deploy-rs directly +deploy '.#little-rascal' --dry-activate + +# Check machine connectivity +ssh sma@little-rascal 'echo "connected"' +``` + +## Best Practices + +1. **Always test with dry-run first** +2. **Use the lab tool instead of direct nixos-rebuild when possible** +3. **Check flake status before deployment** (`nix flake check`) +4. **Keep commits atomic** - one feature/fix per commit +5. **Update this file when adding new commands or workflows** \ No newline at end of file diff --git a/packages/lab-tool/PROJECT_STATUS.md b/packages/lab-tool/PROJECT_STATUS.md deleted file mode 100644 index 16fb3a9..0000000 --- a/packages/lab-tool/PROJECT_STATUS.md +++ /dev/null @@ -1,60 +0,0 @@ -# Lab Tool - Clean Project Structure - -## ๐Ÿ“ Current Structure - -``` -lab-tool/ -โ”œโ”€โ”€ main.scm # Main CLI entry point โœ… WORKING -โ”œโ”€โ”€ lab/ # Core lab modules -โ”‚ โ”œโ”€โ”€ core.scm # Core functionality -โ”‚ โ”œโ”€โ”€ deployment.scm # Deployment operations โœ… FIXED -โ”‚ โ”œโ”€โ”€ machines.scm # Machine management -โ”‚ โ””โ”€โ”€ monitoring.scm # Infrastructure monitoring -โ”œโ”€โ”€ utils/ # Utility modules -โ”‚ โ”œโ”€โ”€ logging.scm # Logging with colors โœ… FIXED -โ”‚ โ”œโ”€โ”€ config.scm # Configuration management -โ”‚ โ”œโ”€โ”€ ssh.scm # SSH utilities -โ”‚ โ””โ”€โ”€ config/ # Modular config system -โ”œโ”€โ”€ mcp/ # MCP server (future enhancement) -โ”œโ”€โ”€ testing/ # All test files โœ… ORGANIZED -โ”œโ”€โ”€ archive/ # Old/backup files -โ”œโ”€โ”€ research/ # Original prototypes -โ””โ”€โ”€ config/ # Runtime configuration -``` - -## โœ… TDD Success Summary - -### Fixed Issues -1. **Syntax errors in deployment.scm** - Missing parentheses and corrupted module definition -2. **Missing exports in utils/logging.scm** - Added `get-color` function to exports -3. **Error handling in main.scm** - Proper exit codes for invalid commands -4. **Module loading** - All modules now load without compilation issues - -### Verified Working Features -- โœ… **CLI Interface**: help, status, machines, deploy, health, test-modules -- โœ… **Real Deployment**: Successfully deploys to actual NixOS machines -- โœ… **Infrastructure Monitoring**: Checks machine status across the lab -- โœ… **Error Handling**: Proper error messages and exit codes -- โœ… **Modular Architecture**: K.I.S.S principles applied throughout - -### Test Organization -- All test files moved to `testing/` directory -- Clear test categories and documentation -- TDD approach validated all functionality - -## ๐Ÿš€ Ready for Production - -The lab tool is now fully functional for core home lab operations: -- Deploy NixOS configurations to any machine -- Monitor infrastructure status -- Manage machine health checks -- Clean, modular codebase following K.I.S.S principles - -## ๐Ÿ“‹ Next Steps - -Priority items from TODO.md: -1. Complete MCP server implementation -2. Enhanced machine discovery -3. Improved health checking - -The core functionality is complete and battle-tested! diff --git a/packages/lab-tool/REFACTORING_SUMMARY.md b/packages/lab-tool/REFACTORING_SUMMARY.md deleted file mode 100644 index e34820f..0000000 --- a/packages/lab-tool/REFACTORING_SUMMARY.md +++ /dev/null @@ -1,119 +0,0 @@ -# K.I.S.S Refactoring Summary - -## Applied Principles - -### 1. Modularization (Keep It Simple, Keep It Small) - -- **Before**: Large monolithic modules (138+ lines) -- **After**: Small focused modules (each under 50 lines) -- **Example**: SSH module split into 5 specialized modules - -### 2. Single Responsibility Principle (UNIX Philosophy: Do One Thing Well) - -- **connection-test.scm**: Only SSH connectivity testing -- **remote-command.scm**: Only remote command execution -- **file-copy.scm**: Only file transfer operations -- **retry.scm**: Only retry logic -- **context.scm**: Only connection context management - -### 3. Functional Programming Patterns - -- **Pure Functions First**: All core logic implemented as pure functions -- **Immutable Data**: Configuration and data structures remain immutable -- **Separation of Concerns**: Pure functions separated from side effects - -### 4. Function-Level Modularity - -Each module exports both: - -- **Pure functions**: For testing, composition, and functional programming -- **Impure wrappers**: For convenience and logging - -## Module Structure - -``` -utils/ -โ”œโ”€โ”€ ssh/ -โ”‚ โ”œโ”€โ”€ connection-test.scm # Pure SSH connectivity testing -โ”‚ โ”œโ”€โ”€ remote-command.scm # Pure command execution logic -โ”‚ โ”œโ”€โ”€ file-copy.scm # Pure file transfer operations -โ”‚ โ”œโ”€โ”€ retry.scm # Pure retry logic with backoff -โ”‚ โ””โ”€โ”€ context.scm # Connection context management -โ”œโ”€โ”€ config/ -โ”‚ โ”œโ”€โ”€ defaults.scm # Pure data: default configuration -โ”‚ โ”œโ”€โ”€ loader.scm # File I/O operations -โ”‚ โ”œโ”€โ”€ accessor.scm # Pure configuration access functions -โ”‚ โ””โ”€โ”€ state.scm # Mutable state management -โ”œโ”€โ”€ logging/ -โ”‚ โ”œโ”€โ”€ format.scm # Pure formatting and color codes -โ”‚ โ”œโ”€โ”€ level.scm # Pure log level management -โ”‚ โ”œโ”€โ”€ state.scm # Mutable log level state -โ”‚ โ”œโ”€โ”€ output.scm # Pure output formatting -โ”‚ โ”œโ”€โ”€ core.scm # Main logging functions -โ”‚ โ””โ”€โ”€ spinner.scm # Progress indication -โ””โ”€โ”€ json/ - โ”œโ”€โ”€ parse.scm # Pure JSON parsing - โ”œโ”€โ”€ serialize.scm # Pure JSON serialization - โ”œโ”€โ”€ file-io.scm # File I/O with pure/impure versions - โ”œโ”€โ”€ validation.scm # Pure schema validation - โ”œโ”€โ”€ manipulation.scm # Pure object manipulation - โ””โ”€โ”€ pretty-print.scm # Output formatting -``` - -## Benefits Achieved - -### 1. Testability - -- Pure functions can be tested in isolation -- No side effects to mock or manage -- Clear input/output contracts - -### 2. Composability - -- Small functions can be easily combined -- Pure functions enable functional composition -- Reusable building blocks - -### 3. Maintainability - -- Single responsibility makes modules easy to understand -- Changes are localized to specific modules -- Clear separation between pure and impure code - -### 4. Code Reuse - -- Pure functions can be reused across different contexts -- Both pure and impure versions available -- Facade modules provide convenient interfaces - -## Usage Examples - -### Pure Function Composition - -```scheme -;; Test connection and get config in one go -(let ((ssh-config (get-ssh-config-pure config "machine-name"))) - (if (test-ssh-connection-pure ssh-config) - (run-remote-command-pure ssh-config "uptime" '()) - #f)) -``` - -### Convenient Impure Wrappers - -```scheme -;; Same operation with logging and error handling -(with-ssh-connection "machine-name" - (lambda () (run-remote-command "machine-name" "uptime"))) -``` - -### Functional Pipeline - -```scheme -;; Pure validation pipeline -(let* ((config (load-config-from-file "config.json")) - (valid? (validate-json-schema config machine-schema)) - (machines (if valid? (get-all-machines-pure config) '()))) - machines) -``` - -This refactoring transforms the codebase from monolithic modules into a collection of small, focused, composable functions that follow functional programming principles while maintaining practical usability. diff --git a/packages/lab-tool/TODO.md b/packages/lab-tool/TODO.md deleted file mode 100644 index ed315ae..0000000 --- a/packages/lab-tool/TODO.md +++ /dev/null @@ -1,35 +0,0 @@ -# Lab Tool Implementation Status - -## โœ… COMPLETED - -- Basic modular utils (logging, config, json, ssh) -- Lab module structure (core, machines, deployment, monitoring) -- MCP server stub -- Module loading tests pass -- **CLI interface working** (status, machines, deploy commands) -- **Infrastructure status checking functional** -- **All module tests passing** -- **TDD FIXES:** Syntax errors, missing exports, error handling -- **DEPLOYMENT WORKING:** Real nixos-rebuild functionality -- **ALL CORE COMMANDS FUNCTIONAL:** help, status, machines, deploy, health, test-modules - -## ๐Ÿ“‹ NEXT TASKS - -### High Priority - -1. ~~**Fix main.scm** - Update to use new lab modules~~ โœ… -2. ~~**Implement core functions** - Add real functionality to lab modules~~ โœ… -3. ~~**Test CLI interface** - Ensure commands work end-to-end~~ โœ… -4. ~~**Fix syntax and module issues** - TDD approach~~ โœ… - -### Medium Priority - -1. **Complete MCP server** - JSON-RPC protocol implementation -2. ~~**Add deployment logic** - Move from research/ to lab/deployment~~ โœ… -3. **Machine management** - Add discovery and health checks - -### Config Enhancement Notes - -- Machine folder creation with hardware config -- Git integration for new machines -- Seamless machine import workflow diff --git a/packages/lab-tool/archive/core/health.scm b/packages/lab-tool/archive/core/health.scm deleted file mode 100644 index d2f37b0..0000000 --- a/packages/lab-tool/archive/core/health.scm +++ /dev/null @@ -1,75 +0,0 @@ -;; lab/core/health.scm - Health check functionality - -(define-module (lab core health) - #:use-module (ice-9 rdelim) - #:use-module (srfi srfi-1) - #:use-module (lab core logging) - #:use-module (lab core ssh) - #:export (check-system-health - check-disk-space - check-system-load - check-critical-services - check-network-connectivity)) - -(define (check-system-health machine-name) - "Perform comprehensive health check on a machine" - (log-info "Performing health check on ~a..." machine-name) - - (let ((health-checks - '(("connectivity" . test-ssh-connection) - ("disk-space" . check-disk-space) - ("system-load" . check-system-load) - ("critical-services" . check-critical-services) - ("network" . check-network-connectivity)))) - - (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (log-debug "Running ~a check..." check-name) - (catch #t - (lambda () - (let ((result (check-proc machine-name))) - `(,check-name . ((status . ,(if result 'pass 'fail)) - (result . ,result))))) - (lambda (key . args) - (log-warn "Health check ~a failed: ~a" check-name key) - `(,check-name . ((status . error) - (error . ,key))))))) - health-checks))) - -(define (check-disk-space machine-name) - "Check if disk space is below critical threshold" - (call-with-values - (lambda () (run-remote-command machine-name "df / | tail -1 | awk '{print $5}' | sed 's/%//'")) - (lambda (success output) - (if success - (let ((usage (string->number (string-trim-right output)))) - (< usage 90)) ; Pass if usage < 90% - #f)))) - -(define (check-system-load machine-name) - "Check if system load is reasonable" - (call-with-values - (lambda () (run-remote-command machine-name "cat /proc/loadavg | cut -d' ' -f1")) - (lambda (success output) - (if success - (let ((load (string->number (string-trim-right output)))) - (< load 5.0)) ; Pass if load < 5.0 - #f)))) - -(define (check-critical-services machine-name) - "Check that critical services are running" - (let ((critical-services '("sshd"))) - (every (lambda (service) - (call-with-values - (lambda () (run-remote-command machine-name "systemctl is-active" service)) - (lambda (success output) - (and success (string=? (string-trim-right output) "active"))))) - critical-services))) - -(define (check-network-connectivity machine-name) - "Check basic network connectivity" - (call-with-values - (lambda () (run-remote-command machine-name "ping -c 1 -W 5 8.8.8.8 > /dev/null 2>&1; echo $?")) - (lambda (success output) - (and success (string=? (string-trim-right output) "0"))))) diff --git a/packages/lab-tool/archive/core/logging.scm b/packages/lab-tool/archive/core/logging.scm deleted file mode 100644 index de03c49..0000000 --- a/packages/lab-tool/archive/core/logging.scm +++ /dev/null @@ -1,29 +0,0 @@ -;; lab/core/logging.scm - Logging functionality - -(define-module (lab core logging) - #:use-module (ice-9 format) - #:export (log-info - log-debug - log-success - log-error - log-warn)) - -(define (log-info format-str . args) - "Log info message" - (apply format #t (string-append "[INFO] " format-str "~%") args)) - -(define (log-debug format-str . args) - "Log debug message" - (apply format #t (string-append "[DEBUG] " format-str "~%") args)) - -(define (log-success format-str . args) - "Log success message" - (apply format #t (string-append "[SUCCESS] " format-str "~%") args)) - -(define (log-error format-str . args) - "Log error message" - (apply format #t (string-append "[ERROR] " format-str "~%") args)) - -(define (log-warn format-str . args) - "Log warning message" - (apply format #t (string-append "[WARN] " format-str "~%") args)) diff --git a/packages/lab-tool/archive/core/ssh.scm b/packages/lab-tool/archive/core/ssh.scm deleted file mode 100644 index d35edd7..0000000 --- a/packages/lab-tool/archive/core/ssh.scm +++ /dev/null @@ -1,24 +0,0 @@ -;; lab/core/ssh.scm - SSH operations - -(define-module (lab core ssh) - #:use-module (ice-9 format) - #:use-module (ice-9 popen) - #:use-module (ice-9 rdelim) - #:use-module (ice-9 textual-ports) - #:export (test-ssh-connection - run-remote-command)) - -(define (test-ssh-connection machine-name) - "Test SSH connection to machine" - (zero? (system (format #f "ssh -o ConnectTimeout=5 -o BatchMode=yes ~a exit 2>/dev/null" machine-name)))) - -(define (run-remote-command machine-name command . args) - "Run command on remote machine via SSH" - (let* ((full-command (if (null? args) - command - (string-join (cons command args) " "))) - (ssh-command (format #f "ssh ~a '~a'" machine-name full-command)) - (port (open-input-pipe ssh-command)) - (output (read-string port)) - (status (close-pipe port))) - (values (zero? status) output))) diff --git a/packages/lab-tool/archive/core/status.scm b/packages/lab-tool/archive/core/status.scm deleted file mode 100644 index f131387..0000000 --- a/packages/lab-tool/archive/core/status.scm +++ /dev/null @@ -1,84 +0,0 @@ -;; lab/core/status.scm - Infrastructure status functionality - -(define-module (lab core status) - #:use-module (ice-9 rdelim) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:use-module (lab core logging) - #:use-module (lab core config) - #:use-module (lab core ssh) - #:export (get-infrastructure-status - get-machine-services-status - get-machine-system-info)) - -(define (get-infrastructure-status . args) - "Get status of all machines or specific machine if provided" - (let ((target-machine (if (null? args) #f (car args))) - (machines (if (null? args) - (get-all-machines) - (list (car args))))) - - (log-info "Checking infrastructure status...") - - (map (lambda (machine-name) - (let ((start-time (current-time))) - (log-debug "Checking ~a..." machine-name) - - (let* ((ssh-config (get-ssh-config machine-name)) - (is-local (and ssh-config (assoc-ref ssh-config 'is-local))) - (connection-status (test-ssh-connection machine-name)) - (services-status (if connection-status - (get-machine-services-status machine-name) - '())) - (system-info (if connection-status - (get-machine-system-info machine-name) - #f)) - (elapsed (- (current-time) start-time))) - - `((machine . ,machine-name) - (type . ,(if is-local 'local 'remote)) - (connection . ,(if connection-status 'online 'offline)) - (services . ,services-status) - (system . ,system-info) - (check-time . ,elapsed))))) - machines))) - -(define (get-machine-services-status machine-name) - "Check status of services on a machine" - (let ((machine-config (get-machine-config machine-name))) - (if machine-config - (let ((services (assoc-ref machine-config 'services))) - (if services - (map (lambda (service) - (call-with-values - (lambda () (run-remote-command machine-name - "systemctl is-active" - (symbol->string service))) - (lambda (success output) - `(,service . ,(if success - (string-trim-right output) - "unknown"))))) - services) - '())) - '()))) - -(define (get-machine-system-info machine-name) - "Get basic system information from a machine" - (let ((info-commands - '(("uptime" "uptime -p") - ("load" "cat /proc/loadavg | cut -d' ' -f1-3") - ("memory" "free -h | grep Mem | awk '{print $3\"/\"$2}'") - ("disk" "df -h / | tail -1 | awk '{print $5}'") - ("kernel" "uname -r")))) - - (fold (lambda (cmd-pair acc) - (let ((key (car cmd-pair)) - (command (cadr cmd-pair))) - (call-with-values - (lambda () (run-remote-command machine-name command)) - (lambda (success output) - (if success - (assoc-set! acc (string->symbol key) (string-trim-right output)) - acc))))) - '() - info-commands))) diff --git a/packages/lab-tool/archive/core/utils.scm b/packages/lab-tool/archive/core/utils.scm deleted file mode 100644 index 9f30c78..0000000 --- a/packages/lab-tool/archive/core/utils.scm +++ /dev/null @@ -1,12 +0,0 @@ -;; lab/core/utils.scm - Utility functions - -(define-module (lab core utils) - #:use-module (ice-9 format) - #:export (with-spinner)) - -(define (with-spinner message proc) - "Execute procedure with spinner (stub implementation)" - (display (format #f "~a..." message)) - (let ((result (proc))) - (display " done.\n") - result)) diff --git a/packages/lab-tool/archive/deployment/deployment.scm b/packages/lab-tool/archive/deployment/deployment.scm deleted file mode 100644 index 0bc5dfd..0000000 --- a/packages/lab-tool/archive/deployment/deployment.scm +++ /dev/null @@ -1,109 +0,0 @@ -;; lab/core/deployment.scm - Deployment functionality - -(define-module (lab core deployment) - #:use-module (ice-9 format) - #:use-module (ice-9 popen) - #:use-module (ice-9 textual-ports) - #:use-module (lab core logging) - #:use-module (lab core config) - #:use-module (lab core utils) - #:export (update-flake - validate-environment - execute-nixos-rebuild)) - -(define (update-flake options) - "Update flake inputs in the home lab repository" - (let ((homelab-root (get-homelab-root)) - (dry-run (option-ref options 'dry-run #f))) - - (log-info "Updating flake inputs...") - - (if dry-run - (begin - (log-info "DRY RUN: Would execute: nix flake update") - #t) - (let* ((update-cmd (format #f "cd ~a && nix flake update" homelab-root)) - (port (open-pipe* OPEN_READ "/bin/sh" "-c" update-cmd)) - (output (get-string-all port)) - (status (close-pipe port))) - - (if (zero? status) - (begin - (log-success "Flake inputs updated successfully") - (log-debug "Update output: ~a" output) - #t) - (begin - (log-error "Flake update failed (exit: ~a)" status) - (log-error "Error output: ~a" output) - #f)))))) - -(define (validate-environment) - "Validate that the home lab environment is properly configured" - (log-info "Validating home lab environment...") - - (let ((checks - `(("homelab-root" . ,(lambda () (file-exists? (get-homelab-root)))) - ("flake-file" . ,(lambda () (file-exists? (string-append (get-homelab-root) "/flake.nix")))) - ("ssh-config" . ,(lambda () (file-exists? (string-append (getenv "HOME") "/.ssh/config")))) - ("nix-command" . ,(lambda () (zero? (system "which nix > /dev/null 2>&1")))) - ("machines-config" . ,(lambda () (not (null? (get-all-machines)))))))) - - (let ((results (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (let ((result (check-proc))) - (if result - (log-success "โœ“ ~a" check-name) - (log-error "โœ— ~a" check-name)) - `(,check-name . ,result)))) - checks))) - - (let ((failures (filter (lambda (result) (not (cdr result))) results))) - (if (null? failures) - (begin - (log-success "Environment validation passed") - #t) - (begin - (log-error "Environment validation failed: ~a" (map car failures)) - #f)))))) - -(define (execute-nixos-rebuild machine-name mode options) - "Execute nixos-rebuild on a machine with comprehensive error handling" - (let ((homelab-root (get-homelab-root)) - (dry-run (option-ref options 'dry-run #f)) - (ssh-config (get-ssh-config machine-name))) - - (if (not ssh-config) - (begin - (log-error "No SSH configuration for machine: ~a" machine-name) - #f) - (let* ((is-local (assoc-ref ssh-config 'is-local)) - (flake-ref (format #f "~a#~a" homelab-root machine-name)) - (rebuild-cmd (if is-local - (format #f "sudo nixos-rebuild ~a --flake ~a" mode flake-ref) - (format #f "nixos-rebuild ~a --flake ~a --target-host ~a --use-remote-sudo" - mode flake-ref (assoc-ref ssh-config 'hostname))))) - - (log-info "Executing nixos-rebuild for ~a (mode: ~a)" machine-name mode) - (log-debug "Command: ~a" rebuild-cmd) - - (if dry-run - (begin - (log-info "DRY RUN: Would execute: ~a" rebuild-cmd) - #t) - (with-spinner - (format #f "Rebuilding ~a" machine-name) - (lambda () - (let* ((port (open-pipe* OPEN_READ "/bin/sh" "-c" rebuild-cmd)) - (output (get-string-all port)) - (status (close-pipe port))) - - (if (zero? status) - (begin - (log-success "nixos-rebuild completed successfully for ~a" machine-name) - (log-debug "Build output: ~a" output) - #t) - (begin - (log-error "nixos-rebuild failed for ~a (exit: ~a)" machine-name status) - (log-error "Error output: ~a" output) - #f)))))))) diff --git a/packages/lab-tool/default.nix b/packages/lab-tool/default.nix index fadee46..39be92d 100644 --- a/packages/lab-tool/default.nix +++ b/packages/lab-tool/default.nix @@ -13,7 +13,7 @@ }: stdenv.mkDerivation { pname = "lab-tool"; - version = "0.1.0"; + version = "0.2.0"; src = ./.; diff --git a/packages/lab-tool/lab/deployment.scm b/packages/lab-tool/lab/deployment.scm index db2b08d..fedb748 100644 --- a/packages/lab-tool/lab/deployment.scm +++ b/packages/lab-tool/lab/deployment.scm @@ -52,7 +52,7 @@ (let* ((deploy-cmd (build-deploy-command machine-name skip-checks auto-rollback magic-rollback)) (start-time (current-time))) - (log-debug "Deploy command: ~a" deploy-cmd) + (log-info "Deploy command: ~a" deploy-cmd) (log-info "Executing deployment with automatic rollback protection...") (let* ((port (open-pipe* OPEN_READ "/bin/sh" "-c" deploy-cmd)) @@ -84,15 +84,15 @@ (set! flags (cons "--skip-checks" flags))) (when auto-rollback - (set! flags (cons "--auto-rollback" flags))) + (set! flags (cons "--auto-rollback=true" flags))) (when magic-rollback - (set! flags (cons "--magic-rollback" flags))) + (set! flags (cons "--magic-rollback=true" flags))) ;; Combine command with flags (if (null? flags) base-cmd - (format #f "~a ~a" base-cmd (string-join flags " "))))) + (format #f "~a ~a" base-cmd (string-join (reverse flags) " "))))) ;; Deploy to all machines (define (deploy-all-machines . args) diff --git a/packages/lab-tool/research/core.scm b/packages/lab-tool/research/core.scm deleted file mode 100644 index f375788..0000000 --- a/packages/lab-tool/research/core.scm +++ /dev/null @@ -1,326 +0,0 @@ -;; lab/core.scm - Core home lab operations - -(define-module (lab core) - #:use-module (ice-9 format) - #:use-module (ice-9 popen) - #:use-module (ice-9 rdelim) - #:use-module (ice-9 textual-ports) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:export (get-infrastructure-status - check-system-health - update-flake - validate-environment - execute-nixos-rebuild - check-network-connectivity - option-ref)) - -;; Simple option reference function -(define (option-ref options key default) - "Get option value from options alist with default" - (let ((value (assoc-ref options key))) - (if value value default))) - -;; Stub logging functions (to be replaced with proper logging module) -(define (log-info format-str . args) - "Log info message" - (apply format #t (string-append "[INFO] " format-str "~%") args)) - -(define (log-debug format-str . args) - "Log debug message" - (apply format #t (string-append "[DEBUG] " format-str "~%") args)) - -(define (log-success format-str . args) - "Log success message" - (apply format #t (string-append "[SUCCESS] " format-str "~%") args)) - -(define (log-error format-str . args) - "Log error message" - (apply format #t (string-append "[ERROR] " format-str "~%") args)) - -(define (log-warn format-str . args) - "Log warning message" - (apply format #t (string-append "[WARN] " format-str "~%") args)) - -;; Stub configuration functions -(define (get-all-machines) - "Get list of all machines" - '(grey-area sleeper-service congenital-optimist reverse-proxy)) - -(define (get-machine-config machine-name) - "Get configuration for a machine" - `((services . (systemd ssh)) - (type . server))) - -(define (get-ssh-config machine-name) - "Get SSH configuration for a machine" - `((hostname . ,(symbol->string machine-name)) - (is-local . #f))) - -(define (get-homelab-root) - "Get home lab root directory" - "/home/geir/Home-lab") - -;; Stub SSH functions -(define (test-ssh-connection machine-name) - "Test SSH connection to machine" - (zero? (system (format #f "ssh -o ConnectTimeout=5 -o BatchMode=yes ~a exit 2>/dev/null" machine-name)))) - -(define (run-remote-command machine-name command . args) - "Run command on remote machine via SSH" - (let* ((full-command (if (null? args) - command - (string-join (cons command args) " "))) - (ssh-command (format #f "ssh ~a '~a'" machine-name full-command)) - (port (open-input-pipe ssh-command)) - (output (read-string port)) - (status (close-pipe port))) - (values (zero? status) output))) - -;; Utility function for spinner (stub) -(define (with-spinner message proc) - "Execute procedure with spinner (stub implementation)" - (display (format #f "~a..." message)) - (let ((result (proc))) - (display " done.\n") - result)) - -;; Get comprehensive infrastructure status -(define (get-infrastructure-status . args) - "Get status of all machines or specific machine if provided" - (let ((target-machine (if (null? args) #f (car args))) - (machines (if (null? args) - (get-all-machines) - (list (car args))))) - - (log-info "Checking infrastructure status...") - - (map (lambda (machine-name) - (let ((start-time (current-time))) - (log-debug "Checking ~a..." machine-name) - - (let* ((ssh-config (get-ssh-config machine-name)) - (is-local (and ssh-config (assoc-ref ssh-config 'is-local))) - (connection-status (test-ssh-connection machine-name)) - (services-status (if connection-status - (get-machine-services-status machine-name) - '())) - (system-info (if connection-status - (get-machine-system-info machine-name) - #f)) - (elapsed (- (current-time) start-time))) - - `((machine . ,machine-name) - (type . ,(if is-local 'local 'remote)) - (connection . ,(if connection-status 'online 'offline)) - (services . ,services-status) - (system . ,system-info) - (check-time . ,elapsed))))) - machines))) - -;; Get services status for a machine -(define (get-machine-services-status machine-name) - "Check status of services on a machine" - (let ((machine-config (get-machine-config machine-name))) - (if machine-config - (let ((services (assoc-ref machine-config 'services))) - (if services - (map (lambda (service) - (call-with-values - (lambda () (run-remote-command machine-name - "systemctl is-active" - (symbol->string service))) - (lambda (success output) - `(,service . ,(if success - (string-trim-right output) - "unknown"))))) - services) - '())) - '()))) - -;; Get basic system information from a machine -(define (get-machine-system-info machine-name) - "Get basic system information from a machine" - (let ((info-commands - '(("uptime" "uptime -p") - ("load" "cat /proc/loadavg | cut -d' ' -f1-3") - ("memory" "free -h | grep Mem | awk '{print $3\"/\"$2}'") - ("disk" "df -h / | tail -1 | awk '{print $5}'") - ("kernel" "uname -r")))) - - (fold (lambda (cmd-pair acc) - (let ((key (car cmd-pair)) - (command (cadr cmd-pair))) - (call-with-values - (lambda () (run-remote-command machine-name command)) - (lambda (success output) - (if success - (assoc-set! acc (string->symbol key) (string-trim-right output)) - acc))))) - '() - info-commands))) - -;; Check system health with comprehensive tests -(define (check-system-health machine-name) - "Perform comprehensive health check on a machine" - (log-info "Performing health check on ~a..." machine-name) - - (let ((health-checks - '(("connectivity" . test-ssh-connection) - ("disk-space" . check-disk-space) - ("system-load" . check-system-load) - ("critical-services" . check-critical-services) - ("network" . check-network-connectivity)))) - - (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (log-debug "Running ~a check..." check-name) - (catch #t - (lambda () - (let ((result (check-proc machine-name))) - `(,check-name . ((status . ,(if result 'pass 'fail)) - (result . ,result)))) - (lambda (key . args) - (log-warn "Health check ~a failed: ~a" check-name key) - `(,check-name . ((status . error) - (error . ,key))))))) - health-checks))) - -;; Individual health check functions -(define (check-disk-space machine-name) - "Check if disk space is below critical threshold" - (call-with-values - (lambda () (run-remote-command machine-name "df / | tail -1 | awk '{print $5}' | sed 's/%//'")) - (lambda (success output) - (if success - (let ((usage (string->number (string-trim-right output)))) - (< usage 90)) ; Pass if usage < 90% - #f)))) - -(define (check-system-load machine-name) - "Check if system load is reasonable" - (call-with-values - (lambda () (run-remote-command machine-name "cat /proc/loadavg | cut -d' ' -f1")) - (lambda (success output) - (if success - (let ((load (string->number (string-trim-right output)))) - (< load 5.0)) ; Pass if load < 5.0 - #f)))) - -(define (check-critical-services machine-name) - "Check that critical services are running" - (let ((critical-services '("sshd"))) - (every (lambda (service) - (call-with-values - (lambda () (run-remote-command machine-name "systemctl is-active" service)) - (lambda (success output) - (and success (string=? (string-trim-right output) "active"))))) - critical-services))) - -(define (check-network-connectivity machine-name) - "Check basic network connectivity" - (call-with-values - (lambda () (run-remote-command machine-name "ping -c 1 -W 5 8.8.8.8 > /dev/null 2>&1; echo $?")) - (lambda (success output) - (and success (string=? (string-trim-right output) "0"))))) - -;; Update flake inputs -(define (update-flake options) - "Update flake inputs in the home lab repository" - (let ((homelab-root (get-homelab-root)) - (dry-run (option-ref options 'dry-run #f))) - - (log-info "Updating flake inputs...") - - (if dry-run - (begin - (log-info "DRY RUN: Would execute: nix flake update") - #t) - (let* ((update-cmd (format #f "cd ~a && nix flake update" homelab-root)) - (port (open-pipe* OPEN_READ "/bin/sh" "-c" update-cmd)) - (output (get-string-all port)) - (status (close-pipe port))) - - (if (zero? status) - (begin - (log-success "Flake inputs updated successfully") - (log-debug "Update output: ~a" output) - #t) - (begin - (log-error "Flake update failed (exit: ~a)" status) - (log-error "Error output: ~a" output) - #f)))))) - -;; Validate home lab environment -(define (validate-environment) - "Validate that the home lab environment is properly configured" - (log-info "Validating home lab environment...") - - (let ((checks - `(("homelab-root" . ,(lambda () (file-exists? (get-homelab-root)))) - ("flake-file" . ,(lambda () (file-exists? (string-append (get-homelab-root) "/flake.nix")))) - ("ssh-config" . ,(lambda () (file-exists? (string-append (getenv "HOME") "/.ssh/config")))) - ("nix-command" . ,(lambda () (zero? (system "which nix > /dev/null 2>&1")))) - ("machines-config" . ,(lambda () (not (null? (get-all-machines)))))))) - - (let ((results (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (let ((result (check-proc))) - (if result - (log-success "โœ“ ~a" check-name) - (log-error "โœ— ~a" check-name)) - `(,check-name . ,result)))) - checks))) - - (let ((failures (filter (lambda (result) (not (cdr result))) results))) - (if (null? failures) - (begin - (log-success "Environment validation passed") - #t) - (begin - (log-error "Environment validation failed: ~a" (map car failures)) - #f)))))) - -;; Execute nixos-rebuild with proper error handling -(define (execute-nixos-rebuild machine-name mode options) - "Execute nixos-rebuild on a machine with comprehensive error handling" - (let ((homelab-root (get-homelab-root)) - (dry-run (option-ref options 'dry-run #f)) - (ssh-config (get-ssh-config machine-name))) - - (if (not ssh-config) - (begin - (log-error "No SSH configuration for machine: ~a" machine-name) - #f) - (let* ((is-local (assoc-ref ssh-config 'is-local)) - (flake-ref (format #f "~a#~a" homelab-root machine-name)) - (rebuild-cmd (if is-local - (format #f "sudo nixos-rebuild ~a --flake ~a" mode flake-ref) - (format #f "nixos-rebuild ~a --flake ~a --target-host ~a --use-remote-sudo" - mode flake-ref (assoc-ref ssh-config 'hostname))))) - - (log-info "Executing nixos-rebuild for ~a (mode: ~a)" machine-name mode) - (log-debug "Command: ~a" rebuild-cmd) - - (if dry-run - (begin - (log-info "DRY RUN: Would execute: ~a" rebuild-cmd) - #t) - (with-spinner - (format #f "Rebuilding ~a" machine-name) - (lambda () - (let* ((port (open-pipe* OPEN_READ "/bin/sh" "-c" rebuild-cmd)) - (output (get-string-all port)) - (status (close-pipe port))) - - (if (zero? status) - (begin - (log-success "nixos-rebuild completed successfully for ~a" machine-name) - (log-debug "Build output: ~a" output) - #t) - (begin - (log-error "nixos-rebuild failed for ~a (exit: ~a)" machine-name status) - (log-error "Error output: ~a" output) - #f)))))))))) diff --git a/packages/lab-tool/research/deployment.scm b/packages/lab-tool/research/deployment.scm deleted file mode 100644 index d462452..0000000 --- a/packages/lab-tool/research/deployment.scm +++ /dev/null @@ -1,329 +0,0 @@ -;; lab/deployment.scm - Deployment operations for Home Lab Tool - -(define-module (lab deployment) - #:use-module (ice-9 format) - #:use-module (ice-9 match) - #:use-module (ice-9 popen) - #:use-module (ice-9 textual-ports) - - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:use-module (utils logging) - #:use-module (utils config) - #:use-module (utils ssh) - #:use-module (lab core) - #:export (deploy-machine - update-all-machines - hybrid-update - validate-deployment - rollback-deployment - deployment-status - option-ref)) - -;; Helper function for option handling -(define (option-ref options key default) - "Get option value with default fallback" - (let ((value (assoc-ref options key))) - (if value value default))) - -;; Deploy configuration to a specific machine -(define (deploy-machine machine-name mode options) - "Deploy NixOS configuration to a specific machine" - (let ((valid-modes '("boot" "test" "switch")) - (dry-run (option-ref options 'dry-run #f))) - - ;; Validate inputs - (if (not (validate-machine-name machine-name)) - #f - (if (not (member mode valid-modes)) - (begin - (log-error "Invalid deployment mode: ~a" mode) - (log-error "Valid modes: ~a" (string-join valid-modes ", ")) - #f) - - ;; Proceed with deployment - (begin - (log-info "Starting deployment: ~a -> ~a (mode: ~a)" - machine-name machine-name mode) - - ;; Pre-deployment validation - (if (not (validate-deployment-prerequisites machine-name)) - (begin - (log-error "Pre-deployment validation failed") - #f) - - ;; Execute deployment - (let ((deployment-result - (execute-deployment machine-name mode options))) - - ;; Post-deployment validation - (if deployment-result - (begin - (log-info "Deployment completed, validating...") - (if (validate-post-deployment machine-name mode) - (begin - (log-success "Deployment successful and validated โœ“") - #t) - (begin - (log-warn "Deployment completed but validation failed") - ;; Don't fail completely - deployment might still be functional - #t))) - (begin - (log-error "Deployment failed") - #f))))))))) - -;; Execute the actual deployment -(define (execute-deployment machine-name mode options) - "Execute the deployment based on machine type and configuration" - (let ((ssh-config (get-ssh-config machine-name)) - (machine-config (get-machine-config machine-name))) - - (if (not ssh-config) - (begin - (log-error "No SSH configuration found for ~a" machine-name) - #f) - - (let ((deployment-method (assoc-ref machine-config 'deployment-method)) - (is-local (assoc-ref ssh-config 'is-local))) - - (log-debug "Using deployment method: ~a" (or deployment-method 'nixos-rebuild)) - - (match (or deployment-method 'nixos-rebuild) - ('nixos-rebuild - (execute-nixos-rebuild machine-name mode options)) - - ('deploy-rs - (execute-deploy-rs machine-name mode options)) - - ('hybrid - (execute-hybrid-deployment machine-name mode options)) - - (method - (log-error "Unknown deployment method: ~a" method) - #f)))))) - -;; Execute deploy-rs deployment -(define (execute-deploy-rs machine-name mode options) - "Deploy using deploy-rs for atomic deployments" - (let ((homelab-root (get-homelab-root)) - (dry-run (option-ref options 'dry-run #f))) - - (log-info "Deploying ~a using deploy-rs..." machine-name) - - (if dry-run - (begin - (log-info "DRY RUN: Would execute deploy-rs for ~a" machine-name) - #t) - - (let* ((deploy-cmd (format #f "cd ~a && deploy '.#~a' --magic-rollback --auto-rollback" - homelab-root machine-name)) - (start-time (current-time))) - - (log-debug "Deploy command: ~a" deploy-cmd) - - (with-spinner - (format #f "Deploying ~a with deploy-rs" machine-name) - (lambda () - (let* ((port (open-pipe* OPEN_READ "/bin/sh" "-c" deploy-cmd)) - (output (get-string-all port)) - (status (close-pipe port)) - (elapsed (- (current-time) start-time))) - - (if (zero? status) - (begin - (log-success "deploy-rs completed in ~as" elapsed) - (log-debug "Deploy output: ~a" output) - #t) - (begin - (log-error "deploy-rs failed (exit: ~a)" status) - (log-error "Error output: ~a" output) - #f))))))))) - -;; Execute hybrid deployment (flake update + deploy) -(define (execute-hybrid-deployment machine-name mode options) - "Execute hybrid deployment: update flake then deploy" - (log-info "Starting hybrid deployment for ~a..." machine-name) - - ;; First update flake - (if (update-flake options) - ;; Then deploy - (execute-nixos-rebuild machine-name mode options) - (begin - (log-error "Flake update failed, aborting deployment") - #f))) - -;; Validate deployment prerequisites -(define (validate-deployment-prerequisites machine-name) - "Validate that deployment prerequisites are met" - (log-debug "Validating deployment prerequisites for ~a..." machine-name) - - (let ((checks - `(("machine-config" . ,(lambda () (get-machine-config machine-name))) - ("ssh-connectivity" . ,(lambda () (test-ssh-connection machine-name))) - ("flake-exists" . ,(lambda () (file-exists? (string-append (get-homelab-root) "/flake.nix")))) - ("machine-flake-config" . ,(lambda () (validate-machine-flake-config machine-name)))))) - - (let ((results (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (let ((result (check-proc))) - (if result - (log-debug "โœ“ Prerequisite: ~a" check-name) - (log-error "โœ— Prerequisite failed: ~a" check-name)) - result))) - checks))) - - (every identity results)))) - -;; Validate machine has flake configuration -(define (validate-machine-flake-config machine-name) - "Check that machine has a configuration in the flake" - (let ((machine-dir (string-append (get-homelab-root) "/machines/" machine-name))) - (and (file-exists? machine-dir) - (file-exists? (string-append machine-dir "/configuration.nix"))))) - -;; Validate post-deployment state -(define (validate-post-deployment machine-name mode) - "Validate system state after deployment" - (log-debug "Validating post-deployment state for ~a..." machine-name) - - ;; Wait a moment for services to stabilize - (sleep 3) - - (let ((checks - `(("ssh-connectivity" . ,(lambda () (test-ssh-connection machine-name))) - ("system-responsive" . ,(lambda () (check-system-responsive machine-name))) - ("critical-services" . ,(lambda () (check-critical-services machine-name)))))) - - (let ((results (map (lambda (check-pair) - (let ((check-name (car check-pair)) - (check-proc (cdr check-pair))) - (catch #t - (lambda () - (let ((result (check-proc))) - (if result - (log-debug "โœ“ Post-deployment: ~a" check-name) - (log-warn "โœ— Post-deployment: ~a" check-name)) - result)) - (lambda (key . args) - (log-warn "Post-deployment check ~a failed: ~a" check-name key) - #f)))) - checks))) - - (every identity results)))) - -;; Check if system is responsive after deployment -(define (check-system-responsive machine-name) - "Check if system is responsive after deployment" - (call-with-values (((success output) - (run-remote-command machine-name "echo 'system-check' && uptime"))) - (and success (string-contains output "system-check")))) - -;; Update all machines -(define (update-all-machines mode options) - "Update all configured machines" - (let ((machines (get-all-machines)) - (dry-run (option-ref options 'dry-run #f))) - - (log-info "Starting update of all machines (mode: ~a)..." mode) - - (if dry-run - (begin - (log-info "DRY RUN: Would update machines: ~a" (string-join machines ", ")) - #t) - - (let ((results - (map (lambda (machine-name) - (log-info "Updating ~a..." machine-name) - (let ((result (deploy-machine machine-name mode options))) - (if result - (log-success "โœ“ ~a updated successfully" machine-name) - (log-error "โœ— ~a update failed" machine-name)) - `(,machine-name . ,result))) - machines))) - - (let ((successful (filter cdr results)) - (failed (filter (lambda (r) (not (cdr r))) results))) - - (log-info "Update summary:") - (log-info " Successful: ~a/~a" (length successful) (length results)) - - (when (not (null? failed)) - (log-warn " Failed: ~a" (map car failed))) - - ;; Return success if all succeeded - (= (length successful) (length results))))))) - -;; Hybrid update: flake update + selective deployment -(define (hybrid-update target options) - "Perform hybrid update: flake update followed by deployment" - (log-info "Starting hybrid update for target: ~a" target) - - ;; First update flake - (if (update-flake options) - - ;; Then deploy based on target - (match target - ("all" - (update-all-machines "boot" options)) - - (machine-name - (if (validate-machine-name machine-name) - (deploy-machine machine-name "boot" options) - #f))) - - (begin - (log-error "Flake update failed, aborting hybrid update") - #f))) - -;; Get deployment status -(define (deployment-status . machine-name) - "Get current deployment status for machines" - (let ((machines (if (null? machine-name) - (get-all-machines) - machine-name))) - - (map (lambda (machine) - (let ((last-deployment (get-last-deployment-info machine)) - (current-generation (get-current-generation machine))) - `((machine . ,machine) - (last-deployment . ,last-deployment) - (current-generation . ,current-generation) - (status . ,(get-deployment-health machine))))) - machines))) - -;; Get last deployment information -(define (get-last-deployment-info machine-name) - "Get information about the last deployment" - (call-with-values (((success output) - (run-remote-command machine-name - "ls -la /nix/var/nix/profiles/system* | tail -1"))) - (if success - (string-trim-right output) - "unknown"))) - -;; Get current system generation -(define (get-current-generation machine-name) - "Get current NixOS generation information" - (call-with-values (((success output) - (run-remote-command machine-name - "nixos-version"))) - (if success - (string-trim-right output) - "unknown"))) - -;; Get deployment health status -(define (get-deployment-health machine-name) - "Check if deployment is healthy" - (if (test-ssh-connection machine-name) - 'healthy - 'unhealthy)) - -;; Rollback deployment (placeholder for future implementation) -(define (rollback-deployment machine-name . generation) - "Rollback to previous generation (deploy-rs feature)" - (log-warn "Rollback functionality not yet implemented") - (log-info "For manual rollback on ~a:" machine-name) - (log-info " 1. SSH to machine") - (log-info " 2. Run: sudo nixos-rebuild switch --rollback") - #f) diff --git a/packages/lab-tool/research/guile-mcp-server.scm b/packages/lab-tool/research/guile-mcp-server.scm deleted file mode 100644 index 4f0ad7e..0000000 --- a/packages/lab-tool/research/guile-mcp-server.scm +++ /dev/null @@ -1,348 +0,0 @@ -#!/usr/bin/env guile -!# - -;; Guile MCP Server for Home Lab Integration -;; Implements Model Context Protocol for VS Code extension - -(use-modules (json) - (ice-9 textual-ports) - (ice-9 popen) - (ice-9 rdelim) - (ice-9 match) - (ice-9 threads) - (srfi srfi-1) - (srfi srfi-19) - (srfi srfi-26)) - -;; MCP Protocol Implementation -(define mcp-protocol-version "2024-11-05") -(define request-id-counter 0) - -;; Server capabilities and state -(define server-capabilities - `((tools . ()) - (resources . ()) - (prompts . ()))) - -(define server-info - `((name . "guile-homelab-mcp") - (version . "0.1.0"))) - -;; Request/Response utilities -(define (make-response id result) - `((jsonrpc . "2.0") - (id . ,id) - (result . ,result))) - -(define (make-error id code message) - `((jsonrpc . "2.0") - (id . ,id) - (error . ((code . ,code) - (message . ,message))))) - -(define (send-response response) - (let ((json-str (scm->json-string response))) - (display json-str) - (newline) - (force-output))) - -;; Home Lab Tools Implementation -(define (list-machines) - "List all available machines in the home lab" - (let* ((proc (open-input-pipe "find /etc/nixos/hosts -name '*.nix' -type f")) - (output (read-string proc))) - (close-pipe proc) - (if (string-null? output) - '() - (map (lambda (path) - (basename path ".nix")) - (string-split (string-trim-right output #\newline) #\newline))))) - -(define (get-machine-status machine) - "Get status of a specific machine" - (let* ((cmd (format #f "ping -c 1 -W 1 ~a > /dev/null 2>&1" machine)) - (status (system cmd))) - (if (= status 0) "online" "offline"))) - -(define (deploy-machine machine method) - "Deploy configuration to a machine" - (match method - ("deploy-rs" - (let ((cmd (format #f "deploy '.#~a'" machine))) - (deploy-with-command cmd machine))) - ("hybrid-update" - (let ((cmd (format #f "nixos-rebuild switch --flake '.#~a' --target-host ~a --use-remote-sudo" machine machine))) - (deploy-with-command cmd machine))) - ("legacy" - (let ((cmd (format #f "nixos-rebuild switch --flake '.#~a'" machine))) - (deploy-with-command cmd machine))) - (_ (throw 'deployment-error "Unknown deployment method" method)))) - -(define (deploy-with-command cmd machine) - "Execute deployment command and return result" - (let* ((proc (open-input-pipe cmd)) - (output (read-string proc)) - (status (close-pipe proc))) - `((success . ,(= status 0)) - (output . ,output) - (machine . ,machine) - (timestamp . ,(date->string (current-date)))))) - -(define (generate-nix-config machine-name services) - "Generate NixOS configuration for a new machine" - (let ((config (format #f "# Generated NixOS configuration for ~a -# Generated on ~a - -{ config, pkgs, ... }: - -{ - imports = [ - ./hardware-configuration.nix - ]; - - # Machine name - networking.hostName = \"~a\"; - - # Basic system configuration - system.stateVersion = \"23.11\"; - - # Enable services -~a - - # Network configuration - networking.firewall.enable = true; - - # SSH access - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keys = [ - # Add your public key here - ]; -} -" - machine-name - (date->string (current-date)) - machine-name - (string-join - (map (lambda (service) - (format #f " services.~a.enable = true;" service)) - services) - "\n")))) - `((content . ,config) - (filename . ,(format #f "~a.nix" machine-name))))) - -(define (get-infrastructure-status) - "Get comprehensive infrastructure status" - (let* ((machines (list-machines)) - (machine-status (map (lambda (m) - `((name . ,m) - (status . ,(get-machine-status m)))) - machines))) - `((machines . ,machine-status) - (timestamp . ,(date->string (current-date))) - (total-machines . ,(length machines)) - (online-machines . ,(length (filter (lambda (m) - (equal? (assoc-ref m 'status) "online")) - machine-status)))))) - -;; MCP Tools Registry -(define mcp-tools - `(((name . "deploy-machine") - (description . "Deploy NixOS configuration to a home lab machine") - (inputSchema . ((type . "object") - (properties . ((machine . ((type . "string") - (description . "Machine hostname to deploy to"))) - (method . ((type . "string") - (enum . ("deploy-rs" "hybrid-update" "legacy")) - (description . "Deployment method to use"))))) - (required . ("machine" "method"))))) - - ((name . "list-machines") - (description . "List all available machines in the home lab") - (inputSchema . ((type . "object") - (properties . ())))) - - ((name . "check-status") - (description . "Check status of home lab infrastructure") - (inputSchema . ((type . "object") - (properties . ((machine . ((type . "string") - (description . "Specific machine to check (optional)"))))))) - - ((name . "generate-nix-config") - (description . "Generate NixOS configuration for a new machine") - (inputSchema . ((type . "object") - (properties . ((machine-name . ((type . "string") - (description . "Name for the new machine"))) - (services . ((type . "array") - (items . ((type . "string"))) - (description . "List of services to enable"))))) - (required . ("machine-name"))))) - - ((name . "list-services") - (description . "List available NixOS services") - (inputSchema . ((type . "object") - (properties . ())))))) - -;; MCP Resources Registry -(define mcp-resources - `(((uri . "homelab://status/all") - (name . "Infrastructure Status") - (description . "Complete status of all home lab machines and services") - (mimeType . "application/json")) - - ((uri . "homelab://status/summary") - (name . "Status Summary") - (description . "Summary of infrastructure health") - (mimeType . "text/plain")) - - ((uri . "homelab://context/copilot") - (name . "Copilot Context") - (description . "Context information for GitHub Copilot integration") - (mimeType . "text/markdown")))) - -;; Tool execution dispatcher -(define (execute-tool name arguments) - "Execute a registered MCP tool" - (match name - ("deploy-machine" - (let ((machine (assoc-ref arguments 'machine)) - (method (assoc-ref arguments 'method))) - (deploy-machine machine method))) - - ("list-machines" - `((machines . ,(list-machines)))) - - ("check-status" - (let ((machine (assoc-ref arguments 'machine))) - (if machine - `((machine . ,machine) - (status . ,(get-machine-status machine))) - (get-infrastructure-status)))) - - ("generate-nix-config" - (let ((machine-name (assoc-ref arguments 'machine-name)) - (services (or (assoc-ref arguments 'services) '()))) - (generate-nix-config machine-name services))) - - ("list-services" - `((services . ("nginx" "postgresql" "redis" "mysql" "docker" "kubernetes" - "grafana" "prometheus" "gitea" "nextcloud" "jellyfin")))) - - (_ (throw 'unknown-tool "Tool not found" name)))) - -;; Resource content providers -(define (get-resource-content uri) - "Get content for a resource URI" - (match uri - ("homelab://status/all" - `((content . ,(get-infrastructure-status)))) - - ("homelab://status/summary" - (let ((status (get-infrastructure-status))) - `((content . ,(format #f "Home Lab Status: ~a/~a machines online" - (assoc-ref status 'online-machines) - (assoc-ref status 'total-machines)))))) - - ("homelab://context/copilot" - (let ((status (get-infrastructure-status))) - `((content . ,(format #f "# Home Lab Infrastructure Context - -## Current Status -- Total Machines: ~a -- Online Machines: ~a -- Last Updated: ~a - -## Available Operations -Use the home lab extension commands or MCP tools for: -- Deploying configurations (deploy-machine) -- Checking infrastructure status (check-status) -- Generating new machine configs (generate-nix-config) -- Managing services across the fleet - -## Machine List -~a - -This context helps GitHub Copilot understand your home lab infrastructure state." - (assoc-ref status 'total-machines) - (assoc-ref status 'online-machines) - (assoc-ref status 'timestamp) - (string-join - (map (lambda (m) - (format #f "- ~a: ~a" - (assoc-ref m 'name) - (assoc-ref m 'status))) - (assoc-ref status 'machines)) - "\n")))))) - - (_ (throw 'unknown-resource "Resource not found" uri)))) - -;; MCP Protocol Handlers -(define (handle-initialize params) - "Handle MCP initialize request" - `((protocolVersion . ,mcp-protocol-version) - (capabilities . ((tools . ((listChanged . #f))) - (resources . ((subscribe . #f) - (listChanged . #f))) - (prompts . ((listChanged . #f))))) - (serverInfo . ,server-info))) - -(define (handle-tools-list params) - "Handle tools/list request" - `((tools . ,mcp-tools))) - -(define (handle-tools-call params) - "Handle tools/call request" - (let ((name (assoc-ref params 'name)) - (arguments (assoc-ref params 'arguments))) - (execute-tool name arguments))) - -(define (handle-resources-list params) - "Handle resources/list request" - `((resources . ,mcp-resources))) - -(define (handle-resources-read params) - "Handle resources/read request" - (let ((uri (assoc-ref params 'uri))) - (get-resource-content uri))) - -;; Main request dispatcher -(define (handle-request request) - "Main request handler" - (let ((method (assoc-ref request 'method)) - (params (assoc-ref request 'params)) - (id (assoc-ref request 'id))) - - (catch #t - (lambda () - (let ((result - (match method - ("initialize" (handle-initialize params)) - ("tools/list" (handle-tools-list params)) - ("tools/call" (handle-tools-call params)) - ("resources/list" (handle-resources-list params)) - ("resources/read" (handle-resources-read params)) - (_ (throw 'method-not-found "Method not supported" method))))) - (send-response (make-response id result)))) - - (lambda (key . args) - (send-response (make-error id -32603 (format #f "~a: ~a" key args))))))) - -;; Main server loop -(define (run-mcp-server) - "Run the MCP server main loop" - (let loop () - (let ((line (read-line))) - (unless (eof-object? line) - (catch #t - (lambda () - (let ((request (json-string->scm line))) - (handle-request request))) - (lambda (key . args) - (send-response (make-error 0 -32700 "Parse error")))) - (loop))))) - -;; Export main function for use as module -(define-public run-mcp-server run-mcp-server) - -;; Run server if called directly -(when (equal? (car (command-line)) (current-filename)) - (run-mcp-server)) diff --git a/packages/lab-tool/research/guile.md b/packages/lab-tool/research/guile.md deleted file mode 100644 index dd0d706..0000000 --- a/packages/lab-tool/research/guile.md +++ /dev/null @@ -1,846 +0,0 @@ -# Guile Scheme Coding Instructions for Home Lab Tool - -## Functional Programming Principles - -**Core Philosophy**: Functional programming is about actions, data, and computation - compose small, pure functions to build complex behaviors. - -### 1. Pure Functions First -- Functions should be deterministic and side-effect free when possible -- Separate pure computation from I/O operations -- Use immutable data structures as default - -```scheme -;; Good: Pure function -(define (calculate-deployment-hash config) - (sha256 (scm->json-string config))) - -;; Better: Separate pure logic from I/O -(define (deployment-ready? machine-config current-state) - (and (eq? (assoc-ref machine-config 'status) 'configured) - (eq? (assoc-ref current-state 'connectivity) 'online))) - -;; I/O operations separate -(define (check-machine-deployment machine) - (let ((config (load-machine-config machine)) - (state (probe-machine-state machine))) - (deployment-ready? config state))) -``` - -### 2. Data-Driven Design -- Represent configurations and state as data structures -- Use association lists (alists) and vectors for structured data -- Leverage Guile's homoiconicity (code as data) - -```scheme -;; Machine configuration as data -(define machine-specs - `((grey-area - (services (ollama jellyfin forgejo)) - (deployment-method deploy-rs) - (backup-schedule weekly)) - (sleeper-service - (services (nfs zfs monitoring)) - (deployment-method hybrid-update) - (backup-schedule daily)))) - -;; Operations on data -(define (get-machine-services machine) - (assoc-ref (assoc-ref machine-specs machine) 'services)) - -(define (machines-with-service service) - (filter (lambda (machine-spec) - (member service (get-machine-services (car machine-spec)))) - machine-specs)) -``` - -## Guile-Specific Idioms - -### 3. Module Organization -- Use meaningful module hierarchies -- Export only necessary public interfaces -- Group related functionality together - -```scheme -;; File: modules/lab/machines.scm -(define-module (lab machines) - #:use-module (srfi srfi-1) ; List processing - #:use-module (srfi srfi-26) ; Cut/cute - #:use-module (ice-9 match) ; Pattern matching - #:use-module (ssh session) - #:export (machine-status - deploy-machine - list-machines - machine-services)) - -;; File: modules/lab/deployment.scm -(define-module (lab deployment) - #:use-module (lab machines) - #:use-module (json) - #:export (deploy-rs - hybrid-update - rollback-deployment)) -``` - -### 4. Error Handling the Scheme Way -- Use exceptions for exceptional conditions -- Return #f or special values for expected failures -- Provide meaningful error context - -```scheme -;; Use exceptions for programming errors -(define (deploy-machine machine method) - (unless (member machine (list-machines)) - (throw 'invalid-machine "Unknown machine" machine)) - (unless (member method '(deploy-rs hybrid-update legacy)) - (throw 'invalid-method "Unknown deployment method" method)) - ;; ... deployment logic) - -;; Return #f for expected failures -(define (machine-reachable? machine) - (catch #t - (lambda () - (ssh-connect machine) - #t) - (lambda (key . args) - #f))) - -;; Provide context with failure info -(define (deployment-result success? machine method details) - `((success . ,success?) - (machine . ,machine) - (method . ,method) - (timestamp . ,(current-time)) - (details . ,details))) -``` - -### 5. Higher-Order Functions and Composition -- Use map, filter, fold for list processing -- Compose functions to build complex operations -- Leverage SRFI-1 for advanced list operations - -```scheme -(use-modules (srfi srfi-1)) - -;; Functional composition -(define (healthy-machines machines) - (filter machine-reachable? - (filter (lambda (m) (not (maintenance-mode? m))) - machines))) - -;; Map operations across machines -(define (update-all-machines) - (map (lambda (machine) - (cons machine (update-machine machine))) - (healthy-machines (list-machines)))) - -;; Fold for aggregation -(define (deployment-summary results) - (fold (lambda (result acc) - (if (assoc-ref result 'success) - (cons 'successful (1+ (assoc-ref acc 'successful))) - (cons 'failed (1+ (assoc-ref acc 'failed))))) - '((successful . 0) (failed . 0)) - results)) -``` - -### 6. Pattern Matching for Control Flow -- Use `match` for destructuring and dispatch -- Pattern match on data structures -- Cleaner than nested if/cond statements - -```scheme -(use-modules (ice-9 match)) - -(define (handle-deployment-event event) - (match event - (('start machine method) - (log-info "Starting deployment of ~a using ~a" machine method)) - - (('progress machine percent) - (update-progress-bar machine percent)) - - (('success machine result) - (log-success "Deployment completed: ~a" machine) - (notify-success machine result)) - - (('error machine error-msg) - (log-error "Deployment failed: ~a - ~a" machine error-msg) - (initiate-rollback machine)) - - (_ (log-warning "Unknown event: ~a" event)))) - -;; Pattern matching for configuration parsing -(define (parse-machine-config config-sexp) - (match config-sexp - (('machine name ('services services ...) ('options options ...)) - `((name . ,name) - (services . ,services) - (options . ,(alist->hash-table options)))) - - (_ (throw 'invalid-config "Malformed machine config" config-sexp)))) -``` - -### 7. REPL-Driven Development -- Design for interactive development -- Provide introspection functions -- Make state queryable and modifiable - -```scheme -;; REPL helpers for development -(define (debug-machine-state machine) - "Display comprehensive machine state for debugging" - (format #t "Machine: ~a~%" machine) - (format #t "Status: ~a~%" (machine-status machine)) - (format #t "Services: ~a~%" (machine-services machine)) - (format #t "Last deployment: ~a~%" (last-deployment machine)) - (format #t "Reachable: ~a~%" (machine-reachable? machine))) - -;; Interactive deployment with confirmation -(define (interactive-deploy machine) - (let ((current-config (get-machine-config machine))) - (display-config current-config) - (when (yes-or-no? "Proceed with deployment?") - (deploy-machine machine 'deploy-rs)))) - -;; State introspection -(define (lab-status) - `((total-machines . ,(length (list-machines))) - (reachable . ,(length (filter machine-reachable? (list-machines)))) - (services-running . ,(total-running-services)) - (pending-deployments . ,(length (pending-deployments))))) -``` - -### 8. Concurrency with Fibers -- Use fibers for concurrent operations -- Non-blocking I/O for better performance -- Coordinate parallel deployments safely - -```scheme -(use-modules (fibers) (fibers channels)) - -;; Concurrent machine checking -(define (check-all-machines-concurrent machines) - (run-fibers - (lambda () - (let ((results-channel (make-channel))) - ;; Spawn fiber for each machine - (for-each (lambda (machine) - (spawn-fiber - (lambda () - (let ((status (check-machine-status machine))) - (put-message results-channel - (cons machine status)))))) - machines) - - ;; Collect results - (let loop ((remaining (length machines)) - (results '())) - (if (zero? remaining) - results - (loop (1- remaining) - (cons (get-message results-channel) results)))))))) - -;; Parallel deployment with coordination -(define (deploy-machines-parallel machines) - (run-fibers - (lambda () - (let ((deployment-channel (make-channel)) - (coordinator (spawn-fiber (deployment-coordinator deployment-channel)))) - (par-map (lambda (machine) - (deploy-with-coordination machine deployment-channel)) - machines))))) -``` - -### 9. MCP Server Implementation Patterns -- Structured message handling -- Capability-based tool organization -- Resource management with caching - -```scheme -;; MCP message dispatch -(define (handle-mcp-request request) - (match (json-ref request "method") - ("tools/list" - (mcp-tools-list)) - - ("tools/call" - (let ((tool (json-ref request "params" "name")) - (args (json-ref request "params" "arguments"))) - (call-lab-tool tool args))) - - ("resources/list" - (mcp-resources-list)) - - ("resources/read" - (let ((uri (json-ref request "params" "uri"))) - (read-lab-resource uri))) - - (method - (mcp-error -32601 "Method not found" method)))) - -;; Tool capability definition -(define lab-tools - `((deploy-machine - (description . "Deploy configuration to a specific machine") - (inputSchema . ,(json-schema - `((type . "object") - (properties . ((machine (type . "string")) - (method (type . "string") - (enum . ("deploy-rs" "hybrid-update"))))) - (required . ("machine"))))) - (handler . ,deploy-machine-tool)) - - (check-status - (description . "Check machine status and connectivity") - (inputSchema . ,(json-schema - `((type . "object") - (properties . ((machines (type . "array") - (items (type . "string")))))))) - (handler . ,check-status-tool)))) -``` - -### 10. Configuration and Environment -- Use parameters for configuration -- Environment-aware defaults -- Validate configuration on startup - -```scheme -;; Configuration parameters -(define lab-config-dir - (make-parameter (or (getenv "LAB_CONFIG_DIR") - "/etc/lab-tool"))) - -(define deployment-timeout - (make-parameter (string->number (or (getenv "DEPLOYMENT_TIMEOUT") "300")))) - -(define ssh-key-path - (make-parameter (or (getenv "LAB_SSH_KEY") - (string-append (getenv "HOME") "/.ssh/lab_key")))) - -;; Configuration validation -(define (validate-lab-config) - (unless (file-exists? (lab-config-dir)) - (throw 'config-error "Lab config directory not found" (lab-config-dir))) - - (unless (file-exists? (ssh-key-path)) - (throw 'config-error "SSH key not found" (ssh-key-path))) - - (unless (> (deployment-timeout) 0) - (throw 'config-error "Invalid deployment timeout" (deployment-timeout)))) - -;; Initialize with validation -(define (init-lab-tool) - (validate-lab-config) - (load-machine-configurations) - (initialize-ssh-agent) - (setup-logging)) -``` - -## Code Style Guidelines - -### 11. Naming Conventions -- Use kebab-case for variables and functions -- Predicates end with `?` -- Mutating procedures end with `!` -- Constants in ALL-CAPS with hyphens - -```scheme -;; Good naming -(define DEFAULT-SSH-PORT 22) -(define machine-deployment-status ...) -(define (machine-reachable? machine) ...) -(define (update-machine-config! machine config) ...) - -;; Avoid -(define defaultSSHPort 22) ; camelCase -(define machine_status ...) ; snake_case -(define (is-machine-reachable ...) ; unnecessary 'is-' -``` - -### 12. Documentation and Comments -- Document module purposes and exports -- Use docstrings for complex functions -- Comment the "why", not the "what" - -```scheme -(define (deploy-machine machine method) - "Deploy configuration to MACHINE using METHOD. - - Returns a deployment result alist with success status, timing, - and any error messages. May throw exceptions for invalid inputs." - - ;; Validate inputs early to fail fast - (validate-machine machine) - (validate-deployment-method method) - - ;; Use atomic operations to prevent partial deployments - (call-with-deployment-lock machine - (lambda () - (let ((start-time (current-time))) - ;; ... deployment logic - )))) -``` - -### 13. Testing Approach -- Write tests for pure functions first -- Mock I/O operations -- Use SRFI-64 testing framework - -```scheme -(use-modules (srfi srfi-64)) - -(test-begin "machine-configuration") - -(test-equal "machine services extraction" - '(ollama jellyfin forgejo) - (get-machine-services 'grey-area)) - -(test-assert "deployment readiness check" - (deployment-ready? - '((status . configured) (health . good)) - '((connectivity . online) (load . normal)))) - -(test-error "invalid machine throws exception" - 'invalid-machine - (deploy-machine 'non-existent-machine 'deploy-rs)) - -(test-end "machine-configuration") -``` - -## Project Structure Best Practices - -### 14. Module Organization -``` -modules/ -โ”œโ”€โ”€ lab/ -โ”‚ โ”œโ”€โ”€ core.scm ; Core data structures and utilities -โ”‚ โ”œโ”€โ”€ machines.scm ; Machine management -โ”‚ โ”œโ”€โ”€ deployment.scm ; Deployment strategies -โ”‚ โ”œโ”€โ”€ monitoring.scm ; Status checking and metrics -โ”‚ โ””โ”€โ”€ config.scm ; Configuration handling -โ”œโ”€โ”€ mcp/ -โ”‚ โ”œโ”€โ”€ server.scm ; MCP server implementation -โ”‚ โ”œโ”€โ”€ tools.scm ; MCP tool definitions -โ”‚ โ””โ”€โ”€ resources.scm ; MCP resource handlers -โ””โ”€โ”€ utils/ - โ”œโ”€โ”€ ssh.scm ; SSH utilities - โ”œโ”€โ”€ json.scm ; JSON helpers - โ””โ”€โ”€ logging.scm ; Logging facilities -``` - -### 15. Build and Development Workflow -- Use Guile's module compilation -- Leverage REPL for iterative development -- Provide development/production configurations - -```scheme -;; Development helpers in separate module -(define-module (lab dev) - #:use-module (lab core) - #:export (reload-config - reset-state - dev-deploy)) - -;; Hot-reload for development -(define (reload-config) - (reload-module (resolve-module '(lab config))) - (init-lab-tool)) - -;; Safe deployment for development -(define (dev-deploy machine) - (if (eq? (current-environment) 'development) - (deploy-machine machine 'deploy-rs) - (error "dev-deploy only available in development mode"))) -``` - -## VS Code and GitHub Copilot Integration - -### 16. MCP Client Integration with VS Code -- Implement MCP client in VS Code extension -- Bridge home lab context to Copilot -- Provide real-time infrastructure state - -```typescript -// VS Code extension structure for MCP integration -// File: vscode-extension/src/extension.ts -import * as vscode from 'vscode'; -import { MCPClient } from './mcp-client'; - -export function activate(context: vscode.ExtensionContext) { - const mcpClient = new MCPClient('stdio', { - command: 'guile', - args: ['-c', '(use-modules (mcp server)) (run-mcp-server)'] - }); - - // Register commands for home lab operations - const deployCommand = vscode.commands.registerCommand( - 'homelab.deploy', - async (machine: string) => { - const result = await mcpClient.callTool('deploy-machine', { - machine: machine, - method: 'deploy-rs' - }); - vscode.window.showInformationMessage( - `Deployment ${result.success ? 'succeeded' : 'failed'}` - ); - } - ); - - // Provide context to Copilot through workspace state - const statusProvider = new HomeLab StatusProvider(mcpClient); - context.subscriptions.push( - vscode.workspace.registerTextDocumentContentProvider( - 'homelab', statusProvider - ) - ); - - context.subscriptions.push(deployCommand); -} - -class HomeLabStatusProvider implements vscode.TextDocumentContentProvider { - constructor(private mcpClient: MCPClient) {} - - async provideTextDocumentContent(uri: vscode.Uri): Promise { - // Fetch current lab state for Copilot context - const resources = await this.mcpClient.listResources(); - const status = await this.mcpClient.readResource('machines://status/all'); - - return `# Home Lab Status -Current Infrastructure State: -${JSON.stringify(status, null, 2)} - -Available Resources: -${resources.map(r => `- ${r.uri}: ${r.description}`).join('\n')} -`; - } -} -``` - -### 17. MCP Server Configuration for IDE Integration -- Provide IDE-specific tools and resources -- Format responses for developer consumption -- Include code suggestions and snippets - -```scheme -;; IDE-specific MCP tools -(define ide-tools - `((generate-nix-config - (description . "Generate NixOS configuration for new machine") - (inputSchema . ,(json-schema - `((type . "object") - (properties . ((machine-name (type . "string")) - (services (type . "array") - (items (type . "string"))) - (hardware-profile (type . "string")))) - (required . ("machine-name"))))) - (handler . ,generate-nix-config-tool)) - - (suggest-deployment-strategy - (description . "Suggest optimal deployment strategy for changes") - (inputSchema . ,(json-schema - `((type . "object") - (properties . ((changed-files (type . "array") - (items (type . "string"))) - (target-machines (type . "array") - (items (type . "string"))))) - (required . ("changed-files"))))) - (handler . ,suggest-deployment-strategy-tool)) - - (validate-config - (description . "Validate NixOS configuration syntax and dependencies") - (inputSchema . ,(json-schema - `((type . "object") - (properties . ((config-path (type . "string")) - (machine (type . "string")))) - (required . ("config-path"))))) - (handler . ,validate-config-tool)))) - -;; IDE-specific resources -(define ide-resources - `(("homelab://templates/machine-config" - (description . "Template for new machine configuration") - (mimeType . "application/x-nix")) - - ("homelab://examples/service-configs" - (description . "Example service configurations") - (mimeType . "application/x-nix")) - - ("homelab://docs/deployment-guide" - (description . "Step-by-step deployment procedures") - (mimeType . "text/markdown")) - - ("homelab://status/real-time" - (description . "Real-time infrastructure status for context") - (mimeType . "application/json")))) - -;; Generate contextual code suggestions -(define (generate-nix-config-tool args) - (let ((machine-name (assoc-ref args "machine-name")) - (services (assoc-ref args "services")) - (hardware-profile (assoc-ref args "hardware-profile"))) - - `((content . ,(format #f "# Generated configuration for ~a -{ config, pkgs, ... }: - -{ - imports = [ - ./hardware-configuration.nix - ~/args - ]; - - # Machine-specific configuration - networking.hostName = \"~a\"; - - # Services configuration -~a - - # System packages - environment.systemPackages = with pkgs; [ - # Add your packages here - ]; - - system.stateVersion = \"24.05\"; -}" - machine-name - machine-name - (if services - (string-join - (map (lambda (service) - (format #f " services.~a.enable = true;" service)) - services) - "\n") - " # No services specified"))) - (isError . #f)))) -``` - -### 18. Copilot Context Enhancement -- Provide infrastructure context to improve suggestions -- Include deployment patterns and best practices -- Real-time system state for informed recommendations - -```scheme -;; Context provider for Copilot integration -(define (provide-copilot-context) - `((infrastructure-state . ,(get-current-infrastructure-state)) - (deployment-patterns . ,(get-common-deployment-patterns)) - (service-configurations . ,(get-service-config-templates)) - (best-practices . ,(get-deployment-best-practices)) - (current-issues . ,(get-active-alerts)))) - -(define (get-current-infrastructure-state) - `((machines . ,(map (lambda (machine) - `((name . ,machine) - (status . ,(machine-status machine)) - (services . ,(machine-services machine)) - (last-deployment . ,(last-deployment-time machine)))) - (list-machines))) - (network-topology . ,(get-network-topology)) - (resource-usage . ,(get-resource-utilization)))) - -(define (get-common-deployment-patterns) - `((safe-deployment . "Use deploy-rs for production, hybrid-update for development") - (rollback-strategy . "Always test deployments in staging first") - (service-dependencies . "Ensure database services start before applications") - (backup-before-deploy . "Create snapshots before major configuration changes"))) - -;; Format context for IDE consumption -(define (format-ide-context context) - (scm->json-string context #:pretty #t)) -``` - -### 19. VS Code Extension Development -- Create extension for seamless MCP integration -- Provide commands, views, and context -- Enable real-time collaboration with infrastructure - -```typescript -// package.json for VS Code extension -{ - "name": "homelab-mcp-integration", - "displayName": "Home Lab MCP Integration", - "description": "Integrate home lab infrastructure with VS Code through MCP", - "version": "0.1.0", - "engines": { - "vscode": "^1.74.0" - }, - "categories": ["Other"], - "activationEvents": [ - "onCommand:homelab.connect", - "workspaceContains:**/flake.nix" - ], - "main": "./out/extension.js", - "contributes": { - "commands": [ - { - "command": "homelab.deploy", - "title": "Deploy Machine", - "category": "Home Lab" - }, - { - "command": "homelab.status", - "title": "Check Status", - "category": "Home Lab" - }, - { - "command": "homelab.generateConfig", - "title": "Generate Config", - "category": "Home Lab" - } - ], - "views": { - "explorer": [ - { - "id": "homelabStatus", - "name": "Home Lab Status", - "when": "homelab:connected" - } - ] - }, - "viewsContainers": { - "activitybar": [ - { - "id": "homelab", - "title": "Home Lab", - "icon": "$(server-environment)" - } - ] - } - } -} - -// MCP Client implementation -class MCPClient { - private transport: MCPTransport; - private capabilities: MCPCapabilities; - - constructor(transportType: 'stdio' | 'websocket', config: any) { - this.transport = this.createTransport(transportType, config); - this.initialize(); - } - - async callTool(name: string, arguments: any): Promise { - return this.transport.request('tools/call', { - name: name, - arguments: arguments - }); - } - - async listResources(): Promise { - const response = await this.transport.request('resources/list', {}); - return response.resources; - } - - async readResource(uri: string): Promise { - return this.transport.request('resources/read', { uri }); - } - - // Integration with Copilot context - async getCopilotContext(): Promise { - const context = await this.readResource('homelab://context/copilot'); - return context.content; - } -} -``` - -### 20. GitHub Copilot Workspace Integration -- Configure workspace for optimal Copilot suggestions -- Provide infrastructure context files -- Set up context patterns for deployment scenarios - -```json -// .vscode/settings.json -{ - "github.copilot.enable": { - "*": true, - "yaml": true, - "nix": true, - "scheme": true - }, - "github.copilot.advanced": { - "length": 500, - "temperature": 0.2 - }, - "homelab.mcpServer": { - "command": "guile", - "args": ["-L", "modules", "-c", "(use-modules (mcp server)) (run-mcp-server)"], - "autoStart": true - }, - "files.associations": { - "*.scm": "scheme", - "flake.lock": "json" - } -} - -// .copilot/context.md for workspace context -```markdown -# Home Lab Infrastructure Context - -## Current Architecture -- NixOS-based infrastructure with multiple machines -- Deploy-rs for safe deployments -- Services: Ollama, Jellyfin, Forgejo, NFS, ZFS -- Network topology: reverse-proxy, grey-area, sleeper-service, congenital-optimist - -## Common Patterns -- Use `deploy-rs` for production deployments -- Test with `hybrid-update` in development -- Always backup before major changes -- Follow NixOS module structure in `/modules/` - -## Configuration Standards -- Machine configs in `/machines/{hostname}/` -- Shared modules in `/modules/` -- Service-specific configs in `services/` subdirectories -``` - -### 21. Real-time Context Updates -- Stream infrastructure changes to VS Code -- Update Copilot context automatically -- Provide deployment feedback in editor - -```scheme -;; Real-time context streaming -(define (start-context-stream port) - "Stream infrastructure changes to connected IDE clients" - (let ((clients (make-hash-table))) - (spawn-fiber - (lambda () - (let loop () - (let ((update (get-infrastructure-update))) - (hash-for-each - (lambda (client-id websocket) - (catch #t - (lambda () - (websocket-send websocket - (scm->json-string update))) - (lambda (key . args) - (hash-remove! clients client-id)))) - clients) - (sleep 5) - (loop))))) - - ;; WebSocket server for IDE connections - (run-websocket-server - (lambda (ws) - (let ((client-id (generate-client-id))) - (hash-set! clients client-id ws) - (websocket-send ws - (scm->json-string - `((type . "welcome") - (context . ,(get-current-context))))) - (handle-client-messages ws client-id clients))) - #:port port))) - -;; Integration with file watchers -(define (watch-config-changes) - "Watch for configuration file changes and update context" - (file-system-watcher - (list "/home/geir/Home-lab/machines" - "/home/geir/Home-lab/modules") - (lambda (event) - (match event - (('modify path) - (when (string-suffix? ".nix" path) - (update-copilot-context path))) - (_ #f))))) -``` \ No newline at end of file diff --git a/packages/lab-tool/research/guile_ecosystem.md b/packages/lab-tool/research/guile_ecosystem.md deleted file mode 100644 index 9001102..0000000 --- a/packages/lab-tool/research/guile_ecosystem.md +++ /dev/null @@ -1,394 +0,0 @@ - -# Guile Scheme Ecosystem Analysis for Home Lab Tool Migration and MCP Integration - -## Executive Summary - -This analysis examines the GNU Guile Scheme ecosystem to evaluate its suitability for migrating the home lab tool from Bash and potentially implementing a Model Context Protocol (MCP) server. Based on comprehensive research, Guile offers a robust ecosystem with numerous libraries that address the core requirements of modern system administration, networking, and infrastructure management. - -**Key Findings:** - -- **Rich ecosystem**: 200+ libraries available through GNU Guix ecosystem -- **Strong system administration capabilities**: SSH, system interaction, process management -- **Excellent networking support**: HTTP servers/clients, WebSocket, JSON-RPC -- **Mature infrastructure**: Well-maintained libraries with active development -- **MCP compatibility**: All necessary components available for MCP server implementation - -## Current State Analysis - -### Existing Lab Tool Capabilities - -Based on the documentation, the current lab tool provides: - -- Machine status checking and connectivity -- Multiple deployment methods (deploy-rs, hybrid-update, legacy) -- NixOS configuration management -- SSH-based operations -- Package updates via flake management - -### Migration Benefits to Guile - -1. **Enhanced error handling** over Bash's limited error management -2. **Structured data handling** for machine configurations and status -3. **Better modularity** and code organization -4. **Advanced networking capabilities** for future expansion -5. **REPL-driven development** for rapid prototyping and debugging - -## Core Libraries for Home Lab Tool Migration - -### 1. System Administration & SSH - -**guile-ssh** - *Essential for remote operations* - -- **Capabilities**: SSH client/server, SFTP, port forwarding, tunneling -- **Use cases**: All remote machine interactions, deployment coordination -- **Maturity**: Very mature, actively maintained -- **Documentation**: Comprehensive with examples - -```scheme -;; Example SSH connection and command execution -(use-modules (ssh session) (ssh channel)) -(let ((session (make-session #:host "sleeper-service"))) - (connect! session) - (authenticate-server session) - (userauth-public-key! session key) - ;; Execute nixos-rebuild or other commands - (call-with-remote-output-pipe session "nixos-rebuild switch" - (lambda (port) (display (read-string port))))) -``` - -### 2. JSON Data Handling - -**guile-json** - *For structured configuration and API communication* - -- **Capabilities**: JSON parsing/generation, RFC 7464 support, pretty printing -- **Use cases**: Configuration management, API responses, deployment metadata -- **Features**: JSON Text Sequences, record mapping, validation - -```scheme -;; Machine configuration as JSON -(define machine-config - `(("name" . "grey-area") - ("services" . #("ollama" "jellyfin" "forgejo")) - ("deployment" . (("method" . "deploy-rs") ("status" . "ready"))))) - -(scm->json machine-config #:pretty #t) -``` - -### 3. HTTP Server/Client Operations - -**guile-webutils** & **guile-curl** - *For web-based interfaces and API calls* - -- **guile-webutils**: Session management, multipart messages, form handling -- **guile-curl**: HTTP client operations, file transfers -- **Use cases**: Web dashboard, API endpoints, remote service integration - -### 4. Process Management & System Interaction - -**guile-bash** - *Bridge between Scheme and shell operations* - -- **Capabilities**: Execute shell commands, capture output, dynamic variables -- **Use cases**: Gradual migration, leveraging existing shell tools -- **Integration**: Call existing scripts while building Scheme alternatives - -### 5. Configuration Management - -**guile-config** - *Declarative configuration handling* - -- **Capabilities**: Declarative config specs, file parsing, command-line args -- **Use cases**: Tool configuration, machine definitions, deployment parameters - -## MCP Server Implementation Libraries - -### 1. JSON-RPC Foundation - -**scheme-json-rpc** - *Core MCP protocol implementation* - -- **Capabilities**: JSON-RPC 2.0 specification compliance -- **Transport**: Works over stdio, WebSocket, HTTP -- **Use cases**: MCP message handling, method dispatch - -### 2. WebSocket Support - -**guile-websocket** - *Real-time communication* - -- **Capabilities**: RFC 6455 compliant WebSocket implementation -- **Features**: Server and client support, binary/text messages -- **Use cases**: MCP transport layer, real-time lab monitoring - -### 3. Web Server Infrastructure - -**artanis** - *Full-featured web application framework* - -- **Capabilities**: Routing, templating, database access, session management -- **Use cases**: MCP HTTP transport, web dashboard, API endpoints - -```scheme -;; MCP server endpoint structure -(define-handler mcp-handler - (lambda (request) - (let ((method (json-ref (request-body request) "method"))) - (case method - (("tools/list") (handle-tools-list)) - (("resources/list") (handle-resources-list)) - (("tools/call") (handle-tool-call request)) - (else (mcp-error "Unknown method")))))) -``` - -## Enhanced Networking & Protocol Libraries - -### 1. Advanced HTTP/Network Operations - -**guile-curl** - *Comprehensive HTTP client* - -- Features: HTTPS, authentication, file uploads, progress callbacks -- Use cases: API integrations, file transfers, service health checks - -**guile-dns** - *DNS operations* - -- Pure Guile DNS implementation -- Use cases: Service discovery, network diagnostics - -### 2. Data Serialization - -**guile-cbor** - *Efficient binary serialization* - -- Alternative to JSON for performance-critical operations -- Smaller payload sizes for resource monitoring - -**guile-yaml** / **guile-yamlpp** - *YAML processing* - -- Configuration file handling -- Integration with existing YAML-based tools - -### 3. Database Integration - -**guile-sqlite3** - *Local data storage* - -- Deployment history, machine states, configuration versioning -- Embedded database for tool state management - -**guile-redis** - *Caching and session storage* - -- Performance optimization for frequent operations -- Distributed state management across lab machines - -## System Integration Libraries - -### 1. File System Operations - -**guile-filesystem** & **f.scm** - *Enhanced file handling* - -- Beyond basic Guile file operations -- Path manipulation, directory traversal, file monitoring - -### 2. Process and Service Management - -**shepherd** - *Service management* - -- GNU Shepherd integration for service lifecycle management -- Alternative to systemd interactions - -### 3. Cryptography and Security - -**guile-gcrypt** - *Cryptographic operations* - -- Key management, encryption/decryption, hashing -- Secure configuration storage, deployment verification - -## Specialized Infrastructure Libraries - -### 1. Containerization Support - -**guile-docker** / Container operations - -- Docker/Podman integration for containerized services -- Image management, container lifecycle - -### 2. Version Control Integration - -**guile-git** - *Git operations* - -- Flake updates, configuration versioning -- Automated commit/push for deployment tracking - -### 3. Monitoring and Metrics - -**prometheus** (Guile implementation) - *Metrics collection* - -- Performance monitoring, deployment success rates -- Integration with existing monitoring infrastructure - -## MCP Server Implementation Strategy - -### Core MCP Capabilities to Implement - -1. **Tools**: Home lab management operations - - `deploy-machine`: Deploy specific machine configurations - - `check-status`: Machine connectivity and health checks - - `update-flake`: Update package definitions - - `rollback-deployment`: Emergency rollback procedures - -2. **Resources**: Lab state and configuration access - - Machine configurations (read-only access to NixOS configs) - - Deployment history and logs - - Service status across all machines - - Network topology and connectivity maps - -3. **Prompts**: Common operational templates - - Deployment workflows - - Troubleshooting procedures - - Security audit checklists - -### Implementation Architecture - -```scheme -(use-modules (json) (web socket) (ssh session) (scheme json-rpc)) - -(define-mcp-server home-lab-mcp - #:tools `(("deploy-machine" - #:description "Deploy configuration to specified machine" - #:parameters ,(make-schema-object - `(("machine" #:type "string" #:required #t) - ("method" #:type "string" #:enum ("deploy-rs" "hybrid-update"))))) - - ("check-status" - #:description "Check machine connectivity and services" - #:parameters ,(make-schema-object - `(("machines" #:type "array" #:items "string"))))) - - #:resources `(("machines://config/{machine}" - #:description "NixOS configuration for machine") - ("machines://status/{machine}" - #:description "Current status and health metrics")) - - #:prompts `(("deployment-workflow" - #:description "Standard deployment procedure") - ("troubleshoot-machine" - #:description "Machine diagnostics checklist"))) -``` - -## Migration Strategy - -### Phase 1: Core Infrastructure (Weeks 1-2) - -1. Set up Guile development environment in NixOS -2. Implement basic SSH operations using guile-ssh -3. Port status checking functionality -4. Create JSON-based machine configuration format - -### Phase 2: Enhanced Features (Weeks 3-4) - -1. Implement deployment methods (deploy-rs integration) -2. Add error handling and logging -3. Create web interface for monitoring -4. Develop basic MCP server capabilities - -### Phase 3: Advanced Integration (Weeks 5-6) - -1. Full MCP server implementation -2. Web dashboard with real-time updates -3. Integration with existing monitoring tools -4. Documentation and testing - -### Phase 4: Production Deployment (Week 7) - -1. Gradual rollout with fallback to Bash tool -2. Performance optimization -3. User training and documentation -4. Monitoring and feedback collection - -## Guile vs. Alternative Languages - -### Advantages of Guile - -- **Homoiconicity**: Code as data enables powerful metaprogramming -- **REPL Development**: Interactive development and debugging -- **GNU Integration**: Seamless integration with GNU tools and philosophy -- **Extensibility**: Easy C library bindings for performance-critical code -- **Stability**: Mature language with stable API - -### Considerations - -- **Learning Curve**: Lisp syntax may be unfamiliar -- **Performance**: Generally slower than compiled languages for CPU-intensive tasks -- **Ecosystem Size**: Smaller than Python/JavaScript ecosystems -- **Tooling**: Fewer IDE integrations compared to mainstream languages - -## Recommended Libraries by Priority - -### Tier 1 (Essential) - -1. **guile-ssh** - Remote operations foundation -2. **guile-json** - Data interchange format -3. **scheme-json-rpc** - MCP protocol implementation -4. **guile-webutils** - Web application utilities - -### Tier 2 (Important) - -1. **guile-websocket** - Real-time communication -2. **artanis** - Web framework -3. **guile-curl** - HTTP client operations -4. **guile-config** - Configuration management - -### Tier 3 (Enhancement) - -1. **guile-git** - Version control integration -2. **guile-sqlite3** - Local data storage -3. **prometheus** - Metrics and monitoring -4. **guile-gcrypt** - Security operations - -## Security Considerations - -### Authentication and Authorization - -- **guile-ssh**: Public key authentication, agent support -- **guile-gcrypt**: Secure credential storage -- **MCP Security**: Implement capability-based access control - -### Network Security - -- **TLS Support**: Via guile-gnutls for encrypted communications -- **SSH Tunneling**: Secure communication channels -- **Input Validation**: JSON schema validation for all inputs - -### Deployment Security - -- **Signed Deployments**: Cryptographic verification of configurations -- **Audit Logging**: Comprehensive operation logging -- **Rollback Capability**: Quick recovery from failed deployments - -## Performance Considerations - -### Optimization Strategies - -1. **Compiled Modules**: Use `.go` files for performance-critical code -2. **Async Operations**: Leverage fibers for concurrent operations -3. **Caching**: Redis integration for frequently accessed data -4. **Native Extensions**: C bindings for system-level operations - -### Expected Performance - -- **SSH Operations**: Comparable to native SSH client -- **JSON Processing**: Adequate for configuration sizes (< 1MB) -- **Web Serving**: Suitable for low-traffic administrative interfaces -- **Startup Time**: Fast REPL startup, moderate for compiled applications - -## Conclusion - -The Guile ecosystem provides comprehensive support for implementing both a sophisticated home lab management tool and a Model Context Protocol server. The availability of mature libraries for SSH operations, JSON handling, web services, and system integration makes Guile an excellent choice for this migration. - -**Key Strengths:** - -- Rich library ecosystem specifically suited to system administration -- Excellent JSON-RPC and WebSocket support for MCP implementation -- Strong SSH and networking capabilities -- Active development community with good documentation - -**Recommended Approach:** - -1. Start with core SSH and JSON functionality -2. Gradually migrate features from Bash to Guile -3. Implement MCP server capabilities incrementally -4. Maintain backwards compatibility during transition - -The migration to Guile will provide significant benefits in code maintainability, error handling, and extensibility while enabling advanced features like MCP integration that would be difficult to implement in Bash. diff --git a/packages/lab-tool/research/guile_scripting_solution.md b/packages/lab-tool/research/guile_scripting_solution.md deleted file mode 100644 index 295d292..0000000 --- a/packages/lab-tool/research/guile_scripting_solution.md +++ /dev/null @@ -1,334 +0,0 @@ -# Replacing Bash with Guile Scheme for Home Lab Tools - -This document outlines a proposal to migrate the `home-lab-tools` script from Bash to GNU Guile Scheme. This change aims to address the increasing complexity of the script and leverage the benefits of a more powerful programming language. - -## 1. Introduction: Why Guile Scheme? - -GNU Guile is the official extension language for the GNU Project. It is an implementation of the Scheme programming language, a dialect of Lisp. Using Guile for scripting offers several advantages over Bash, especially as scripts grow in size and complexity. - -Key reasons for considering Guile: - -* **Expressiveness and Power:** Scheme is a full-fledged programming language with features like first-class functions, macros, and a rich standard library. This allows for more elegant and maintainable solutions to complex problems. -* **Better Error Handling:** Guile provides robust error handling mechanisms (conditions and handlers) that are more sophisticated than Bash's `set -e` and trap. -* **Modularity:** Guile supports modules, making it easier to organize code into reusable components. -* **Data Manipulation:** Scheme excels at handling structured data, which can be beneficial for managing configurations or parsing output from commands. -* **Readability (for Lisp programmers):** While Lisp syntax can be initially unfamiliar, it can lead to very clear and concise code once learned. -* **Interoperability:** Guile can easily call external programs and libraries, and can be extended with C code if needed. - -## 2. Advantages over Bash for `home-lab-tools` - -Migrating `home-lab-tools` from Bash to Guile offers specific benefits: - -* **Improved Logic Handling:** Complex conditional logic, loops, and function definitions are more naturally expressed in Guile. The current Bash script uses case statements and string comparisons extensively, which can become unwieldy. -* **Structured Data Management:** Machine definitions, deployment modes, and status information could be represented as Scheme data structures (lists, association lists, records), making them easier to manage and query. -* **Enhanced Error Reporting:** More descriptive error messages and better control over script termination in case of failures. -* **Code Reusability:** Functions for common tasks (e.g., SSHing to a machine, running `nixos-rebuild`) can be more cleanly defined and reused. -* **Easier Testing:** Guile's nature as a programming language makes it more amenable to unit testing individual functions or modules. -* **Future Extensibility:** Adding new commands, machines, or features will be simpler and less error-prone in a more structured language. - -## 3. Setting up Guile - -Guile is often available through system package managers. On NixOS, it can be added to your environment or system configuration. - -```nix -# Example: Adding Guile to a Nix shell -nix-shell -p guile -``` - -A Guile script typically starts with a shebang line: - -```scheme -#!/usr/bin/env guile -!# -``` - -The `!#` at the end is a Guile-specific convention that allows the script to be both executable and loadable into a Guile REPL. - -## 4. Basic Guile Scripting Concepts - -* **S-expressions:** Code is written using S-expressions (Symbolic Expressions), which are lists enclosed in parentheses, e.g., `(function arg1 arg2)`. -* **Definitions:** `(define variable value)` and `(define (function-name arg1 arg2) ...body...)`. -* **Procedures (Functions):** Core of Guile programming. -* **Control Flow:** `(if condition then-expr else-expr)`, `(cond (test1 expr1) (test2 expr2) ... (else else-expr))`, `(case ...)` -* **Modules:** `(use-modules (ice-9 popen))` for using libraries. - -## 5. Interacting with the System - -Guile provides modules for system interaction: - -* **(ice-9 popen):** For running external commands and capturing their output (similar to backticks or `$(...)` in Bash). - * `open-pipe* command mode`: Opens a pipe to a command. - * `get-string-all port`: Reads all output from a port. -* **(ice-9 rdelim):** For reading lines from ports. -* **(ice-9 filesys):** For file system operations (checking existence, deleting, etc.). - * `file-exists? path` - * `delete-file path` -* **(srfi srfi-1):** List processing utilities. -* **(srfi srfi-26):** `cut` for partial application, useful for creating specialized functions. -* **Environment Variables:** `(getenv "VAR_NAME")`, `(setenv "VAR_NAME" "value")`. - -## Example: Running a command** - -```scheme -(use-modules (ice-9 popen)) - -(define (run-command . args) - (let* ((cmd (string-join args " ")) - (port (open-pipe* cmd OPEN_READ))) - (let ((output (get-string-all port))) - (close-pipe port) - output))) - -(display (run-command "echo" "Hello from Guile")) -(newline) -``` - -## 6. Error Handling - -Guile uses a condition system for error handling. - -* `catch`: Allows you to catch specific types of errors. -* `throw`: Raises an error. - -```scheme -(use-modules (ice-9 exceptions)) - -(catch #t - (lambda () - (display "Trying something that might fail... -") - ;; Example: Force an error - (if #t (error "Something went wrong!")) - (display "This won't be printed if an error occurs above. -")) - (lambda (key . args) - (format (current-error-port) "Caught an error: ~a - Args: ~a -" key args) - #f)) ; Return value indicating an error was caught -``` - -For `home-lab-tools`, this means we can provide more specific feedback when a deployment fails or a machine is unreachable. - -## 7. Modularity and Code Organization - -Guile's module system allows splitting the code into logical units. For `home-lab-tools`, we could have modules for: - -* `lab-config`: Machine definitions, paths. -* `lab-deploy`: Functions related to deploying configurations. -* `lab-ssh`: SSH interaction utilities. -* `lab-status`: Functions for checking machine status. -* `lab-utils`: General helper functions, logging. - -**Example module structure:** - -```scheme -;; file: lab-utils.scm -(define-module (lab utils) - #:export (log success warn error)) - -(define blue "") -(define nc "") - -(define (log msg) - (format #t "~a[lab]~a ~a -" blue nc msg)) -;; ... other logging functions -``` - -```scheme -;; file: main-lab-script.scm -#!/usr/bin/env guile -!# -(use-modules (lab utils) (ice-9 popen)) - -(log "Starting lab script...") -;; ... rest of the script -``` - -## 8. Example: Rewriting a Small Part of `home-lab-tools.nix` (Conceptual) - -Let's consider the `log` function and a simplified `deploy_machine` for local deployment. - -**Current Bash:** - -```bash -BLUE='' -NC='' # No Color - -log() { - echo -e "''${BLUE}[lab]''${NC} $1" -} - -deploy_machine() { - local machine="$1" - # ... - if [[ "$machine" == "congenital-optimist" ]]; then - log "Deploying $machine (mode: $mode) locally" - sudo nixos-rebuild $mode --flake "$HOMELAB_ROOT#$machine" - fi - # ... -} -``` - -**Conceptual Guile Scheme:** - -```scheme -;; main-lab-script.scm -#!/usr/bin/env guile -!# - -(use-modules (ice-9 popen) - (ice-9 rdelim) - (ice-9 pretty-print) - (ice-9 exceptions) - (srfi srfi-1)) ;; For list utilities like `string-join` - -;; Configuration (could be in a separate module) -(define homelab-root "/home/geir/Home-lab") - -;; Color Definitions -(define RED "") -(define GREEN "") -(define YELLOW "") -(define BLUE "") -(define NC "") - -;; Logging functions -(define (log level-color level-name message) - (format #t "~a[~a]~a ~a -" level-color level-name NC message)) - -(define (info . messages) - (log BLUE "lab" (apply string-append (map (lambda (m) (if (string? m) m (format #f "~s" m))) messages)))) - -(define (success . messages) - (log GREEN "lab" (apply string-append (map (lambda (m) (if (string? m) m (format #f "~s" m))) messages)))) - -(define (warn . messages) - (log YELLOW "lab" (apply string-append (map (lambda (m) (if (string? m) m (format #f "~s" m))) messages)))) - -(define (err . messages) - (log RED "lab" (apply string-append (map (lambda (m) (if (string? m) m (format #f "~s" m))) messages))) - (exit 1)) ;; Exit on error - -;; Function to run shell commands and handle output/errors -(define (run-shell-command . command-parts) - (let ((command-string (string-join command-parts " "))) - (info "Executing: " command-string) - (let ((pipe (open-pipe* command-string OPEN_READ))) - (let loop ((lines '())) - (let ((line (read-line pipe))) - (if (eof-object? line) - (begin - (close-pipe pipe) - (reverse lines)) ;; Return lines in order - (begin - (display line) (newline) ;; Display live output - (loop (cons line lines))))))) - ;; TODO: Add proper error checking based on exit status of the command - ;; For now, we assume success if open-pipe* doesn't fail. - ;; A more robust solution would check `close-pipe` status or use `system*`. - )) - -;; Simplified deploy_machine -(define (deploy-machine machine mode) - (info "Deploying " machine " (mode: " mode ")") - (cond - ((string=? machine "congenital-optimist") - (info "Deploying " machine " locally") - (catch #t - (lambda () - (run-shell-command "sudo" "nixos-rebuild" mode "--flake" (string-append homelab-root "#" machine)) - (success "Successfully deployed " machine)) - (lambda (key . args) - (err "Failed to deploy " machine ". Error: " key " Args: " args)))) - ;; Add other machines here - (else - (err "Unknown machine: " machine)))) - -;; Main script logic (parsing arguments, calling functions) -(define (main args) - (if (< (length args) 3) - (begin - (err "Usage: