From b477d662de1bf91d4576689b38076bdd09e6b40e Mon Sep 17 00:00:00 2001 From: Kiskae Date: Fri, 29 Sep 2023 23:09:18 +0200 Subject: [PATCH] WIP updater --- dev/flake.lock | 67 +++- dev/flake.nix | 38 +- dev/treefmt.nix | 35 +- dev/updater/module.nix | 103 ++++++ .../nvchecker_source/tesla_releases.py | 27 ++ dev/updater/nvchecker_source/unix_drivers.py | 61 ++++ flake.nix | 13 +- nixos/default.nix | 3 + nixos/nvidia/legacy.nix | 1 + pkgs/nvidia-driver/components.nix | 84 +++++ pkgs/nvidia-driver/default.nix | 30 ++ .../nvidia-installer/default.nix | 33 ++ .../nvidia-installer/parse-manifest.py | 0 pkgs/overlay.nix | 5 + {pkgs => pkgs_old}/kmod.nix | 0 pkgs_old/nvidia-driver/components.nix | 1 + .../nvidia-driver}/default.nix | 0 .../nvidia-driver}/hashes.json | 0 .../nvidia-installer/components.nix | 0 .../nvidia-installer/default.nix | 2 +- .../nvidia-installer/parse-manifest.py | 329 ++++++++++++++++++ .../nvidia-driver}/refs.json | 0 .../nvidia-driver}/refs/get-hash.sh | 0 .../nvidia-driver}/refs/tesla.jq | 0 .../nvidia-driver}/refs/tesla.json | 0 .../nvidia-driver}/refs/versions.jq | 0 .../nvidia-driver}/refs/versions.json | 0 .../nvidia-driver}/refs/vulkan-dev.jq | 0 .../nvidia-driver}/refs/vulkan-dev.json | 0 {pkgs => pkgs_old}/nvidia-settings.nix | 0 {pkgs => pkgs_old}/vulkan-samples.nix | 0 31 files changed, 799 insertions(+), 33 deletions(-) create mode 100644 dev/updater/module.nix create mode 100644 dev/updater/nvchecker_source/tesla_releases.py create mode 100644 dev/updater/nvchecker_source/unix_drivers.py create mode 100644 nixos/default.nix create mode 100644 nixos/nvidia/legacy.nix create mode 100644 pkgs/nvidia-driver/components.nix create mode 100644 pkgs/nvidia-driver/default.nix create mode 100644 pkgs/nvidia-driver/nvidia-installer/default.nix rename {nvidia-driver => pkgs/nvidia-driver}/nvidia-installer/parse-manifest.py (100%) create mode 100644 pkgs/overlay.nix rename {pkgs => pkgs_old}/kmod.nix (100%) create mode 100644 pkgs_old/nvidia-driver/components.nix rename {nvidia-driver => pkgs_old/nvidia-driver}/default.nix (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/hashes.json (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/nvidia-installer/components.nix (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/nvidia-installer/default.nix (99%) create mode 100755 pkgs_old/nvidia-driver/nvidia-installer/parse-manifest.py rename {nvidia-driver => pkgs_old/nvidia-driver}/refs.json (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/get-hash.sh (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/tesla.jq (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/tesla.json (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/versions.jq (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/versions.json (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/vulkan-dev.jq (100%) rename {nvidia-driver => pkgs_old/nvidia-driver}/refs/vulkan-dev.json (100%) rename {pkgs => pkgs_old}/nvidia-settings.nix (100%) rename {pkgs => pkgs_old}/vulkan-samples.nix (100%) diff --git a/dev/flake.lock b/dev/flake.lock index fa1a83e..8e3720b 100644 --- a/dev/flake.lock +++ b/dev/flake.lock @@ -3,7 +3,7 @@ "flake": { "locked": { "lastModified": 0, - "narHash": "sha256-ZCBBS14pkSYLUbDtVEN6BRCEMm2e9ahpQHOG7toP9Rw=", + "narHash": "sha256-jyyAhyStjFcaG9VOipFdN0Souk/fzgTD3uQjT/I2z3w=", "path": "../", "type": "path" }, @@ -12,13 +12,46 @@ "type": "path" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1693611461, + "narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-root": { + "locked": { + "lastModified": 1692742795, + "narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=", + "owner": "srid", + "repo": "flake-root", + "rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1694068030, - "narHash": "sha256-q21JdfZjK4XN5QwWTzCHF/G6uuZtwASNW9/ZBaak65M=", + "lastModified": 1695962284, + "narHash": "sha256-vygWW+zz/5NHWpH7nIfNFlKiJChp6kVRXkzAcPK08hY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9e26139b45147aadd25ab7ab3bc4a93d6d5e94e7", + "rev": "cdd726e1deb44c031ee8975528d6b283ed8cf021", "type": "github" }, "original": { @@ -28,9 +61,29 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1693471703, + "narHash": "sha256-0l03ZBL8P1P6z8MaSDS/MvuU8E75rVxe5eE1N6gxeTo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3e52e76b70d5508f3cec70b882a29199f4d1ee85", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake": "flake", + "flake-parts": "flake-parts", + "flake-root": "flake-root", "nixpkgs": "nixpkgs", "systems": "systems", "treefmt-nix": "treefmt-nix" @@ -57,11 +110,11 @@ ] }, "locked": { - "lastModified": 1693817438, - "narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=", + "lastModified": 1695822946, + "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "b8d3a059f5487d6767d07c3716386753e3132d9f", + "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a", "type": "github" }, "original": { diff --git a/dev/flake.nix b/dev/flake.nix index 00c9fb7..db55fe5 100644 --- a/dev/flake.nix +++ b/dev/flake.nix @@ -8,28 +8,32 @@ treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; + + flake-parts.url = "github:hercules-ci/flake-parts"; + flake-root.url = "github:srid/flake-root"; }; - outputs = { - self, - nixpkgs, + outputs = inputs @ { + flake-parts, systems, - treefmt-nix, ... - }: let - # Small tool to iterate over each systems - eachSystem = f: nixpkgs.lib.genAttrs (import systems) (system: f nixpkgs.legacyPackages.${system}); - - # Eval the treefmt modules from ./treefmt.nix - treefmtEval = eachSystem (pkgs: treefmt-nix.lib.evalModule pkgs ./treefmt.nix); - in { - # for `nix fmt` - formatter = eachSystem (pkgs: treefmtEval.${pkgs.system}.config.build.wrapper); - # for `nix flake check` - checks = eachSystem (pkgs: { - formatting = treefmtEval.${pkgs.sytem}.config.build.check self; + }: + flake-parts.lib.mkFlake {inherit inputs;} ({self, ...}: { + flake = { + inherit (inputs.flake) nixosModules overlays; + }; + debug = true; + systems = import systems; + imports = [ + ./treefmt.nix + ./updater/module.nix + ]; + perSystem = {pkgs, ...}: { + packages = let + nvPkgs = (pkgs.extend self.overlays.default).nvidiaPackages; + in {driver = nvPkgs.driver.test;}; + }; }); - }; /* diff --git a/dev/treefmt.nix b/dev/treefmt.nix index 912e794..a3c5f4b 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -1,9 +1,30 @@ -{pkgs, ...}: { - projectRootFile = ".git/config"; - programs.alejandra.enable = true; - programs.black.enable = true; - programs.isort = { - enable = true; - profile = "black"; +{ + lib, + inputs, + ... +}: { + imports = [ + inputs.treefmt-nix.flakeModule + ]; + + perSystem = { + pkgs, + config, + ... + }: { + treefmt = { + inherit (config.flake-root) projectRootFile; + + # Nix + programs.alejandra.enable = true; + programs.statix.enable = true; + + # Python + programs.black.enable = true; + programs.isort = { + enable = true; + profile = "black"; + }; + }; }; } diff --git a/dev/updater/module.nix b/dev/updater/module.nix new file mode 100644 index 0000000..b7f204d --- /dev/null +++ b/dev/updater/module.nix @@ -0,0 +1,103 @@ +{inputs, ...}: { + imports = [ + inputs.flake-root.flakeModule + ]; + + perSystem = { + pkgs, + config, + ... + }: { + flake-root.projectRootFile = "pkgs/overlay.nix"; + + devShells.update = let + inherit (pkgs) lib python3 formats; + + pytools = python3.withPackages (ps: [ + ps.nvchecker + ps.lxml + ]); + + extra_sources = lib.fileset.toSource { + root = ./.; + fileset = ./nvchecker_source; + }; + + configFormat = formats.toml {}; + configFile = configFormat.generate "nvchecker.toml" { + "__config__" = { + "oldver" = "\${FLAKE_ROOT}/pkgs/refs.json"; + #"newver" = "\${FLAKE_ROOT}/dev/current-versions.json"; + }; + "driver.vulkan-dev" = { + source = "htmlparser"; + prefix = "Linux "; + include_regex = "^Linux [.0-9]+$"; + url = "https://developer.nvidia.com/vulkan-driver"; + xpath = ''//*[@id="content"]/div/section/h3''; + }; + "driver.display" = { + source = "unix_drivers"; + known = lib.mapAttrsToList (match: data: + { + inherit match; + } + // data) { + "Production Branch Version" = { + maturity = "long-lived-branch-release"; + }; + "New Feature Branch Version" = {}; + "Beta Version" = { + maturity = "beta"; + }; + "Legacy GPU version (470.xx series)" = { + branch = "R470_00"; + }; + "Legacy GPU version (390.xx series)" = { + branch = "R390_00"; + }; + "Legacy GPU version (340.xx series)" = { + branch = "R340_00"; + }; + "Legacy GPU version (304.xx series)" = { + branch = "R304_00"; + }; + "Legacy GPU Version (71.86.xx series)" = { + branch = "L7160"; + }; + "Legacy GPU Version (96.43.xx series)" = { + branch = "L9622"; + }; + "Legacy GPU Version (173.14.xx series)" = { + branch = "R173_14"; + }; + }; + }; + "driver.tesla" = { + source = "tesla_releases"; + url = "https://docs.nvidia.com/datacenter/tesla/drivers/releases.json"; + from_pattern = ''(?:[^-]+-)*(.+)''; + to_pattern = ''\1''; + }; + }; + in + pkgs.mkShell { + packages = [pytools pkgs.jq]; + + inputsFrom = [config.flake-root.devShell]; + + env = { + PYTHONPATH = toString extra_sources; + NVCHECKER_CONFIG = toString configFile; + }; + + shellHook = '' + export SCRATCHDIR=$(mktemp -d) + + nvchecker -c ''${NVCHECKER_CONFIG} + + exit + ''; + }; + }; +} diff --git a/dev/updater/nvchecker_source/tesla_releases.py b/dev/updater/nvchecker_source/tesla_releases.py new file mode 100644 index 0000000..b69ee2e --- /dev/null +++ b/dev/updater/nvchecker_source/tesla_releases.py @@ -0,0 +1,27 @@ +import asyncio + +from nvchecker.api import AsyncCache, BaseWorker, RawResult + + +class Worker(BaseWorker): + async def run(self) -> None: + self.cache = AsyncCache() + await asyncio.gather( + *[self._run_entry(name, entry) for (name, entry) in self.tasks] + ) + + async def _run_entry(self, name, entry): + async with self.task_sem: + data = await self.cache.get_json(entry["url"]) + for branch, data in data.items(): + versions = [ + f"{r['release_date']}-{r['release_version']}" + for r in data["driver_info"] + ] + await self.result_q.put( + RawResult( + f"{name}.r{branch}", + versions, + entry, + ) + ) diff --git a/dev/updater/nvchecker_source/unix_drivers.py b/dev/updater/nvchecker_source/unix_drivers.py new file mode 100644 index 0000000..08fca28 --- /dev/null +++ b/dev/updater/nvchecker_source/unix_drivers.py @@ -0,0 +1,61 @@ +import re + +from nvchecker.api import BaseWorker, RawResult, session + +PATTERN = re.compile(rb"Latest (?P