From a53aa5ac9a080f292a45cea93e8249be2530d20b Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Sat, 11 Apr 2020 15:38:52 +0200 Subject: [PATCH] Use Niv to pin nixpkgs releases Before using Niv, we were following channels meaning we can not reproduce CI jobs easily. In this change, we use Niv to pin these dependencies. We are also addding a tests/default.nix to be able to run these tests locally. For instance, to run the test extern.nix on the nixpkgs-19.09 release: nix-build tests/default.nix -A extern.nixpkgs_19_09 Fixes #178 --- .gitlab-ci.yml | 37 +++-------- nix/sources.json | 38 +++++++++++ nix/sources.nix | 134 +++++++++++++++++++++++++++++++++++++++ tests/clamav.nix | 4 +- tests/default.nix | 57 +++++++++++++++++ tests/extern.nix | 4 +- tests/intern.nix | 4 +- tests/lib/pkgs.nokvm.nix | 3 +- 8 files changed, 249 insertions(+), 32 deletions(-) create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100644 tests/default.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c654eb7..773d1c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,59 +7,40 @@ - ls -l /dev/kvm || true - touch /dev/kvm script: - - export NIX_PATH="nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-${VERSION}.tar.gz" - - nix-build --arg pkgs 'import tests/lib/pkgs.nokvm.nix' tests/${SUITE}.nix - -19-03-intern: - extends: .template - variables: - VERSION: "19.03" - SUITE: "intern" - -19-03-extern: - extends: .template - variables: - VERSION: "19.03" - SUITE: "extern" - -19-03-clamav: - extends: .template - variables: - VERSION: "19.03" - SUITE: "clamav" + - nix-build tests/default.nix -A ${SUITE}.${VERSION} --arg nonKVM true 19-09-intern: extends: .template variables: - VERSION: "19.09" + VERSION: "nixpkgs_19_09" SUITE: "intern" 19-09-extern: extends: .template variables: - VERSION: "19.09" + VERSION: "nixpkgs_19_09" SUITE: "extern" 19-09-clamav: extends: .template variables: - VERSION: "19.09" + VERSION: "nixpkgs_19_09" SUITE: "clamav" unstable-intern: extends: .template variables: - VERSION: "unstable" + VERSION: "nixpkgs_unstable" SUITE: "intern" -unstable-extern: +nixpkgs_unstable-extern: extends: .template variables: - VERSION: "unstable" + VERSION: "nixpkgs_unstable" SUITE: "extern" -unstable-clamav: +nixpkgs_unstable-clamav: extends: .template variables: - VERSION: "unstable" + VERSION: "nixpkgs_unstable" SUITE: "clamav" diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 0000000..f8396b8 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,38 @@ +{ + "nixpkgs-19.09": { + "branch": "nixos-19.09", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs-channels", + "rev": "60c4ddb97fd5a730b93d759754c495e1fe8a3544", + "sha256": "1a1pvfz130c4cma5a21wjl7yrivc7ls1ksqqmac23srk64ipzakf", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/60c4ddb97fd5a730b93d759754c495e1fe8a3544.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-20.03": { + "branch": "nixos-20.03", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs-channels", + "rev": "29eddfc36d720dcc4822581175217543b387b1e8", + "sha256": "1gqv2m7plkladd3va664xyqb962pqs4pizzibvkm1nh0f4rfpxvy", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/29eddfc36d720dcc4822581175217543b387b1e8.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-unstable": { + "branch": "nixos-unstable", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs-channels", + "rev": "9b0d2f3fd153167b0c8ce84bb71e766a39ed4c9d", + "sha256": "0yyr83zdaa04v8dsla28pk0nqnihxzkgfr8j0m3prgj2w4gd9ajz", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/9b0d2f3fd153167b0c8ce84bb71e766a39ed4c9d.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 0000000..8a725cb --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,134 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: spec: + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; } + else + pkgs.fetchurl { inherit (spec) url sha256; }; + + fetch_tarball = pkgs: spec: + if spec.builtin or true then + builtins_fetchTarball { inherit (spec) url sha256; } + else + pkgs.fetchzip { inherit (spec) url sha256; }; + + fetch_git = spec: + builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; + + fetch_builtin-tarball = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-tarball" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=tarball -a builtin=true + '' + builtins_fetchTarball { inherit (spec) url sha256; }; + + fetch_builtin-url = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-url" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=file -a builtin=true + '' + (builtins_fetchurl { inherit (spec) url sha256; }); + + # + # Various helpers + # + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {}; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs spec + else if spec.type == "tarball" then fetch_tarball pkgs spec + else if spec.type == "git" then fetch_git spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec + else if spec.type == "builtin-url" then fetch_builtin-url spec + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball { inherit url; } + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl { inherit url; } + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = fetch config.pkgs name spec; } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? ./sources.json + , sources ? builtins.fromJSON (builtins.readFile sourcesFile) + , pkgs ? mkPkgs sources + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/tests/clamav.nix b/tests/clamav.nix index 955d1ac..62b8d7d 100644 --- a/tests/clamav.nix +++ b/tests/clamav.nix @@ -14,7 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -import { +{ pkgs ? import {}}: + +import (pkgs.path + "/nixos/tests/make-test.nix") { nodes = { server = { config, pkgs, lib, ... }: diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..7de029e --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,57 @@ +# Generate an attribute sets containing all tests for all releaeses +# It looks like: +# - extern.nixpkgs_19_09 +# - extern.nixpkgs_20.03 +# - extern.nixpkgs_unstable +# - intern.nixpkgs_19_09 +# - intern.nixpkgs_20.03 +# - intern.nixpkgs_unstable + +# Modify pkgs to run the tests on non KVM machines +{ noKVM ? false }: + +with builtins; + +let + sources = import ../nix/sources.nix; + + releases = listToAttrs (map genRelease releaseNames); + + genRelease = name: { + name = name; + value = import sources."${name}" {}; + }; + + genTest = testName: release: + let + pkgs = releases."${release}"; + noKVMPkgs = p: if noKVM + then import ./lib/pkgs.nokvm.nix { pkgs = p; } + else p; + test = pkgs.callPackage (./. + "/${testName}.nix") { }; + in { + "name"= builtins.replaceStrings ["." "-"] ["_" "_"] release; + "value"= test { + pkgs = (noKVMPkgs pkgs); + }; + }; + + releaseNames = [ + "nixpkgs-19.09" + "nixpkgs-20.03" + "nixpkgs-unstable" + ]; + + testNames = [ + "intern" + "extern" + "clamav" + ]; + + # Generate an attribute set containing one test per releases + genTests = testName: { + name = testName; + value = listToAttrs (map (genTest testName) (builtins.attrNames releases)); + }; + +in listToAttrs (map genTests testNames) diff --git a/tests/extern.nix b/tests/extern.nix index cde05d8..6a4f6e7 100644 --- a/tests/extern.nix +++ b/tests/extern.nix @@ -14,7 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -import { +{ pkgs ? import {}}: + +import (pkgs.path + "/nixos/tests/make-test.nix") { nodes = { server = { config, pkgs, ... }: diff --git a/tests/intern.nix b/tests/intern.nix index 5abf379..62e03a8 100644 --- a/tests/intern.nix +++ b/tests/intern.nix @@ -14,7 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -import { +{ pkgs ? import {}}: + +import (pkgs.path + "/nixos/tests/make-test.nix") { machine = { config, pkgs, ... }: diff --git a/tests/lib/pkgs.nokvm.nix b/tests/lib/pkgs.nokvm.nix index fa13fde..8832f08 100644 --- a/tests/lib/pkgs.nokvm.nix +++ b/tests/lib/pkgs.nokvm.nix @@ -1,5 +1,6 @@ +{ pkgs }: + let - pkgs = (import { system = builtins.currentSystem; config = {}; }); patchedMachinePM = pkgs.writeTextFile { name = "Machine.pm.patched-to-wait-longer-for-vm"; text = builtins.replaceStrings ["alarm 600;"] ["alarm 1200;"] (builtins.readFile (+"/nixos/lib/test-driver/Machine.pm"));