Use Niv to pin nixpkgs releases

Before using Niv, we were following channels meaning we can not
reproduce CI jobs easily.

In this change, we use Niv to pin these dependencies. We are also
addding a tests/default.nix to be able to run these tests locally.

For instance, to run the test extern.nix on the nixpkgs-19.09 release:

    nix-build tests/default.nix -A extern.nixpkgs_19_09

Fixes #178

.gitlab-ci.yml

@@ -7,59 +7,40 @@
     - ls -l /dev/kvm || true
     - touch /dev/kvm
   script:
-    - export NIX_PATH="nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-${VERSION}.tar.gz"
+    - nix-build tests/default.nix -A ${SUITE}.${VERSION} --arg nonKVM true
-    - nix-build --arg pkgs 'import tests/lib/pkgs.nokvm.nix' tests/${SUITE}.nix
-
-19-03-intern:
-  extends: .template
-  variables:
-    VERSION: "19.03"
-    SUITE:   "intern"
-
-19-03-extern:
-  extends: .template
-  variables:
-    VERSION: "19.03"
-    SUITE:   "extern"
-
-19-03-clamav:
-  extends: .template
-  variables:
-    VERSION: "19.03"
-    SUITE:   "clamav"
 
 19-09-intern:
   extends: .template
   variables:
-    VERSION: "19.09"
+    VERSION: "nixpkgs_19_09"
     SUITE:   "intern"
 
 19-09-extern:
   extends: .template
   variables:
-    VERSION: "19.09"
+    VERSION: "nixpkgs_19_09"
     SUITE:   "extern"
 
 19-09-clamav:
   extends: .template
   variables:
-    VERSION: "19.09"
+    VERSION: "nixpkgs_19_09"
     SUITE:   "clamav"
 
 unstable-intern:
   extends: .template
   variables:
-    VERSION: "unstable"
+    VERSION: "nixpkgs_unstable"
     SUITE:   "intern"
 
-unstable-extern:
+nixpkgs_unstable-extern:
   extends: .template
   variables:
-    VERSION: "unstable"
+    VERSION: "nixpkgs_unstable"
     SUITE:   "extern"
 
-unstable-clamav:
+nixpkgs_unstable-clamav:
   extends: .template
   variables:
-    VERSION: "unstable"
+    VERSION: "nixpkgs_unstable"
     SUITE:   "clamav"

nix/sources.json

@@ -0,0 +1,38 @@
+{
+    "nixpkgs-19.09": {
+        "branch": "nixos-19.09",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "60c4ddb97fd5a730b93d759754c495e1fe8a3544",
+        "sha256": "1a1pvfz130c4cma5a21wjl7yrivc7ls1ksqqmac23srk64ipzakf",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/60c4ddb97fd5a730b93d759754c495e1fe8a3544.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-20.03": {
+        "branch": "nixos-20.03",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "29eddfc36d720dcc4822581175217543b387b1e8",
+        "sha256": "1gqv2m7plkladd3va664xyqb962pqs4pizzibvkm1nh0f4rfpxvy",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/29eddfc36d720dcc4822581175217543b387b1e8.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-unstable": {
+        "branch": "nixos-unstable",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "9b0d2f3fd153167b0c8ce84bb71e766a39ed4c9d",
+        "sha256": "0yyr83zdaa04v8dsla28pk0nqnihxzkgfr8j0m3prgj2w4gd9ajz",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/9b0d2f3fd153167b0c8ce84bb71e766a39ed4c9d.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,134 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; };
+
+  fetch_tarball = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchTarball { inherit (spec) url sha256; }
+    else
+      pkgs.fetchzip { inherit (spec) url sha256; };
+
+  fetch_git = spec:
+    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+
+  fetch_builtin-tarball = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-tarball" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=tarball -a builtin=true
+      ''
+      builtins_fetchTarball { inherit (spec) url sha256; };
+
+  fetch_builtin-url = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-url" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=file -a builtin=true
+      ''
+      (builtins_fetchurl { inherit (spec) url sha256; });
+
+  #
+  # Various helpers
+  #
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
+      if builtins.hasAttr "nixpkgs" sources
+      then sourcesNixpkgs
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
+          ''
+            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+            add a package called "nixpkgs" to your sources.json.
+          '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs spec
+    else if spec.type == "tarball" then fetch_tarball pkgs spec
+    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
+    else if spec.type == "builtin-url" then fetch_builtin-url spec
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball { inherit url; }
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl { inherit url; }
+      else
+        fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (
+      name: spec:
+        if builtins.hasAttr "outPath" spec
+        then abort
+          "The values in sources.json should not have an 'outPath' attribute"
+        else
+          spec // { outPath = fetch config.pkgs name spec; }
+    ) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig =
+    { sourcesFile ? ./sources.json
+    , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
+    , pkgs ? mkPkgs sources
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

tests/clamav.nix

@@ -14,7 +14,9 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-import <nixpkgs/nixos/tests/make-test.nix> {
+{ pkgs ? import <nixpkgs> {}}:
+
+import (pkgs.path + "/nixos/tests/make-test.nix") {
 
   nodes = {
     server = { config, pkgs, lib, ... }:

tests/default.nix

@@ -0,0 +1,57 @@
+# Generate an attribute sets containing all tests for all releaeses
+# It looks like:
+# - extern.nixpkgs_19_09
+# - extern.nixpkgs_20.03
+# - extern.nixpkgs_unstable
+# - intern.nixpkgs_19_09
+# - intern.nixpkgs_20.03
+# - intern.nixpkgs_unstable
+
+# Modify pkgs to run the tests on non KVM machines
+{ noKVM ? false }: 
+
+with builtins;
+
+let
+  sources = import ../nix/sources.nix;
+
+  releases = listToAttrs (map genRelease releaseNames);
+
+  genRelease = name: {
+    name = name;
+    value = import sources."${name}" {};
+  };
+
+  genTest = testName: release:
+  let
+    pkgs = releases."${release}";
+    noKVMPkgs = p: if noKVM
+      then import ./lib/pkgs.nokvm.nix { pkgs = p; }
+      else p;
+    test = pkgs.callPackage (./. + "/${testName}.nix") { };
+  in {
+    "name"= builtins.replaceStrings ["." "-"] ["_" "_"] release;
+    "value"= test {
+      pkgs = (noKVMPkgs pkgs);
+    };
+  };
+
+  releaseNames = [
+    "nixpkgs-19.09"
+    "nixpkgs-20.03"
+    "nixpkgs-unstable"
+  ];
+
+  testNames = [
+    "intern"
+    "extern"
+    "clamav"
+  ];
+
+  # Generate an attribute set containing one test per releases
+  genTests = testName: {
+    name = testName;
+    value = listToAttrs (map (genTest testName) (builtins.attrNames releases));
+  };
+  
+in listToAttrs (map genTests testNames)

tests/extern.nix

@@ -14,7 +14,9 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-import <nixpkgs/nixos/tests/make-test.nix> {
+{ pkgs ? import <nixpkgs> {}}:
+
+import (pkgs.path + "/nixos/tests/make-test.nix") {
 
   nodes = {
     server = { config, pkgs, ... }:

tests/intern.nix

@@ -14,7 +14,9 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-import <nixpkgs/nixos/tests/make-test.nix> {
+{ pkgs ? import <nixpkgs> {}}:
+
+import (pkgs.path + "/nixos/tests/make-test.nix") {
 
   machine =
     { config, pkgs, ... }:

tests/lib/pkgs.nokvm.nix

@@ -1,5 +1,6 @@
+{ pkgs }:
+
 let
-    pkgs = (import <nixpkgs> { system = builtins.currentSystem; config = {}; });
     patchedMachinePM = pkgs.writeTextFile {
         name = "Machine.pm.patched-to-wait-longer-for-vm";
         text = builtins.replaceStrings ["alarm 600;"] ["alarm 1200;"] (builtins.readFile (<nixpkgs>+"/nixos/lib/test-driver/Machine.pm"));