Commit 56dff185 authored by Florian Franzen's avatar Florian Franzen
Browse files

flake: migrate to flake only

parent 601c267d
default:
image: nixos/nix:2.3
image: nixpkgs/cachix-flakes:nixos-20.09
before_script:
- nix-env --quiet -iA cachix -f https://cachix.org/api/v1/install
- cachix use w3fpkgs
stages:
- dependencies
- implementations
- specifications
- fetch
- build
dependencies:
stage: dependencies
script: nix-build release.nix -A dependencies | cachix push w3fpkgs
timeout: 1h
.nix-build:
script: nix build .#${CI_JOB_NAME} && nix path-info .#{CI_JOB_NAME} | cachix push w3fpkgs
timeout: 15m
gossamer:
stage: implementations
script: nix-build release.nix -A gossamer | cachix push w3fpkgs
polkadot-spec:
extends: .nix-build
stage: fetch
.build:
extends: .nix-build
stage: build
substrate-host:
extends: .build
timeout: 6h
kagome-host:
extends: .build
timeout: 2h
kagome:
stage: implementations
script: nix-build release.nix -A kagome | cachix push w3fpkgs
gossamer-host:
extends: .build
timeout: 2h
parity-polkadot:
stage: implementations
script: nix-build release.nix -A parity-polkadot | cachix push w3fpkgs
timeout: 6h
substrate-adapter:
extends: .build
timeout: 2h
kagome-adapter:
extends: .build
needs: ["kagome-host"]
gossamer-adapter:
extends: .build
hostapi-runtime:
extends: .build
timeout: 2h
test-runtime:
extends: .build
timeout: 2h
polkadot-host-spec:
extends: .build
polkadot-runtime-spec:
extends: .build
specifications:
stage: specifications
script: nix-build release.nix -A specifications | cachix push w3fpkgs
polkadot-testsuite:
extends: .build
......@@ -8,19 +8,6 @@ __WORK IN PROGRESS__
This is a collection of useful derivations, environments and functions used to utilize the power of nix in our daily work at the Web3 Foundation.
Derivations can be build using ```default.nix```, which just overlays your current nixpkgs with our overlay and returns the resulting attribute set. That means you can just build any derivation from the overlay by selecting them from the resulting attribute set using nix-build:
Currently one contains the pinned version of each implementation as used by the spec teams conformance and integration test.
```
nix-build /path/to/w3fpkgs -A parity-polkadot
```
The local result folder then links to the resulting derivation in the nix store.
Alternativly, to link the result into your environment you can run:
```
nix-env -f /path/to/w3fpkgs -iA parity-polkadot
```
Alternativly add ```overlay.nix``` to your list of nixpkgs overlays and the packages as part of nixpkg as usual.
There is also a ```release.nix``` mostly used by our CI, which applies the overlay to a pinned version of nixpkgs to improve caching.
To use this flake as an overlay in your local nixpkgs config, you can use `(builtins.getFlake "<path-to-flake>").overlay`.
{ src, version, buildGoModule, patchelf, glibc }:
{ polkadot-spec, buildGoModule, patchelf, glibc }:
buildGoModule rec {
inherit src version;
pname = "gossamer-adapter";
version = polkadot-spec.shortRev;
src = polkadot-spec;
modRoot = "test/adapters/gossamer";
runVend = true;
vendorSha256 = "0sj807x65v16j4cs9mmwgg5psb1q6cjrmw9fqy5a102q8n110hiz";
vendorSha256 = "0g1qjsc0civrjqggp2ya747n6rwriflc12m98v7amsv597v8x1x6";
nativeBuildInputs = [ patchelf ];
......
{ src, version, lib, stdenv, cmake, kagome, libyamlcpp }:
{ polkadot-spec, stdenv, cmake, kagome-host, libyamlcpp }:
stdenv.mkDerivation {
inherit src version;
pname = "kagome-adapter";
version = polkadot-spec.shortRev;
src = polkadot-spec;
sourceRoot = "source/test/adapters/kagome";
nativeBuildInputs = [ cmake ];
buildInputs = [ kagome libyamlcpp ];
buildInputs = [ kagome-host libyamlcpp ];
cmakeFlags = [ "-DHUNTER_ENABLED=FALSE" ];
}
{ src, version, lib, rustWasmPlatform }:
{ polkadot-spec, lib, rustWasmPlatform }:
rustWasmPlatform.buildRustPackage {
inherit src version;
pname = "substrate-adapter";
version = polkadot-spec.shortRev;
src = polkadot-spec;
sourceRoot = "source/test/adapters/substrate";
cargoSha256 = "148jxglx0sn9v1fs4z77h8js86wl9xn8irz79dxg8acianqm23j0";
cargoSha256 = "07idsdjn2hn7hj0srzkbg6nbqh4b10pk4hkwb0ilh8swvz0li92p";
}
{ pkgs ? import <nixpkgs> {} }:
let
self = pkgs.extend (import ./overlay.nix);
in
self
{
"nodes": {
"flake-utils": {
"locked": {
"lastModified": 1605370193,
"narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5021eac20303a61fafe17224c087f5519baed54d",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gossamer": {
"flake": false,
"locked": {
"lastModified": 1606754926,
"narHash": "sha256-i3UE9/y2cGMorZP4f2sIbqD6pBz+lP3/YgIgxaKYjMc=",
"owner": "ChainSafe",
"repo": "gossamer",
"rev": "580a26b9ef56afabfcbc44220e96a930fa8d95e9",
"type": "github"
},
"original": {
"owner": "ChainSafe",
"repo": "gossamer",
"type": "github"
}
},
"kagome": {
"flake": false,
"locked": {
"lastModified": 1608564467,
"narHash": "sha256-As7grbpz/lQjZj4ezlor5K7PIcHkvolgPwvm+qzNeKQ=",
"owner": "soramitsu",
"repo": "kagome",
"rev": "afca0f68ce8c3476957d37b9ecdca1120f5da2fc",
"type": "github"
},
"original": {
"owner": "soramitsu",
"repo": "kagome",
"type": "github"
}
},
"mozilla": {
"flake": false,
"locked": {
......@@ -82,11 +35,11 @@
"polkadot-spec": {
"flake": false,
"locked": {
"lastModified": 1606314645,
"narHash": "sha256-2zfHJQ0hvwcatH54Gq4HqjFZ3ZT1cq9xtQ27AZtO75Q=",
"lastModified": 1615374962,
"narHash": "sha256-2+cceP8V6r7O5qUUHa2ZuyCimingyigZuAVVJsJf9rE=",
"owner": "w3f",
"repo": "polkadot-spec",
"rev": "7c14873c1ff7af3d09713595e821e5af928a2f6f",
"rev": "34dc9fc535bd1778043e1786ddcc4628d92b1299",
"type": "github"
},
"original": {
......@@ -97,12 +50,25 @@
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"gossamer": "gossamer",
"kagome": "kagome",
"mozilla": "mozilla",
"nixpkgs": "nixpkgs",
"polkadot-spec": "polkadot-spec"
"polkadot-spec": "polkadot-spec",
"utils": "utils"
}
},
"utils": {
"locked": {
"lastModified": 1614513358,
"narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5466c5bbece17adaab2d82fae80b46e807611bf3",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
......
......@@ -3,32 +3,34 @@
inputs = {
# Nix base libraries
flake-utils.url = "github:numtide/flake-utils";
utils.url = "github:numtide/flake-utils";
# Basis for included packages
nixpkgs.url = "github:nixos/nixpkgs/nixos-20.09";
# Overlay used for nightly rust support
mozilla = { url = "github:mozilla/nixpkgs-mozilla"; flake = false; };
# Polkadot specification and testsuite repo
polkadot-spec = { url = "github:w3f/polkadot-spec"; flake = false; };
# Implementation sources
kagome = { url = "github:soramitsu/kagome"; flake = false; };
gossamer = { url = "github:ChainSafe/gossamer"; flake = false; };
};
outputs = { self, flake-utils, nixpkgs, ... } @ sources :
flake-utils.lib.eachDefaultSystem (system:
outputs = { self, utils, nixpkgs, ... } @ sources :
utils.lib.eachDefaultSystem (system:
let
rust-overlay = import "${sources.mozilla}/rust-overlay.nix";
pkgs = import nixpkgs {
inherit system;
overlays = [ self.overlay ];
overlays = [
self.overlay
rust-overlay
];
};
lib = pkgs.lib // flake-utils.lib;
in {
lib = pkgs.lib // utils.lib;
in {
apps = {
testsuite = lib.mkApp {
......@@ -36,15 +38,19 @@
};
};
packages = let
filterRecurse = pkgs.lib.filterAttrs (n: _: n != "recurseForDerivations");
in
filterRecurse (pkgs.implementations // pkgs.specifications);
packages = pkgs.implementations // pkgs.specifications;
}
) // {
checks.x86_64-linux = self.packages.x86_64-linux;
overlay = import ./overlay.nix;
overlay = let
# Current stable nixUnstable does not support submodules yet
polkadot-spec = builtins.fetchGit {
url = "git://github.com/w3f/polkadot-spec";
inherit (sources.polkadot-spec) rev;
submodules = true;
};
in import ./overlay.nix { inherit polkadot-spec; };
};
}
diff --git a/chain/gssmr/config.toml b/chain/gssmr/config.toml
index 811b4b4..39276ac 100644
index 67be0403..a87551c9 100644
--- a/chain/gssmr/config.toml
+++ b/chain/gssmr/config.toml
@@ -13,9 +13,6 @@ runtime = ""
babe = ""
@@ -12,9 +12,6 @@ babe = ""
grandpa = ""
sync = ""
-[init]
-genesis-raw = "./chain/gssmr/genesis-raw.json"
......@@ -13,7 +13,7 @@ index 811b4b4..39276ac 100644
key = ""
unlock = ""
diff --git a/chain/gssmr/defaults.go b/chain/gssmr/defaults.go
index c0651c7..4899e99 100644
index d09d708d..9c21cf34 100644
--- a/chain/gssmr/defaults.go
+++ b/chain/gssmr/defaults.go
@@ -29,7 +29,7 @@ var (
......@@ -35,12 +35,12 @@ index c0651c7..4899e99 100644
// AccountConfig
diff --git a/chain/ksmcc/config.toml b/chain/ksmcc/config.toml
index e7d09c2..ccc01a4 100644
index a8ffb0b1..bb8bebcb 100644
--- a/chain/ksmcc/config.toml
+++ b/chain/ksmcc/config.toml
@@ -13,9 +13,6 @@ runtime = ""
babe = ""
@@ -12,9 +12,6 @@ babe = ""
grandpa = ""
sync = ""
-[init]
-genesis-raw = "./chain/ksmcc/genesis-raw.json"
......@@ -49,7 +49,7 @@ index e7d09c2..ccc01a4 100644
key = ""
unlock = ""
diff --git a/chain/ksmcc/defaults.go b/chain/ksmcc/defaults.go
index d37b772..bd64e46 100644
index 5cea4421..2007fe07 100644
--- a/chain/ksmcc/defaults.go
+++ b/chain/ksmcc/defaults.go
@@ -29,7 +29,7 @@ var (
......@@ -70,3 +70,39 @@ index d37b772..bd64e46 100644
// AccountConfig
diff --git a/chain/polkadot/config.toml b/chain/polkadot/config.toml
index 44a1b090..b22c6878 100644
--- a/chain/polkadot/config.toml
+++ b/chain/polkadot/config.toml
@@ -12,9 +12,6 @@ babe = ""
grandpa = ""
sync = ""
-[init]
-genesis-raw = "./chain/polkadot/genesis-raw.json"
-
[account]
key = ""
unlock = ""
diff --git a/chain/polkadot/defaults.go b/chain/polkadot/defaults.go
index 258976cf..c7c0ffd8 100644
--- a/chain/polkadot/defaults.go
+++ b/chain/polkadot/defaults.go
@@ -29,7 +29,7 @@ var (
// DefaultID Default chain ID
DefaultID = string("polkadot")
// DefaultConfig Default toml configuration path
- DefaultConfig = string("./chain/polkadot/config.toml")
+ DefaultConfig = string("@out@/share/polkadot/config.toml")
// DefaultBasePath Default node base directory path
DefaultBasePath = string("~/.gossamer/polkadot")
@@ -39,7 +39,7 @@ var (
// InitConfig
// DefaultGenesisRaw Default genesis configuration path
- DefaultGenesisRaw = string("./chain/polkadot/genesis-raw.json")
+ DefaultGenesisRaw = string("@out@/share/polkadot/genesis-raw.json")
// AccountConfig
{ lib, buildGoModule, fetchFromGitHub, patchelf, glibc }:
let
sources = import ../../sources.nix;
in
{ polkadot-spec, buildGoModule, patchelf, glibc }:
buildGoModule rec {
inherit (sources.gossamer) src version;
pname = "gossamer-host";
version = "spec-${polkadot-spec.shortRev}";
src = polkadot-spec;
pname = "gossamer";
sourceRoot = "source/test/hosts/gossamer";
patches = [ ./config_path.patch ./leb128_wagon.patch ];
patches = [ ./config_path.patch ];
postPatch = ''
substituteAllInPlace chain/gssmr/defaults.go
substituteAllInPlace chain/ksmcc/defaults.go
substituteAllInPlace chain/polkadot/defaults.go
'';
runVend = true;
vendorSha256 = "1j6yprc00qv1n577ni8j6zwap2abgz953x6sdyxj83wbsgdq7sgn";
vendorSha256 = "0qyni3gppfbl6j2vlvby2v55iz3wkvdx0sjqh0kznrn776f8byc9";
subPackages = [ "cmd/gossamer" ];
......
{ lib
{ polkadot-spec
, stdenv
, fetchFromGitHub
, cmake
, openssl
, protobuf
......@@ -20,16 +19,18 @@
, schnorrkel_crust
}:
let
sources = import ../../sources.nix;
in
stdenv.mkDerivation rec {
inherit (sources.kagome) src version;
pname = "kagome-host";
version = "spec-${polkadot-spec.shortRev}";
src = polkadot-spec;
pname = "kagome";
sourceRoot = "source/test/hosts/kagome";
patches = [ ./dependencies.patch ./protobuf_path.patch ];
patches = [
./dependencies.patch
./protobuf_path.patch
];
nativeBuildInputs = [
cmake
......
{ polkadot-spec
, stdenv
, rustPlatform
, clang
, llvmPackages
, protobuf
}:
rustPlatform.buildRustPackage {
pname = "substrate-host";
version = "spec-${polkadot-spec.shortRev}";
src = polkadot-spec;
sourceRoot = "source/test/hosts/substrate";
cargoSha256 = "17k2ak7ih2zq6cpgj188q2l3y60s2n9gpjzsjy2va07vnwim4war";
nativeBuildInputs = [ clang ];
LIBCLANG_PATH = "${llvmPackages.libclang}/lib";
PROTOC = "${protobuf}/bin/protoc";
# Currently not needed
SKIP_WASM_BUILD = 1;
# Needs WASM runtimes.
doCheck = false;
}
# Receive additional attrs to add to overlay
{ polkadot-spec, ... } @ external:
# Web3 Specification Team Nix Overlay
final: prev:
......@@ -30,56 +33,37 @@ let
spdlog_cmake = callPackage ./deps/spdlog.nix {};
xxHash_cmake = callPackage ./deps/xxhash.nix {};
libsecp256k1_cmake = callPackage ./deps/libsecp256k1.nix {};
# Support recursive build of whole attrs set
recurseForDerivations = true;
};
# Implementation derivations (buildable)
implementations = with final; {
# Go implementations
gossamer = callPackage ./pkgs/gossamer {};
gossamer-host = callPackage ./hosts/gossamer {};
# C++ implementations
kagome = callPackage ./pkgs/kagome {};
kagome-host = callPackage ./hosts/kagome {};
# Rust implementations
parity-polkadot = callPackage ./pkgs/parity-polkadot {};
# Support recursive build of whole attrs set
recurseForDerivations = true;
substrate-host = callPackage ./hosts/substrate {};
};
# Specification Testsuite binaries (buildable)
specifications = with final; let
sources = import ./sources.nix;
common = sources.polkadot-spec;
in {
specifications = with final; {
# Publications
polkadot-host-spec = callPackage ./spec/spec-host.nix common;
polkadot-runtime-spec = callPackage ./spec/spec-runtime.nix common;
polkadot-host-spec = callPackage ./spec/host.nix {};
polkadot-runtime-spec = callPackage ./spec/runtime.nix {};
# Adapters
substrate-adapter = callPackage ./spec/adapter-substrate.nix common;
kagome-adapter = callPackage ./spec/adapter-kagome.nix common;
gossamer-adapter = callPackage ./spec/adapter-gossamer.nix common;
substrate-adapter-legacy = callPackage ./spec/adapter-substrate-legacy.nix common;
kagome-adapter-legacy = callPackage ./spec/adapter-kagome-legacy.nix common;
gossamer-adapter-legacy = callPackage ./spec/adapter-gossamer-legacy.nix common;
substrate-adapter = callPackage ./adapters/substrate.nix {};
kagome-adapter = callPackage ./adapters/kagome.nix {};
gossamer-adapter = callPackage ./adapters/gossamer.nix {};
# Runtimes
hostapi-runtime = callPackage ./spec/runtime-hostapi.nix common;
tester-runtime = callPackage ./spec/runtime-tester.nix common;
hostapi-runtime-legacy = callPackage ./spec/runtime-hostapi-legacy.nix common;
tester-runtime-legacy = callPackage ./spec/runtime-tester-legacy.nix common;
hostapi-runtime = callPackage ./runtimes/hostapi.nix {};
tester-runtime = callPackage ./runtimes/tester.nix {};
# Testsuite incl. fixtures
polkadot-testsuite = callPackage ./spec/testsuite.nix common;
# Support recursive build of whole attrs set
recurseForDerivations = true;
polkadot-testsuite = callPackage ./testsuite {};
};
# Environment definitions used for development
......@@ -91,10 +75,10 @@ let
substrate = callPackage ./envs/substrate.nix {};
};
# Export some of the package sets as subset, used e.g. for caching and on ci.
# Export some of the package sets as subset.
virtual = {
inherit dependencies implementations specifications environments;
inherit implementations specifications environments;
};
in
support // dependencies // implementations // specifications // virtual
support // dependencies // implementations // specifications // virtual // external
commit 1229631fa46d64fcdc6709b10f5fa42c899fddae
Author: Florian Franzen <Florian.Franzen@gmail.com>
Date: Tue Dec 1 14:01:55 2020 +0100
leb128: implement and remove wagon
diff --git a/dot/network/utils.go b/dot/network/utils.go
index 9329a8c..e28f4b1 100644
--- a/dot/network/utils.go
+++ b/dot/network/utils.go
@@ -26,7 +26,6 @@ import (
"path"
"path/filepath"
- "github.com/go-interpreter/wagon/wasm/leb128"
"github.com/libp2p/go-libp2p-core/crypto"