Add nightly CI check against latest substrate. (paritytech#335)

* Add nightly check and use specific version (currently the same) in PR checks

* wrong var name, whoops

* Create GH issue if nightly run fails

* need tokio feature for RpcClient; match dependency in root Cargo.toml.

* Ensure substrate command is properly cleaned up in the event of a panic, to prevent blocking

* increase retry limit again

* fmt

* Tweak text for nightly-run-failed issue template

* Run nightly-CI every 5mins for testing

* I clearly dont use cron enough

* Put back cron timings to 8am daily

* break a test

* revert nightly workflow again (doesnt run off master) and testing rust workflow

* more testing..

* undo testing things; seems to work

* tidy up; substrate url at top of both CI files

.github/issue_templates/nightly_run_failed.md

@@ -0,0 +1,5 @@
+---
+title: Subxt integration tests failed against latest Substrate build.
+---
+
+The nightly CI run which downloads the latest version of Substrate ran into test failures, which likely means that there are breaking changes that need fixing in Subxt.

.github/workflows/nightly.yml

@@ -0,0 +1,58 @@
+name: Daily compatibility check against latest substrate
+
+on:
+  schedule:
+    # Run at 8am every day
+    - cron: "* 8 * * *"
+
+
+env:
+  CARGO_TERM_COLOR: always
+  # Use latest substrate for nightly runs:
+  SUBSTRATE_URL: https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate
+
+jobs:
+  tests:
+    name: Cargo test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Download Substrate
+        run: |
+            curl $SUBSTRATE_URL --output substrate --location
+            chmod +x substrate
+            mkdir -p ~/.local/bin
+            mv substrate ~/.local/bin
+
+      - name: Install Rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v1.3.0
+
+      - name: Cargo test
+        uses: actions-rs/cargo@v1.0.3
+        with:
+          command: test
+          args: --all-targets --workspace
+
+      # If the previous step fails, create a new Github issue
+      # to nofity us about it.
+      - if: ${{ failure() }}
+        uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Use this issue template:
+          filename: .github/issue_templates/nightly_run_failed.md
+          # Don't create a new issue; skip updating existing:
+          update_existing: false
+          # Look for new *open* issues in this search (we want to
+          # create a new one if we only find closed versions):
+          search_existing: open

.github/workflows/rust.yml

@@ -14,6 +14,8 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  # TODO: Currently pointing at latest substrate; is there a suitable binary we can pin to here?
+  SUBSTRATE_URL: https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate
 
 jobs:
   build:
@@ -25,7 +27,7 @@ jobs:
 
       - name: Download Substrate
         run: |
-            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            curl $SUBSTRATE_URL --output substrate --location
             chmod +x substrate
             mkdir -p ~/.local/bin
             mv substrate ~/.local/bin
@@ -79,7 +81,7 @@ jobs:
 
       - name: Download Substrate
         run: |
-            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            curl $SUBSTRATE_URL --output substrate --location
             chmod +x substrate
             mkdir -p ~/.local/bin
             mv substrate ~/.local/bin
@@ -95,7 +97,7 @@ jobs:
         uses: Swatinem/rust-cache@v1.3.0
 
       - name: Check internal documentation links
-        run: RUSTDOCFLAGS="--deny broken_intra_doc_links" cargo doc --verbose --workspace --no-deps --document-private-items
+        run: RUSTDOCFLAGS="--deny rustdoc::broken_intra_doc_links" cargo doc -vv --workspace --no-deps --document-private-items
 
   tests:
     name: Cargo test
@@ -106,7 +108,7 @@ jobs:
 
       - name: Download Substrate
         run: |
-            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            curl $SUBSTRATE_URL --output substrate --location
             chmod +x substrate
             mkdir -p ~/.local/bin
             mv substrate ~/.local/bin
@@ -136,7 +138,7 @@ jobs:
 
       - name: Download Substrate
         run: |
-            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            curl $SUBSTRATE_URL --output substrate --location
             chmod +x substrate
             mkdir -p ~/.local/bin
             mv substrate ~/.local/bin

test-runtime/Cargo.toml

@@ -10,5 +10,5 @@ codec = { package = "parity-scale-codec", version = "2", default-features = fals
 
 [build-dependencies]
 subxt = { path = ".." }
-async-std = { version = "1.9.0", features = ["attributes"] }
 sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate/", branch = "master" }
+async-std = { version = "1.9.0", features = ["attributes", "tokio1"] }

test-runtime/build.rs

@@ -18,6 +18,10 @@ use std::{
     env,
     fs,
     net::TcpListener,
+    ops::{
+        Deref,
+        DerefMut,
+    },
     path::Path,
     process::Command,
     sync::atomic::{
@@ -32,6 +36,10 @@ static SUBSTRATE_BIN_ENV_VAR: &str = "SUBSTRATE_NODE_PATH";
 
 #[async_std::main]
 async fn main() {
+    run().await;
+}
+
+async fn run() {
     // Select substrate binary to run based on env var.
     let substrate_bin =
         env::var(SUBSTRATE_BIN_ENV_VAR).unwrap_or_else(|_| "substrate".to_owned());
@@ -45,7 +53,7 @@ async fn main() {
         .arg(format!("--rpc-port={}", port))
         .spawn();
     let mut cmd = match cmd {
-        Ok(cmd) => cmd,
+        Ok(cmd) => KillOnDrop(cmd),
         Err(e) => {
             panic!("Cannot spawn substrate command '{}': {}", substrate_bin, e)
         }
@@ -55,7 +63,6 @@ async fn main() {
     let metadata_bytes: sp_core::Bytes = {
         const MAX_RETRIES: usize = 20;
         let mut retries = 0;
-        let mut wait_secs = 1;
         loop {
             if retries >= MAX_RETRIES {
                 panic!("Cannot connect to substrate node after {} retries", retries);
@@ -72,9 +79,8 @@ async fn main() {
                     break res
                 }
                 _ => {
-                    thread::sleep(time::Duration::from_secs(wait_secs));
+                    thread::sleep(time::Duration::from_secs(1));
                     retries += 1;
-                    wait_secs += 1;
                 }
             };
         }
@@ -142,3 +148,26 @@ fn next_open_port() -> Option<u16> {
         }
     }
 }
+
+/// If the substrate process isn't explicilty killed on drop,
+/// it seems that panics that occur while the command is running
+/// will leave it running and block the build step from ever finishing.
+/// Wrapping it in this prevents this from happening.
+struct KillOnDrop(std::process::Child);
+
+impl Deref for KillOnDrop {
+    type Target = std::process::Child;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for KillOnDrop {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+impl Drop for KillOnDrop {
+    fn drop(&mut self) {
+        let _ = self.0.kill();
+    }
+}