Add riscof based CI testing infrastructure for testing sail-riscv

against spike using ACTs

sail-riscv is tested against spike in CI using RISCOF, for rv32, rv64
and rv32e configurations.

Signed-off-by: Abdul Wadood <abdulwadood.afzal88@gmail.com>

.github/workflows/apt-packages.txt

@@ -0,0 +1,8 @@
+opam
+zlib1g-dev
+pkg-config
+libgmp-dev
+z3
+device-tree-compiler
+build-essential
+python3-setuptools

.github/workflows/arch-tests.yml

@@ -0,0 +1,84 @@
+name: arch-test
+
+on:
+    push:
+      branches:
+        - master
+    pull_request:
+      branches:
+        - master
+
+jobs:
+    model-testing:
+        runs-on: ubuntu-latest
+
+        steps:
+        - uses: actions/checkout@v2
+          with:
+            fetch-depth: 0
+
+        - name: Install Dependencies
+          run: |
+            sudo xargs apt-get install -y < .github/workflows/apt-packages.txt
+            pip3 install git+https://github.com/riscv/riscof.git
+            wget -c https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2024.04.12/riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz
+            tar -xzf riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz
+            echo $GITHUB_WORKSPACE/riscv/bin >> $GITHUB_PATH
+
+        - name: Build spike
+          run: |
+            ci-tests/build-spike
+            echo $GITHUB_WORKSPACE/install/bin >> $GITHUB_PATH
+
+        - name: Build Sail
+          run: |
+            ci-tests/build-sail
+            echo $GITHUB_WORKSPACE/c_emulator >> $GITHUB_PATH
+
+        - name: Init arch-tests
+          run: |
+            cd ci-tests/riscof
+            git clone https://github.com/riscv-non-isa/riscv-arch-test
+            cd riscv-arch-test && git fetch --tags && git checkout tags/3.8.10
+
+        - name: Run RV32E
+          run: |
+            cd ci-tests/riscof
+            sed -i 's/\(ispec=\)\(.*\)/\1spike\/spike_isa32e.yaml/' config.ini
+            ./run-tests.sh rv32e_work
+
+        - name: Upload Artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: artifacts_rv32e
+            path: |
+              ci-tests/riscof/rv32e_work/report.html
+              ci-tests/riscof/rv32e_work/style.css
+
+        - name: Run RV32I
+          run: |
+            cd ci-tests/riscof
+            sed -i 's/\(ispec=\)\(.*\)/\1spike\/spike_isa32.yaml/' config.ini
+            ./run-tests.sh rv32i_work
+
+        - name: Upload Artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: artifacts_rv32i
+            path: |
+              ci-tests/riscof/rv32i_work/report.html
+              ci-tests/riscof/rv32i_work/style.css
+
+        - name: Run RV64I
+          run: |
+            cd ci-tests/riscof
+            sed -i 's/\(ispec=\)\(.*\)/\1spike\/spike_isa64.yaml/' config.ini
+            ./run-tests.sh rv64i_work
+
+        - name: Upload Artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: artifacts_rv64i
+            path: |
+              ci-tests/riscof/rv64i_work/report.html
+              ci-tests/riscof/rv64i_work/style.css

build_simulators.sh

@@ -10,8 +10,8 @@ function test_build () {
     fi
 }
 
-test_build make ARCH=RV32 ocaml_emulator/riscv_ocaml_sim_RV32
-test_build make ARCH=RV64 ocaml_emulator/riscv_ocaml_sim_RV64
+test_build make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" ARCH=RV32 ocaml_emulator/riscv_ocaml_sim_RV32
+test_build make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" ARCH=RV64 ocaml_emulator/riscv_ocaml_sim_RV64
 
-test_build make ARCH=RV32 c_emulator/riscv_sim_RV32
-test_build make ARCH=RV64 c_emulator/riscv_sim_RV64
+test_build make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" ARCH=RV32 c_emulator/riscv_sim_RV32
+test_build make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" ARCH=RV64 c_emulator/riscv_sim_RV64

ci-tests/build-sail

@@ -0,0 +1,8 @@
+# !bin/bash
+
+set -e
+
+opam init --disable-sandboxing -y
+eval $(opam config env)
+opam install -y sail
+./build_simulators.sh

ci-tests/build-spike

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+rm -rf riscv-isa-sim install
+mkdir install
+git clone https://github.com/riscv-software-src/riscv-isa-sim
+cd riscv-isa-sim
+mkdir build && cd build
+CXXFLAGS="-Wnon-virtual-dtor" CFLAGS="-Werror -Wignored-qualifiers -Wunused-function -Wunused-parameter -Wunused-variable" ../configure --prefix=`pwd`/../../install
+make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)"
+make check
+make install

ci-tests/riscof/config.ini

@@ -0,0 +1,14 @@
+[RISCOF]
+ReferencePlugin=sail_cSim
+ReferencePluginPath=sail_cSim
+DUTPlugin=spike
+DUTPluginPath=spike
+
+[spike]
+pluginpath=spike
+ispec=spike/spike_isa32.yaml
+pspec=spike/spike_platform.yaml
+target_run=1
+
+[sail_cSim]
+pluginpath=sail_cSim

ci-tests/riscof/run-tests.sh

@@ -0,0 +1,23 @@
+# !bin/bash
+
+set -e
+
+# Check if argument (work directory name) provided
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <argument>"
+    exit 1
+fi
+
+riscof -v debug run --config=config.ini \
+           --suite=riscv-arch-test/riscv-test-suite/ \
+           --env=riscv-arch-test/riscv-test-suite/env \
+           --no-browser --work-dir "$1"
+
+if grep -rniq "$1"/report.html -e '>0failed<'
+then
+    echo "Test successful!"
+    exit 0
+else
+    echo "Test FAILED!"
+    exit 1
+fi

ci-tests/riscof/sail_cSim/env/link.ld

@@ -0,0 +1,17 @@
+OUTPUT_ARCH( "riscv" )
+ENTRY(rvtest_entry_point)
+
+SECTIONS
+{
+  . = 0x80000000;
+  .text.init : { *(.text.init) }
+  . = ALIGN(0x1000);
+  .tohost : { *(.tohost) }
+  . = ALIGN(0x1000);
+  .text : { *(.text) }
+  . = ALIGN(0x1000);
+  .data : { *(.data) }
+  .data.string : { *(.data.string)}
+  .bss : { *(.bss) }
+  _end = .;
+}

ci-tests/riscof/sail_cSim/env/model_test.h

@@ -0,0 +1,33 @@
+#ifndef _COMPLIANCE_MODEL_H
+#define _COMPLIANCE_MODEL_H
+
+#define RVMODEL_HALT                                                           \
+  li x1, 1;                                                                    \
+  write_tohost:                                                                \
+  sw x1, tohost, x3;                                                           \
+  j write_tohost;
+
+#define RVMODEL_BOOT
+
+#define RVMODEL_DATA_BEGIN                                                     \
+  RVMODEL_DATA_SECTION.align 4;                                                \
+  .global begin_signature;                                                     \
+  begin_signature:
+
+#define RVMODEL_DATA_END                                                       \
+  .align 4;                                                                    \
+  .global end_signature;                                                       \
+  end_signature:
+
+#define RVMODEL_IO_INIT
+#define RVMODEL_IO_WRITE_STR(_R, _STR)
+#define RVMODEL_IO_CHECK()
+#define RVMODEL_IO_ASSERT_GPR_EQ(_S, _R, _I)
+#define RVMODEL_IO_ASSERT_SFPR_EQ(_F, _R, _I)
+#define RVMODEL_IO_ASSERT_DFPR_EQ(_D, _R, _I)
+#define RVMODEL_SET_MSW_INT
+#define RVMODEL_CLEAR_MSW_INT
+#define RVMODEL_CLEAR_MTIMER_INT
+#define RVMODEL_CLEAR_MEXT_INT
+
+#endif

ci-tests/riscof/sail_cSim/riscof_sail_cSim.py

@@ -0,0 +1,146 @@
+import os
+import re
+import shutil
+import subprocess
+import shlex
+import logging
+import random
+import string
+from string import Template
+
+import riscof.utils as utils
+from riscof.pluginTemplate import pluginTemplate
+import riscof.constants as constants
+from riscv_isac.isac import isac
+
+logger = logging.getLogger()
+
+class sail_cSim(pluginTemplate):
+    __model__ = "sail_c_simulator"
+    __version__ = "0.5.0"
+
+    def __init__(self, *args, **kwargs):
+        sclass = super().__init__(*args, **kwargs)
+
+        config = kwargs.get('config')
+        if config is None:
+            logger.error("Config node for sail_cSim missing.")
+            raise SystemExit(1)
+        self.num_jobs = str(config['jobs'] if 'jobs' in config else 1)
+        self.pluginpath = os.path.abspath(config['pluginpath'])
+        self.sail_exe = { '32' : os.path.join(config['PATH'] if 'PATH' in config else "","riscv_sim_RV32"),
+                '64' : os.path.join(config['PATH'] if 'PATH' in config else "","riscv_sim_RV64")}
+        self.isa_spec = os.path.abspath(config['ispec']) if 'ispec' in config else ''
+        self.platform_spec = os.path.abspath(config['pspec']) if 'ispec' in config else ''
+        self.make = config['make'] if 'make' in config else 'make'
+        logger.debug("SAIL CSim plugin initialised using the following configuration.")
+        for entry in config:
+            logger.debug(entry+' : '+config[entry])
+        return sclass
+
+    def initialise(self, suite, work_dir, archtest_env):
+        self.suite = suite
+        self.work_dir = work_dir
+        self.objdump_cmd = 'riscv64-unknown-elf-objdump -D {0} > {1};'
+        # self.archtest_env = archtest_env.replace("riscv-arch-test", "../my-repo/riscv-arch-test")
+        self.compile_cmd = 'riscv64-unknown-elf-gcc -march={0} \
+         -static -mcmodel=medany -fvisibility=hidden -nostdlib -nostartfiles\
+         -T '+self.pluginpath+'/env/link.ld\
+         -I '+self.pluginpath+'/env/\
+         -I ' + archtest_env
+
+        # workaround to avoid clang format error in sail-riscv CI.
+        # *clang suggest style that is syntactically incorrect*
+        # Insert the following macro at run-time.
+        modelTest_path = 'sail_cSim/env/model_test.h'
+        macro = """
+#define RVMODEL_DATA_SECTION .pushsection                                      \\
+  .tohost, "aw", @progbits;                                                    \\
+  .align 8;                                                                    \\
+  .global tohost;                                                              \\
+  tohost:                                                                      \\
+  .dword 0;                                                                    \\
+  .align 8;                                                                    \\
+  .global fromhost;                                                            \\
+  fromhost:                                                                    \\
+  .dword 0;                                                                    \\
+  .popsection;                                                                 \\
+  .align 8;                                                                    \\
+  .global begin_regstate;                                                      \\
+  begin_regstate:                                                              \\
+  .word 128;                                                                   \\
+  .align 8;                                                                    \\
+  .global end_regstate;                                                        \\
+  end_regstate:                                                                \\
+  .word 4;
+"""
+        with open(modelTest_path, 'r') as file:
+            modelTest = file.readlines()
+        modelTest.insert(2, macro)
+        with open(modelTest_path, 'w') as file:
+            file.writelines(modelTest)
+
+    def build(self, isa_yaml, platform_yaml):
+        ispec = utils.load_yaml(isa_yaml)['hart0']
+        self.xlen = ('64' if 64 in ispec['supported_xlen'] else '32')
+        self.isa = ' '  # 'rv' + self.xlen
+        ilp32 = 'ilp32e ' if "E" in ispec["ISA"] else 'ilp32 '
+        self.compile_cmd = self.compile_cmd+' -mabi='+('lp64 ' if 64 in ispec['supported_xlen'] else ilp32)
+        # if "I" in ispec["ISA"]:
+        #     self.isa += 'i'
+        # if "M" in ispec["ISA"]:
+        #     self.isa += 'm'
+        # if "C" in ispec["ISA"]:
+        #     self.isa += 'c'
+        # if "F" in ispec["ISA"]:
+        #     self.isa += 'f'
+        # if "D" in ispec["ISA"]:
+        #     self.isa += 'd'
+
+        # flag for sail-riscv to enable zcb extension.
+        # sail model does not yet offer flags for other extensions, supported extensions are enabled by default
+        if "Zcb" in ispec["ISA"]:
+            self.isa += ' --enable-zcb'
+
+        objdump = "riscv64-unknown-elf-objdump"
+        if shutil.which(objdump) is None:
+            logger.error(objdump+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        compiler = "riscv64-unknown-elf-gcc"
+        if shutil.which(compiler) is None:
+            logger.error(compiler+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        if shutil.which(self.sail_exe[self.xlen]) is None:
+            logger.error(self.sail_exe[self.xlen]+ ": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        if shutil.which(self.make) is None:
+            logger.error(self.make+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+
+    def runTests(self, testList, cgf_file=None):
+        if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
+        make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
+        make.makeCommand = self.make + ' -j' + self.num_jobs
+        for file in testList:
+            testentry = testList[file]
+            test = testentry['test_path']
+            test_dir = testentry['work_dir']
+            test_name = test.rsplit('/',1)[1][:-2]
+
+            elf = 'ref.elf'
+
+            execute = "@cd "+testentry['work_dir']+";"
+
+            cmd = self.compile_cmd.format(testentry['isa'].lower(), self.xlen) + ' ' + test + ' -o ' + elf
+            compile_cmd = cmd + ' -D' + " -D".join(testentry['macros'])
+            execute+=compile_cmd+";"
+
+            # execute += self.objdump_cmd.format(elf, 'ref.disass')
+            sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
+
+            # execute += self.sail_exe[self.xlen] + ' --test-signature={0} {1} > {2}.log 2>&1;'.format(sig_file, elf, test_name)
+            execute += self.sail_exe[self.xlen] + ' {0} --no-trace --test-signature={1} {2} > /dev/null;'.format(self.isa, sig_file, elf)
+
+            make.add_target(execute)
+        make.execute_all(self.work_dir)

ci-tests/riscof/spike/env/link.ld

@@ -0,0 +1,17 @@
+OUTPUT_ARCH( "riscv" )
+ENTRY(rvtest_entry_point)
+
+SECTIONS
+{
+  . = 0x80000000;
+  .text.init : { *(.text.init) }
+  . = ALIGN(0x1000);
+  .tohost : { *(.tohost) }
+  . = ALIGN(0x1000);
+  .text : { *(.text) }
+  . = ALIGN(0x1000);
+  .data : { *(.data) }
+  .data.string : { *(.data.string)}
+  .bss : { *(.bss) }
+  _end = .;
+}