Skip to content

Commit faa30a0

Browse files
author
laurennat
authored
Added E2E execution plan tests with coverage (#18929)
* start execution plan testing, need to do coverage * coverage testing with report figured out * cleaned up code a bit * brought up coverage for execution plan tests * added resuable screenshot check function * added separate readme for webview testing * added limitations * deleted unecessary nyc output files * cleaned up files * updated readme and cleaned up tests * added more context to readme * updated gulpfile * changed gulp smoke test task until integrated into pipeline * comment out smoke test pipeline job until smoke tests are integrated into pipeline * updated comments * write coverage for multiple reactviews * function to merge coverage reports with unit tests, and starting on pipelines * add back pipeline task * pipeline changes * update correct pipeline task * comment out smoke tests pipeline tasks * updated tests * getting pipelines working again * cleaned up uncessary code * Integrated playwright tests and coverage in pipelines Rename tasks * removed unecessary retest in coverage calculation * removed query result test file * fixed gulp smoke task * pr comments * fixed build size * removed unecessary command * ensure the extension has loaded before tests * turned off new ui flag for old ui playwright tests * publish extra coverage data * remove extra pipeline tasks * pr comments * changed pipeline name to be more descriptive * changed job name too
1 parent c17e83c commit faa30a0

27 files changed

+2796
-146
lines changed

‎.github/actions/build-extension/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ runs:
4343
shell: bash
4444
run: |
4545
cd ${{ inputs.source-dir }}
46-
yarn build --prod
46+
yarn build --prod=true
4747
4848
inputs:
4949
source-dir:

‎.github/workflows/pr-checks.yml ‎.github/workflows/build-and-test.yml

+140-18
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,24 @@
1-
name: PR Checks
1+
name: Build and Test (Unit + E2E)
22
# Trigger the workflow on PRs to the main branch.
33
# It performs the following checks:
44
# 1. Calculate the size difference between the webview bundles of the main branch and the PR branch.
55
# 2. Calculate the size difference between the VSIX files of the main branch and the PR branch.
66
# 3. Does a check if the PR has properly localized strings.
77

88
on:
9+
schedule:
10+
- cron: '0 0 * * *'
11+
push:
12+
branches:
13+
- main
914
pull_request:
1015
branches:
1116
- main
17+
- 'release/**'
18+
workflow_dispatch:
1219

1320
jobs:
14-
pr-checks:
21+
build-and-test:
1522
runs-on: ubuntu-latest
1623
permissions:
1724
contents: read
@@ -38,6 +45,12 @@ jobs:
3845
with:
3946
node-version: '22'
4047

48+
- name: Setup .NET Core # Required to execute ReportGenerator
49+
uses: actions/setup-dotnet@v4
50+
with:
51+
dotnet-version: 8.x
52+
dotnet-quality: 'ga'
53+
4154
- name: Install tools
4255
run: |
4356
echo "Installing Yarn"
@@ -59,24 +72,14 @@ jobs:
5972
with:
6073
source-dir: './pr'
6174

62-
- name: Code coverage
75+
- name: Run lint
6376
run: |
64-
cd main
65-
DISPLAY=:10 yarn test
66-
xml_coverage_main=$(find ./coverage -name 'cobertura-coverage.xml')
67-
line_rate_main=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_main" | sed 's/line-rate="\([0-9.]*\)"/\1/')
68-
line_rate_main=$(printf "%.2f" $(echo "$line_rate_main * 100" | bc))
69-
echo "line_rate_main=$line_rate_main" >> $GITHUB_ENV
70-
cd ../pr
71-
DISPLAY=:10 yarn test
72-
xml_coverage_pr=$(find ./coverage -name 'cobertura-coverage.xml')
73-
line_rate_pr=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_pr" | sed 's/line-rate="\([0-9.]*\)"/\1/')
74-
line_rate_pr=$(printf "%.2f" $(echo "$line_rate_pr * 100" | bc))
75-
echo "line_rate_pr=$line_rate_pr" >> $GITHUB_ENV
76-
echo "line_rate_diff=$(echo "$line_rate_pr - $line_rate_main" | bc)" >> $GITHUB_ENV
77-
coverage_text_color=$(if [ $line_rate_diff -gt 0 ]; then echo "red"; else echo "lightgreen"; fi)
78-
echo "coverage_text_color=$coverage_text_color" >> $GITHUB_ENV
77+
cd ./pr
78+
yarn lint
7979
80+
# Calculate sizes and package before testing;
81+
# Testing generates sourcemaps and instrumented code
82+
# that increase size
8083
- name: Package both branches
8184
run: |
8285
cd main
@@ -122,6 +125,125 @@ jobs:
122125
vsix_text_color=$(if [ $percentage_change -gt 0 ]; then echo "red"; else echo "lightgreen"; fi)
123126
echo "vsix_text_color=$vsix_text_color" >> $GITHUB_ENV
124127
128+
- name: Run unit tests
129+
run: |
130+
cd main
131+
DISPLAY=:10 yarn test
132+
cd ../pr
133+
DISPLAY=:10 yarn test
134+
135+
- name: PR Unit Test Report
136+
uses: dorny/test-reporter@v1
137+
if: success() || failure()
138+
with:
139+
name: 'Unit Test Report'
140+
path: ./test-reports/**/*.xml
141+
reporter: jest-junit
142+
working-directory: ./pr
143+
badge-title: 'unit-tests'
144+
145+
- name: Build mappings for React Webviews
146+
run: |
147+
cd ./pr
148+
gulp ext:compile-reactviews --prod=false
149+
cd ../main
150+
gulp ext:compile-reactviews --prod=false
151+
152+
- name: Setup environment for smoke tests
153+
run: |
154+
echo "Setting up environment for smoke tests"
155+
echo "PASSWORD=$(LC_ALL=C tr -dc 'A-Za-z0-9!?%=' < /dev/urandom | head -c 16)$(LC_ALL=C tr -dc '@^#' < /dev/urandom | head -c 1)" >> $GITHUB_ENV
156+
sudo cp build/mssql.init /etc/init.d/mssql-server
157+
sudo chmod +x /etc/init.d/mssql-server
158+
curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc
159+
sudo add-apt-repository "$(wget -qO- https://packages.microsoft.com/config/ubuntu/22.04/mssql-server-2022.list)"
160+
sudo apt-get update
161+
sudo apt-get install -y mssql-server
162+
sudo ACCEPT_EULA='Y' MSSQL_PID='Developer' MSSQL_SA_PASSWORD=${{ env.PASSWORD }} /opt/mssql/bin/mssql-conf setup || true
163+
sudo service mssql-server start
164+
working-directory: ./pr
165+
166+
- name: Run smoke tests
167+
run: |
168+
VS_CODE_VERSION=stable
169+
SERVER_NAME=localhost
170+
AUTHENTICATION_TYPE="SQL Login"
171+
USER_NAME=sa
172+
PASSWORD=${{ env.PASSWORD }}
173+
SAVE_PASSWORD=No
174+
PROFILE_NAME=test-server
175+
cd pr
176+
DISPLAY=:10 yarn smoketest
177+
cd ../main
178+
DISPLAY=:10 yarn smoketest
179+
180+
- name: Upload Smoke Test Screenshots
181+
uses: actions/upload-artifact@v4
182+
if: failure()
183+
with:
184+
name: smoke-test-failure-screenshots
185+
path: ./pr/test-results/**/
186+
retention-days: 7
187+
188+
- name: Smoke Test Report
189+
uses: dorny/test-reporter@v1
190+
if: success() || failure()
191+
with:
192+
name: 'Smoke Test Report'
193+
path: ./test-reports/**/smoke-results.xml
194+
reporter: jest-junit
195+
badge-title: 'smoke-tests'
196+
working-directory: ./pr
197+
198+
- name: Merge Smoke and Unit Test Coverage Reports
199+
run: |
200+
if [ -f main/test/resources/mergeReports.js ]; then
201+
node main/test/resources/mergeReports.js main/coverage/coverage-e2e/cobertura-coverage.xml main/coverage/cobertura-coverage.xml
202+
else
203+
echo "mergeReports.js not found in ./main, skipping..."
204+
fi
205+
206+
if [ -f pr/test/resources/mergeReports.js ]; then
207+
node pr/test/resources/mergeReports.js pr/coverage/coverage-e2e/cobertura-coverage.xml pr/coverage/cobertura-coverage.xml
208+
else
209+
echo "mergeReports.js not found in ./pr, skipping..."
210+
fi
211+
212+
- name: Generate Coverage Report
213+
uses: danielpalme/ReportGenerator-GitHub-Action@5.4.4
214+
with:
215+
reports: './pr/coverage/cobertura-coverage.xml'
216+
targetdir: 'coveragereport'
217+
reporttypes: 'Html'
218+
toolpath: 'reportgeneratortool'
219+
220+
- name: Upload coverage report artifact
221+
uses: actions/upload-artifact@v4
222+
with:
223+
name: CoverageReport # Artifact name
224+
path: coveragereport # Directory containing files to upload
225+
226+
- name: Code coverage
227+
run: |
228+
cd main
229+
xml_coverage_main="./coverage/cobertura-coverage.xml"
230+
# Pull out the line rate coverage from the main branch's cobertura report
231+
line_rate_main=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_main" | sed 's/line-rate="\([0-9.]*\)"/\1/')
232+
line_rate_main=$(printf "%.2f" $(echo "$line_rate_main * 100" | bc))
233+
# Save formatted line rate in env variable
234+
echo "line_rate_main=$line_rate_main" >> $GITHUB_ENV
235+
cd ../pr
236+
xml_coverage_pr="./coverage/cobertura-coverage.xml"
237+
# Pull out the line rate coverage from the pr branch's cobertura report
238+
line_rate_pr=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_pr" | sed 's/line-rate="\([0-9.]*\)"/\1/')
239+
line_rate_pr=$(printf "%.2f" $(echo "$line_rate_pr * 100" | bc))
240+
# Save formatted line rate in env variable
241+
echo "line_rate_pr=$line_rate_pr" >> $GITHUB_ENV
242+
# Calculate difference in line coverage rate
243+
echo "line_rate_diff=$(echo "$line_rate_pr - $line_rate_main" | bc)" >> $GITHUB_ENV
244+
coverage_text_color=$(if [ $line_rate_diff -gt 0 ]; then echo "red"; else echo "lightgreen"; fi)
245+
echo "coverage_text_color=$coverage_text_color" >> $GITHUB_ENV
246+
125247
- name: Write PR results to markdown
126248
run: |
127249
echo "### PR Changes" >> results.md

‎.github/workflows/daily-build-and-test.yml

-88
This file was deleted.

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,5 @@ package.nls.*.json
2929
webviews-metafile.json
3030
# Ignore all l10n files except the default one (bundle.l10n.json)
3131
/localization/l10n/bundle.l10n.*.json
32-
.eslintcache
32+
.eslintcache
33+
.nyc_output

‎.vscodeignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ tslint*.json
2626
**/*.ts
2727
webviews-metafile.json
2828
images/ux
29-
FEATURES.md
29+
FEATURES.md
30+
.nyc_output

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
[![Daily Build and Test](https://github.com/microsoft/vscode-mssql/actions/workflows/daily-build-and-test.yml/badge.svg?branch=main&event=schedule)](https://github.com/microsoft/vscode-mssql/actions/workflows/daily-build-and-test.yml)
1+
[![PR check](https://github.com/microsoft/vscode-mssql/actions/workflows/pr-checks.yml/badge.svg)](https://github.com/microsoft/vscode-mssql/actions/workflows/pr-checks.yml)
22
[![Gitter](https://img.shields.io/badge/chat-on%20gitter-blue.svg)](https://gitter.im/Microsoft/mssql)
33

44
# mssql for Visual Studio Code

‎gulpfile.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ const concat = require('gulp-concat');
99
const minifier = require('gulp-uglify/minifier');
1010
const uglifyjs = require('uglify-js');
1111
const argv = require('yargs').argv;
12-
const min = (argv.min === undefined) ? false : true;
13-
const prod = (argv.prod === undefined) ? false : true;
12+
const min = argv.min === "true";
13+
const prod = argv.prod === "true";
1414
const vscodeTest = require('@vscode/test-electron');
1515
const { exec } = require('child_process');
1616
const gulpESLintNew = require('gulp-eslint-new');

‎package.json

+7-2
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,15 @@
5555
"watch": "gulp watch",
5656
"lint": "eslint --quiet --cache",
5757
"localization": "gulp ext:extract-localization-strings",
58-
"smoketest": "gulp ext:smoke",
58+
"smoketest": "yarn instrument && yarn e2eTest",
5959
"test": "node ./out/test/unit/runTest.js",
6060
"package": "vsce package",
6161
"prepare": "husky",
6262
"lint-staged": "lint-staged --quiet",
6363
"precommit": "run-p lint-staged localization",
6464
"clean-package": "git clean -xfd && yarn install && yarn build && yarn gulp package:online",
65-
"testWithCoverage": "yarn test && yarn gulp cover"
65+
"instrument": "nyc instrument ./out/src ./out/src --in-place --exclude=**/views/htmlcontent/**",
66+
"e2eTest": "npx playwright test; npx nyc report --reporter=cobertura --reporter=text-summary --reporter=html --report-dir=\"coverage/coverage-e2e\" --temp-dir=\"coverage/reactviews-coverage\" --include=\"src/reactviews/pages/**/*.tsx\""
6667
},
6768
"lint-staged": {
6869
"*.ts": "eslint --quiet --cache",
@@ -110,6 +111,7 @@
110111
"@types/vscode-webview": "^1.57.5",
111112
"@typescript-eslint/eslint-plugin": "^8.7.0",
112113
"@typescript-eslint/parser": "^8.7.0",
114+
"@vitejs/plugin-react": "^4.3.4",
113115
"@vscode/l10n": "^0.0.18",
114116
"@vscode/l10n-dev": "^0.0.35",
115117
"@vscode/test-cli": "^0.0.10",
@@ -171,8 +173,11 @@
171173
"typescript": "^5.6.2",
172174
"typescript-eslint": "^8.7.0",
173175
"uglify-js": "mishoo/UglifyJS2#harmony-v2.8.22",
176+
"vite": "^6.2.0",
177+
"vite-plugin-istanbul": "^7.0.0",
174178
"vscode-nls-dev": "2.0.1",
175179
"xliff": "^6.2.1",
180+
"xml2js": "^0.6.2",
176181
"yargs": "^17.7.2"
177182
},
178183
"dependencies": {

‎playwright.config.ts

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ dotenv.config({ path: './test/e2e/.env' });
2020
*/
2121
export default defineConfig({
2222
testDir: './test/e2e',
23+
testMatch: ["**/*.spec.ts"],
2324
/* Run tests in files in parallel */
2425
fullyParallel: false,
2526
/* Fail the build on CI if you accidentally left test.only in the source code. */

0 commit comments

Comments
 (0)