diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c3b716e68f..95f62c90363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Add Solace PubSub+ Event Broker Scaler ([#1945](https://github.com/kedacore/keda/pull/1945)) - Add fallback functionality ([#1872](https://github.com/kedacore/keda/issues/1872)) - Introduce Idle Replica Mode ([#1958](https://github.com/kedacore/keda/pull/1958)) +- Add new scaler for Selenium Grid ([#1971](https://github.com/kedacore/keda/pull/1971)) - Support using regex to select the queues in RabbitMQ Scaler ([#1957](https://github.com/kedacore/keda/pull/1957)) ### Improvements diff --git a/pkg/scalers/selenium_grid_scaler.go b/pkg/scalers/selenium_grid_scaler.go new file mode 100644 index 00000000000..dd8a92f9e7f --- /dev/null +++ b/pkg/scalers/selenium_grid_scaler.go @@ -0,0 +1,230 @@ +package scalers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + + v2beta2 "k8s.io/api/autoscaling/v2beta2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/metrics/pkg/apis/external_metrics" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + kedautil "github.com/kedacore/keda/v2/pkg/util" +) + +type seleniumGridScaler struct { + metadata *seleniumGridScalerMetadata + client *http.Client +} + +type seleniumGridScalerMetadata struct { + url string + browserName string + targetValue int64 + browserVersion string +} + +type seleniumResponse struct { + Data data `json:"data"` +} + +type data struct { + SessionsInfo sessionsInfo `json:"sessionsInfo"` +} + +type sessionsInfo struct { + SessionQueueRequests []string `json:"sessionQueueRequests"` + Sessions []seleniumSession `json:"sessions"` +} + +type seleniumSession struct { + ID string `json:"id"` + Capabilities string `json:"capabilities"` + NodeID string `json:"nodeId"` +} + +type capability struct { + BrowserName string `json:"browserName"` + BrowserVersion string `json:"browserVersion"` +} + +const ( + DefaultBrowserVersion string = "latest" +) + +var seleniumGridLog = logf.Log.WithName("selenium_grid_scaler") + +func NewSeleniumGridScaler(config *ScalerConfig) (Scaler, error) { + meta, err := parseSeleniumGridScalerMetadata(config) + + if err != nil { + return nil, fmt.Errorf("error parsing selenium grid metadata: %s", err) + } + + httpClient := kedautil.CreateHTTPClient(config.GlobalHTTPTimeout) + + return &seleniumGridScaler{ + metadata: meta, + client: httpClient, + }, nil +} + +func parseSeleniumGridScalerMetadata(config *ScalerConfig) (*seleniumGridScalerMetadata, error) { + meta := seleniumGridScalerMetadata{ + targetValue: 1, + } + + if val, ok := config.TriggerMetadata["url"]; ok { + meta.url = val + } else { + return nil, fmt.Errorf("no selenium grid url given in metadata") + } + + if val, ok := config.TriggerMetadata["browserName"]; ok { + meta.browserName = val + } else { + return nil, fmt.Errorf("no browser name given in metadata") + } + + if val, ok := config.TriggerMetadata["browserVersion"]; ok && val != "" { + meta.browserVersion = val + } else { + meta.browserVersion = DefaultBrowserVersion + } + + return &meta, nil +} + +// No cleanup required for selenium grid scaler +func (s *seleniumGridScaler) Close() error { + return nil +} + +func (s *seleniumGridScaler) GetMetrics(ctx context.Context, metricName string, metricSelector labels.Selector) ([]external_metrics.ExternalMetricValue, error) { + v, err := s.getSessionsCount() + if err != nil { + return []external_metrics.ExternalMetricValue{}, fmt.Errorf("error requesting selenium grid endpoint: %s", err) + } + + metric := external_metrics.ExternalMetricValue{ + MetricName: metricName, + Value: *v, + Timestamp: metav1.Now(), + } + + return append([]external_metrics.ExternalMetricValue{}, metric), nil +} + +func (s *seleniumGridScaler) GetMetricSpecForScaling() []v2beta2.MetricSpec { + targetValue := resource.NewQuantity(s.metadata.targetValue, resource.DecimalSI) + metricName := kedautil.NormalizeString(fmt.Sprintf("%s-%s-%s-%s", "seleniumgrid", s.metadata.url, s.metadata.browserName, s.metadata.browserVersion)) + externalMetric := &v2beta2.ExternalMetricSource{ + Metric: v2beta2.MetricIdentifier{ + Name: metricName, + }, + Target: v2beta2.MetricTarget{ + Type: v2beta2.AverageValueMetricType, + AverageValue: targetValue, + }, + } + metricSpec := v2beta2.MetricSpec{ + External: externalMetric, Type: externalMetricType, + } + return []v2beta2.MetricSpec{metricSpec} +} + +func (s *seleniumGridScaler) IsActive(ctx context.Context) (bool, error) { + v, err := s.getSessionsCount() + if err != nil { + return false, err + } + + return v.AsApproximateFloat64() > 0.0, nil +} + +func (s *seleniumGridScaler) getSessionsCount() (*resource.Quantity, error) { + body, err := json.Marshal(map[string]string{ + "query": "{ sessionsInfo { sessionQueueRequests, sessions { id, capabilities, nodeId } } }", + }) + + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", s.metadata.url, bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + + res, err := s.client.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + msg := fmt.Sprintf("selenium grid returned %d", res.StatusCode) + return nil, errors.New(msg) + } + + defer res.Body.Close() + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + v, err := getCountFromSeleniumResponse(b, s.metadata.browserName, s.metadata.browserVersion) + if err != nil { + return nil, err + } + return v, nil +} + +func getCountFromSeleniumResponse(b []byte, browserName string, browserVersion string) (*resource.Quantity, error) { + var count int64 + var seleniumResponse = seleniumResponse{} + + if err := json.Unmarshal(b, &seleniumResponse); err != nil { + return nil, err + } + + var sessionQueueRequests = seleniumResponse.Data.SessionsInfo.SessionQueueRequests + for _, sessionQueueRequest := range sessionQueueRequests { + var capability = capability{} + if err := json.Unmarshal([]byte(sessionQueueRequest), &capability); err == nil { + if capability.BrowserName == browserName { + if strings.HasPrefix(capability.BrowserVersion, browserVersion) { + count++ + } else if capability.BrowserVersion == "" && browserVersion == DefaultBrowserVersion { + count++ + } + } + } else { + seleniumGridLog.Error(err, fmt.Sprintf("Error when unmarshaling session queue requests: %s", err)) + } + } + + var sessions = seleniumResponse.Data.SessionsInfo.Sessions + for _, session := range sessions { + var capability = capability{} + if err := json.Unmarshal([]byte(session.Capabilities), &capability); err == nil { + if capability.BrowserName == browserName { + if strings.HasPrefix(capability.BrowserVersion, browserVersion) { + count++ + } else if browserVersion == DefaultBrowserVersion { + count++ + } + } + } else { + seleniumGridLog.Error(err, fmt.Sprintf("Error when unmarshaling sessions info: %s", err)) + } + } + + return resource.NewQuantity(count, resource.DecimalSI), nil +} diff --git a/pkg/scalers/selenium_grid_scaler_test.go b/pkg/scalers/selenium_grid_scaler_test.go new file mode 100644 index 00000000000..a6ae05b5f8c --- /dev/null +++ b/pkg/scalers/selenium_grid_scaler_test.go @@ -0,0 +1,237 @@ +package scalers + +import ( + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func Test_getCountFromSeleniumResponse(t *testing.T) { + type args struct { + b []byte + browserName string + browserVersion string + } + tests := []struct { + name string + args args + want *resource.Quantity + wantErr bool + }{ + { + name: "nil response body should through error", + args: args{ + b: []byte(nil), + browserName: "", + }, + // want: resource.NewQuantity(0, resource.DecimalSI), + wantErr: true, + }, + { + name: "empty response body should through error", + args: args{ + b: []byte(""), + browserName: "", + }, + // want: resource.NewQuantity(0, resource.DecimalSI), + wantErr: true, + }, + { + name: "no active sessions should return count as 0", + args: args{ + b: []byte(`{ + "data": { + "sessionsInfo": { + "sessionQueueRequests": [], + "sessions": [] + } + } + }`), + browserName: "", + }, + want: resource.NewQuantity(0, resource.DecimalSI), + wantErr: false, + }, + { + name: "active sessions with no matching browsername should return count as 0", + args: args{ + b: []byte(`{ + "data": { + "sessionsInfo": { + "sessionQueueRequests": ["{\n \"browserName\": \"chrome\"\n}","{\n \"browserName\": \"chrome\"\n}"], + "sessions": [ + { + "id": "0f9c5a941aa4d755a54b84be1f6535b1", + "capabilities": "{\n \"acceptInsecureCerts\": false,\n \"browserName\": \"chrome\",\n \"browserVersion\": \"91.0.4472.114\",\n \"chrome\": {\n \"chromedriverVersion\": \"91.0.4472.101 (af52a90bf87030dd1523486a1cd3ae25c5d76c9b-refs\\u002fbranch-heads\\u002f4472@{#1462})\",\n \"userDataDir\": \"\\u002ftmp\\u002f.com.google.Chrome.DMqx9m\"\n },\n \"goog:chromeOptions\": {\n \"debuggerAddress\": \"localhost:35839\"\n },\n \"networkConnectionEnabled\": false,\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"proxy\": {\n },\n \"se:cdp\": \"http:\\u002f\\u002flocalhost:35839\",\n \"se:cdpVersion\": \"91.0.4472.114\",\n \"se:vncEnabled\": true,\n \"se:vncLocalAddress\": \"ws:\\u002f\\u002flocalhost:7900\\u002fwebsockify\",\n \"setWindowRect\": true,\n \"strictFileInteractability\": false,\n \"timeouts\": {\n \"implicit\": 0,\n \"pageLoad\": 300000,\n \"script\": 30000\n },\n \"unhandledPromptBehavior\": \"dismiss and notify\",\n \"webauthn:extension:largeBlob\": true,\n \"webauthn:virtualAuthenticators\": true\n}", + "nodeId": "d44dcbc5-0b2c-4d5e-abf4-6f6aa5e0983c" + } + ] + } + } + }`), + browserName: "", + browserVersion: "latest", + }, + want: resource.NewQuantity(0, resource.DecimalSI), + wantErr: false, + }, + { + name: "active sessions with matching browsername should return count as 2", + args: args{ + b: []byte(`{ + "data": { + "sessionsInfo": { + "sessionQueueRequests": ["{\n \"browserName\": \"chrome\"\n}","{\n \"browserName\": \"chrome\"\n}"], + "sessions": [] + } + } + }`), + browserName: "chrome", + browserVersion: "latest", + }, + want: resource.NewQuantity(2, resource.DecimalSI), + wantErr: false, + }, + { + name: "active sessions with matching browsername should return count as 3", + args: args{ + b: []byte(`{ + "data": { + "sessionsInfo": { + "sessionQueueRequests": ["{\n \"browserName\": \"chrome\"\n}","{\n \"browserName\": \"chrome\"\n}"], + "sessions": [ + { + "id": "0f9c5a941aa4d755a54b84be1f6535b1", + "capabilities": "{\n \"acceptInsecureCerts\": false,\n \"browserName\": \"chrome\",\n \"browserVersion\": \"91.0.4472.114\",\n \"chrome\": {\n \"chromedriverVersion\": \"91.0.4472.101 (af52a90bf87030dd1523486a1cd3ae25c5d76c9b-refs\\u002fbranch-heads\\u002f4472@{#1462})\",\n \"userDataDir\": \"\\u002ftmp\\u002f.com.google.Chrome.DMqx9m\"\n },\n \"goog:chromeOptions\": {\n \"debuggerAddress\": \"localhost:35839\"\n },\n \"networkConnectionEnabled\": false,\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"proxy\": {\n },\n \"se:cdp\": \"http:\\u002f\\u002flocalhost:35839\",\n \"se:cdpVersion\": \"91.0.4472.114\",\n \"se:vncEnabled\": true,\n \"se:vncLocalAddress\": \"ws:\\u002f\\u002flocalhost:7900\\u002fwebsockify\",\n \"setWindowRect\": true,\n \"strictFileInteractability\": false,\n \"timeouts\": {\n \"implicit\": 0,\n \"pageLoad\": 300000,\n \"script\": 30000\n },\n \"unhandledPromptBehavior\": \"dismiss and notify\",\n \"webauthn:extension:largeBlob\": true,\n \"webauthn:virtualAuthenticators\": true\n}", + "nodeId": "d44dcbc5-0b2c-4d5e-abf4-6f6aa5e0983c" + } + ] + } + } + }`), + browserName: "chrome", + browserVersion: "latest", + }, + want: resource.NewQuantity(3, resource.DecimalSI), + wantErr: false, + }, + { + name: "active sessions with matching browsername and version should return count as 2", + args: args{ + b: []byte(`{ + "data": { + "sessionsInfo": { + "sessionQueueRequests": ["{\n \"browserName\": \"chrome\",\n \"browserVersion\": \"91.0\"\n}","{\n \"browserName\": \"chrome\"\n}"], + "sessions": [ + { + "id": "0f9c5a941aa4d755a54b84be1f6535b1", + "capabilities": "{\n \"acceptInsecureCerts\": false,\n \"browserName\": \"chrome\",\n \"browserVersion\": \"91.0.4472.114\",\n \"chrome\": {\n \"chromedriverVersion\": \"91.0.4472.101 (af52a90bf87030dd1523486a1cd3ae25c5d76c9b-refs\\u002fbranch-heads\\u002f4472@{#1462})\",\n \"userDataDir\": \"\\u002ftmp\\u002f.com.google.Chrome.DMqx9m\"\n },\n \"goog:chromeOptions\": {\n \"debuggerAddress\": \"localhost:35839\"\n },\n \"networkConnectionEnabled\": false,\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"proxy\": {\n },\n \"se:cdp\": \"http:\\u002f\\u002flocalhost:35839\",\n \"se:cdpVersion\": \"91.0.4472.114\",\n \"se:vncEnabled\": true,\n \"se:vncLocalAddress\": \"ws:\\u002f\\u002flocalhost:7900\\u002fwebsockify\",\n \"setWindowRect\": true,\n \"strictFileInteractability\": false,\n \"timeouts\": {\n \"implicit\": 0,\n \"pageLoad\": 300000,\n \"script\": 30000\n },\n \"unhandledPromptBehavior\": \"dismiss and notify\",\n \"webauthn:extension:largeBlob\": true,\n \"webauthn:virtualAuthenticators\": true\n}", + "nodeId": "d44dcbc5-0b2c-4d5e-abf4-6f6aa5e0983c" + } + ] + } + } + }`), + browserName: "chrome", + browserVersion: "91.0", + }, + want: resource.NewQuantity(2, resource.DecimalSI), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getCountFromSeleniumResponse(tt.args.b, tt.args.browserName, tt.args.browserVersion) + if (err != nil) != tt.wantErr { + t.Errorf("getCountFromSeleniumResponse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getCountFromSeleniumResponse() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_parseSeleniumGridScalerMetadata(t *testing.T) { + type args struct { + config *ScalerConfig + } + tests := []struct { + name string + args args + want *seleniumGridScalerMetadata + wantErr bool + }{ + { + name: "invalid url string should throw error", + args: args{ + config: &ScalerConfig{ + TriggerMetadata: map[string]string{}, + }, + }, + wantErr: true, + }, + { + name: "invalid browsername string should throw error", + args: args{ + config: &ScalerConfig{ + TriggerMetadata: map[string]string{ + "url": "", + }, + }, + }, + wantErr: true, + }, + { + name: "valid url and browsername should return metadata", + args: args{ + config: &ScalerConfig{ + TriggerMetadata: map[string]string{ + "url": "http://selenium-hub:4444/graphql", + "browserName": "chrome", + }, + }, + }, + wantErr: false, + want: &seleniumGridScalerMetadata{ + url: "http://selenium-hub:4444/graphql", + browserName: "chrome", + targetValue: 1, + browserVersion: "latest", + }, + }, + { + name: "valid url and browsername should return metadata", + args: args{ + config: &ScalerConfig{ + TriggerMetadata: map[string]string{ + "url": "http://selenium-hub:4444/graphql", + "browserName": "chrome", + "browserVersion": "91.0", + }, + }, + }, + wantErr: false, + want: &seleniumGridScalerMetadata{ + url: "http://selenium-hub:4444/graphql", + browserName: "chrome", + targetValue: 1, + browserVersion: "91.0", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSeleniumGridScalerMetadata(tt.args.config) + if (err != nil) != tt.wantErr { + t.Errorf("parseSeleniumGridScalerMetadata() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseSeleniumGridScalerMetadata() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index 90dd6e66990..40a3a1564e8 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -449,6 +449,8 @@ func buildScaler(triggerType string, config *scalers.ScalerConfig) (scalers.Scal return scalers.NewRedisStreamsScaler(true, config) case "redis-streams": return scalers.NewRedisStreamsScaler(false, config) + case "selenium-grid": + return scalers.NewSeleniumGridScaler(config) case "solace-event-queue": return scalers.NewSolaceScaler(config) case "stan": diff --git a/tests/scalers/selenium-grid-test.ts b/tests/scalers/selenium-grid-test.ts new file mode 100644 index 00000000000..03cfbcf1fc4 --- /dev/null +++ b/tests/scalers/selenium-grid-test.ts @@ -0,0 +1,626 @@ +import test from 'ava' +import * as sh from 'shelljs' +import * as tmp from 'tmp' +import * as fs from 'fs' + +const seleniumGridNamespace = 'selenium-grid'; +const seleniumGridHostName = `selenium-hub.${seleniumGridNamespace}`; +const seleniumGridPort = "4444"; +const seleniumGridGraphQLUrl = `http://${seleniumGridHostName}:${seleniumGridPort}/graphql`; +const seleniumGridTestName = 'selenium-random-tests'; + +test.before(t => { + sh.exec(`kubectl create namespace ${seleniumGridNamespace}`); + + const seleniumGridDeployTmpFile = tmp.fileSync(); + fs.writeFileSync(seleniumGridDeployTmpFile.name, seleniumGridYaml.replace(/{{NAMESPACE}}/g, seleniumGridNamespace)); + + t.is(0, sh.exec(`kubectl apply --namespace ${seleniumGridNamespace} -f ${seleniumGridDeployTmpFile.name}`).code, 'creating a Selenium Grid deployment should work.') + + let seleniumHubReplicaCount = '0'; + + for (let i = 0; i < 30; i++) { + seleniumHubReplicaCount = sh.exec(`kubectl get deploy/selenium-hub -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + if (seleniumHubReplicaCount == '1') { + break; + } + console.log('Waiting for selenium hub to be ready'); + sh.exec('sleep 2s') + } + t.is('1', seleniumHubReplicaCount, 'Selenium Hub is not in a ready state') +}); + +test.serial('should have one node for chrome and firefox each at start', t => { + let seleniumChromeNodeReplicaCount = '0'; + let seleniumFireFoxReplicaCount = '0'; + for (let i = 0; i < 30; i++) { + seleniumChromeNodeReplicaCount = sh.exec(`kubectl get deploy/selenium-chrome-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + seleniumFireFoxReplicaCount = sh.exec(`kubectl get deploy/selenium-firefox-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + if (seleniumChromeNodeReplicaCount == '1' && seleniumFireFoxReplicaCount == '1') { + break; + } + console.log('Waiting for chrome and firefox node to be ready'); + sh.exec('sleep 2s') + } + + t.is('1', seleniumChromeNodeReplicaCount, 'Selenium Chrome Node did not scale up to 1 pods') + t.is('1', seleniumFireFoxReplicaCount, 'Selenium Firefox Node did not scale up to 1 pods') +}); + +test.serial('should scale down browser nodes to 0', t => { + const scaledObjectDeployTmpFile = tmp.fileSync(); + fs.writeFileSync(scaledObjectDeployTmpFile.name, scaledObjectYaml.replace(/{{NAMESPACE}}/g, seleniumGridNamespace).replace(/{{SELENIUM_GRID_GRAPHQL_URL}}/g, seleniumGridGraphQLUrl)); + + t.is(0, sh.exec(`kubectl apply --namespace ${seleniumGridNamespace} -f ${scaledObjectDeployTmpFile.name}`).code, 'creating a Scaled Object CRD should work.') + + let seleniumChromeNodeReplicaCount = '1'; + let seleniumFireFoxReplicaCount = '1'; + for (let i = 0; i < 60; i++) { + seleniumChromeNodeReplicaCount = sh.exec(`kubectl get deploy/selenium-chrome-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + seleniumFireFoxReplicaCount = sh.exec(`kubectl get deploy/selenium-firefox-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + if (seleniumChromeNodeReplicaCount == '0' && seleniumFireFoxReplicaCount == '0') { + break; + } + console.log('Waiting for chrome and firefox to scale down to 0 pods') + sh.exec('sleep 5s') + } + + t.is('0', seleniumChromeNodeReplicaCount, 'Selenium Chrome Node did not scale down to 0 pods') + t.is('0', seleniumFireFoxReplicaCount, 'Selenium Firefox Node did not scale down to 0 pods') +}); + +test.serial('should create one chrome and firefox node', t => { + const seleniumGridTestDeployTmpFile = tmp.fileSync(); + fs.writeFileSync( + seleniumGridTestDeployTmpFile.name, + seleniumGridTestsYaml + .replace(/{{JOB_NAME}}/g, seleniumGridTestName) + .replace(/{{CONTAINER_NAME}}/g, seleniumGridTestName) + .replace(/{{HOST_NAME}}/g, seleniumGridHostName) + .replace(/{{PORT}}/g, seleniumGridPort) + .replace(/{{WITH_VERSION}}/g, "false") + ); + + t.is(0, sh.exec(`kubectl apply --namespace ${seleniumGridNamespace} -f ${seleniumGridTestDeployTmpFile.name}`).code, 'creating a Selenium Grid Tests deployment should work.'); + + // wait for selenium grid tests to start running + for (let i = 0; i < 20; i++) { + const running = sh.exec(`kubectl get job ${seleniumGridTestName} --namespace ${seleniumGridNamespace} -o jsonpath='{.items[0].status.running}'`).stdout + if (running == '1') { + break; + } + sh.exec('sleep 1s') + } + + let seleniumChromeNodeReplicaCount = '0'; + let seleniumFireFoxReplicaCount = '0'; + for (let i = 0; i < 30; i++) { + seleniumChromeNodeReplicaCount = seleniumChromeNodeReplicaCount != '1' ? sh.exec(`kubectl get deploy/selenium-chrome-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout : seleniumChromeNodeReplicaCount; + seleniumFireFoxReplicaCount = seleniumFireFoxReplicaCount != '1' ? sh.exec(`kubectl get deploy/selenium-firefox-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout : seleniumFireFoxReplicaCount; + if (seleniumChromeNodeReplicaCount == '1' && seleniumFireFoxReplicaCount == '1') { + break; + } + console.log('Waiting for chrome to scale up 1 pod and firefox to 1 pod'); + sh.exec('sleep 2s') + } + + t.is('1', seleniumChromeNodeReplicaCount, 'Selenium Chrome Node did not scale up to 1 pod') + t.is('1', seleniumFireFoxReplicaCount, 'Selenium Firefox Node did not scale up to 1 pod') + + // wait for selenium grid tests to complete + let succeeded = '0'; + for (let i = 0; i < 60; i++) { + succeeded = sh.exec(`kubectl get job ${seleniumGridTestName} --namespace ${seleniumGridNamespace} -o jsonpath='{.items[0].status.succeeded}'`).stdout + if (succeeded == '1') { + break; + } + sh.exec('sleep 1s') + } + + sh.exec(`kubectl delete job/${seleniumGridTestName} --namespace ${seleniumGridNamespace}`) +}); + +test.serial('should scale down chrome and firefox nodes to 0', t => { + + let seleniumChromeNodeReplicaCount = '1'; + let seleniumFireFoxReplicaCount = '1'; + for (let i = 0; i < 65; i++) { + seleniumChromeNodeReplicaCount = sh.exec(`kubectl get deploy/selenium-chrome-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout; + seleniumFireFoxReplicaCount = sh.exec(`kubectl get deploy/selenium-firefox-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout; + if (seleniumChromeNodeReplicaCount == '0' && seleniumFireFoxReplicaCount == '0') { + break; + } + console.log('Waiting for chrome and firefox to scale down to 0 pod'); + sh.exec('sleep 5s') + } + + t.is('0', seleniumChromeNodeReplicaCount, 'Selenium Chrome Node did not scale down to 0 pod') + t.is('0', seleniumFireFoxReplicaCount, 'Selenium Firefox Node did not scale down to 0 pod') +}); + +test.serial('should create two chrome and one firefox nodes', t => { + const chrome91DeployTmpFile = tmp.fileSync(); + fs.writeFileSync(chrome91DeployTmpFile.name, chrome91Yaml.replace(/{{NAMESPACE}}/g, seleniumGridNamespace).replace(/{{SELENIUM_GRID_GRAPHQL_URL}}/g, seleniumGridGraphQLUrl)); + + t.is(0, sh.exec(`kubectl apply --namespace ${seleniumGridNamespace} -f ${chrome91DeployTmpFile.name}`).code, 'creating Chrome 91 node should work.') + + let seleniumChrome91NodeReplicaCount = '1'; + for (let i = 0; i < 60; i++) { + seleniumChrome91NodeReplicaCount = sh.exec(`kubectl get deploy/selenium-chrome-node-91 -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout + if (seleniumChrome91NodeReplicaCount == '0') { + break; + } + console.log('Waiting for chrome 91 to scale down to 0 pods') + sh.exec('sleep 5s') + } + + const seleniumGridTestDeployTmpFile = tmp.fileSync(); + fs.writeFileSync( + seleniumGridTestDeployTmpFile.name, + seleniumGridTestsYaml + .replace(/{{JOB_NAME}}/g, seleniumGridTestName) + .replace(/{{CONTAINER_NAME}}/g, seleniumGridTestName) + .replace(/{{HOST_NAME}}/g, seleniumGridHostName) + .replace(/{{PORT}}/g, seleniumGridPort) + .replace(/{{WITH_VERSION}}/g, "true") + ); + + t.is(0, sh.exec(`kubectl apply --namespace ${seleniumGridNamespace} -f ${seleniumGridTestDeployTmpFile.name}`).code, 'creating a Selenium Grid Tests deployment should work.'); + + // wait for selenium grid tests to start running + for (let i = 0; i < 20; i++) { + const running = sh.exec(`kubectl get job ${seleniumGridTestName} --namespace ${seleniumGridNamespace} -o jsonpath='{.items[0].status.running}'`).stdout + if (running == '1') { + break; + } + sh.exec('sleep 1s') + } + + let seleniumChromeNodeReplicaCount = '0'; + let seleniumFireFoxReplicaCount = '0'; + seleniumChrome91NodeReplicaCount = '0'; + for (let i = 0; i < 30; i++) { + seleniumChromeNodeReplicaCount = seleniumChromeNodeReplicaCount != '1' ? sh.exec(`kubectl get deploy/selenium-chrome-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout : seleniumChromeNodeReplicaCount; + seleniumFireFoxReplicaCount = seleniumFireFoxReplicaCount != '1' ? sh.exec(`kubectl get deploy/selenium-firefox-node -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout : seleniumFireFoxReplicaCount; + seleniumChrome91NodeReplicaCount = seleniumChrome91NodeReplicaCount != '1' ? sh.exec(`kubectl get deploy/selenium-chrome-node-91 -n ${seleniumGridNamespace} -o jsonpath='{.spec.replicas}'`).stdout : seleniumChrome91NodeReplicaCount; + if (seleniumChromeNodeReplicaCount == '1' && seleniumFireFoxReplicaCount == '1' && seleniumChrome91NodeReplicaCount == '1') { + break; + } + console.log('Waiting for chrome to scale up 2 pods and firefox to 1 pod'); + sh.exec('sleep 2s') + } + + sh.exec(`kubectl delete job/${seleniumGridTestName} --namespace ${seleniumGridNamespace}`) + + t.is('1', seleniumChromeNodeReplicaCount, 'Selenium Chrome Node did not scale up to 1 pod') + t.is('1', seleniumChrome91NodeReplicaCount, 'Selenium Chrome 91 Node did not scale up to 1 pod') + t.is('1', seleniumFireFoxReplicaCount, 'Selenium Firefox Node did not scale up to 1 pod') +}); + +test.after.always.cb('clean up prometheus deployment', t => { + let resources = [ + 'scaledobject.keda.sh/selenium-grid-chrome-scaledobject', + 'scaledobject.keda.sh/selenium-grid-firefox-scaledobject', + 'service/selenium-chrome-node', + 'deployment.apps/selenium-chrome-node', + 'service/selenium-firefox-node', + 'deployment.apps/selenium-firefox-node', + 'service/selenium-hub', + 'deployment.apps/selenium-hub', + `job/${seleniumGridTestName}`, + 'config/selenium-event-bus-config' + ] + + for (const resource of resources) { + sh.exec(`kubectl delete ${resource} --namespace ${seleniumGridNamespace}`) + } + sh.exec(`kubectl delete namespace ${seleniumGridNamespace}`) + + t.end() +}); + +const seleniumGridYaml = `--- +# Source: selenium-grid/templates/event-bus-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: selenium-event-bus-config + namespace: {{NAMESPACE}} + labels: + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +data: + SE_EVENT_BUS_HOST: selenium-hub + SE_EVENT_BUS_PUBLISH_PORT: "4442" + SE_EVENT_BUS_SUBSCRIBE_PORT: "4443" +--- +# Source: selenium-grid/templates/chrome-node-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: selenium-chrome-node + namespace: {{NAMESPACE}} + labels: + name: selenium-chrome-node + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + type: ClusterIP + selector: + app: selenium-chrome-node + ports: + - name: tcp-chrome + protocol: TCP + port: 6900 + targetPort: 5900 +--- +# Source: selenium-grid/templates/firefox-node-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: selenium-firefox-node + namespace: {{NAMESPACE}} + labels: + name: selenium-firefox-node + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + type: ClusterIP + selector: + app: selenium-firefox-node + ports: + - name: tcp-firefox + protocol: TCP + port: 6900 + targetPort: 5900 +--- +# Source: selenium-grid/templates/hub-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: selenium-hub + namespace: {{NAMESPACE}} + labels: + app: selenium-hub + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + selector: + app: selenium-hub + type: NodePort + ports: + - name: http-hub + protocol: TCP + port: 4444 + targetPort: 4444 + - name: tcp-hub-pub + protocol: TCP + port: 4442 + targetPort: 4442 + - name: tcp-hub-sub + protocol: TCP + port: 4443 + targetPort: 4443 +--- +# Source: selenium-grid/templates/chrome-node-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: selenium-chrome-node + namespace: {{NAMESPACE}} + labels: &chrome_node_labels + app: selenium-chrome-node + app.kubernetes.io/name: selenium-chrome-node + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: selenium-chrome-node + template: + metadata: + labels: *chrome_node_labels + annotations: + checksum/event-bus-configmap: 0e5e9d25a669359a37dd0d684c485f4c05729da5a26a841ad9a2743d99460f73 + spec: + containers: + - name: selenium-chrome-node + image: selenium/node-chrome:4.0.0-rc-1-prerelease-20210618 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: selenium-event-bus-config + ports: + - containerPort: 5553 + protocol: TCP + volumeMounts: + - name: dshm + mountPath: /dev/shm + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi +--- +# Source: selenium-grid/templates/firefox-node-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: selenium-firefox-node + namespace: {{NAMESPACE}} + labels: &firefox_node_labels + app: selenium-firefox-node + app.kubernetes.io/name: selenium-firefox-node + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: selenium-firefox-node + template: + metadata: + labels: *firefox_node_labels + annotations: + checksum/event-bus-configmap: 0e5e9d25a669359a37dd0d684c485f4c05729da5a26a841ad9a2743d99460f73 + spec: + containers: + - name: selenium-firefox-node + image: selenium/node-firefox:4.0.0-rc-1-prerelease-20210618 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: selenium-event-bus-config + ports: + - containerPort: 5553 + protocol: TCP + volumeMounts: + - name: dshm + mountPath: /dev/shm + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi +--- +# Source: selenium-grid/templates/hub-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: selenium-hub + namespace: {{NAMESPACE}} + labels: &hub_labels + app: selenium-hub + app.kubernetes.io/name: selenium-hub + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: selenium-hub + template: + metadata: + labels: *hub_labels + spec: + containers: + - name: selenium-hub + image: selenium/hub:4.0.0-rc-1-prerelease-20210618 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 4444 + protocol: TCP + - containerPort: 4442 + protocol: TCP + - containerPort: 4443 + protocol: TCP + livenessProbe: + httpGet: + path: /wd/hub/status + port: 4444 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /wd/hub/status + port: 4444 + initialDelaySeconds: 12 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10` + +const chrome91Yaml = `# Source: selenium-grid/templates/chrome-node-91-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: selenium-chrome-node-91 + namespace: {{NAMESPACE}} + labels: &chrome_node_labels + app: selenium-chrome-node-91 + app.kubernetes.io/name: selenium-chrome-node-91 + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: selenium-chrome-node-91 + template: + metadata: + labels: *chrome_node_labels + annotations: + checksum/event-bus-configmap: 0e5e9d25a669359a37dd0d684c485f4c05729da5a26a841ad9a2743d99460f73 + spec: + containers: + - name: selenium-chrome-node-91 + image: selenium/node-chrome:4.0.0-rc-1-prerelease-20210618 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: selenium-event-bus-config + ports: + - containerPort: 5553 + protocol: TCP + volumeMounts: + - name: dshm + mountPath: /dev/shm + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi +--- +# Source: selenium-grid/templates/chrome-node-91-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: selenium-chrome-node-91 + namespace: {{NAMESPACE}} + labels: + name: selenium-chrome-node-91 + app.kubernetes.io/managed-by: helm + app.kubernetes.io/instance: selenium-hpa + app.kubernetes.io/version: 4.0.0-beta-1-prerelease-20210114 + app.kubernetes.io/component: selenium-grid-4.0.0-beta-1-prerelease-20210114 + helm.sh/chart: selenium-grid-0.2.0 +spec: + type: ClusterIP + selector: + app: selenium-chrome-node-91 + ports: + - name: tcp-chrome + protocol: TCP + port: 6900 + targetPort: 5900 +--- +# Source: selenium-grid/templates/chrome-node-91-hpa.yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-91-scaledobject + namespace: {{NAMESPACE}} + labels: + deploymentName: selenium-chrome-node-91 +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node-91 + triggers: + - type: selenium-grid + metadata: + url: '{{SELENIUM_GRID_GRAPHQL_URL}}' + browserName: 'chrome' + browserVersion: '91.0' +---` + +const scaledObjectYaml = `--- +# Source: selenium-grid/templates/chrome-node-hpa.yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-scaledobject + namespace: {{NAMESPACE}} + labels: + deploymentName: selenium-chrome-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node + triggers: + - type: selenium-grid + metadata: + url: '{{SELENIUM_GRID_GRAPHQL_URL}}' + browserName: 'chrome' +--- +# Source: selenium-grid/templates/firefox-node-hpa.yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-firefox-scaledobject + namespace: {{NAMESPACE}} + labels: + deploymentName: selenium-firefox-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-firefox-node + triggers: + - type: selenium-grid + metadata: + url: '{{SELENIUM_GRID_GRAPHQL_URL}}' + browserName: 'firefox'` + +const seleniumGridTestsYaml = `apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: {{JOB_NAME}} + name: {{JOB_NAME}} +spec: + template: + metadata: + labels: + app: {{JOB_NAME}} + spec: + containers: + - name: {{CONTAINER_NAME}} + image: prashanth0007/selenium-random-tests:v1.0.2 + imagePullPolicy: Always + env: + - name: HOST_NAME + value: "{{HOST_NAME}}" + - name: PORT + value: "{{PORT}}" + - name: WITH_VERSION + value: "{{WITH_VERSION}}" + restartPolicy: Never`