diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 79ed04c9d4..3825692424 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -44,6 +44,7 @@ on: - '**/*.md' - 'CHANGELOG/**' - 'tests/build-backward-compatible/**' + - 'scripts/generate_list_env_vars/**' concurrency: group: ${{ github.workflow }}-${{ github.ref == github.ref_protected && github.run_id || github.event.pull_request.number || github.ref }} diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index 7be006961e..3a55892afb 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -29,8 +29,8 @@ | SE_HUB_PORT | 4444 | Hub config, port the Hub should listen on (default 4444) | --port | | SE_ROUTER_PORT | 4444 | Router config, port the Router should listen on (default 4444) | --port | | SE_NODE_GRID_GRAPHQL_URL | | Video recording config, GraphQL URL to query test metadata for dynamic file name | | -| SE_VIDEO_FILE_NAME_TRIM_REGEX | [:alnum:]-_ | Bash regex to trim the file name if it is too long | | -| SE_VIDEO_FILE_NAME_SUFFIX | | Append a suffix session id along with test metadata | | +| SE_VIDEO_FILE_NAME_TRIM_REGEX | [^a-zA-Z0-9-_] | Python regex to trim the file name if it is too long | | +| SE_VIDEO_FILE_NAME_SUFFIX | true | Append a suffix session id along with test metadata | | | SE_RCLONE_CONFIG | | | | | SE_UPLOAD_COMMAND | | | | | SE_UPLOAD_OPTS | | | | diff --git a/README.md b/README.md index b8100f33b8..dcfe77203e 100644 --- a/README.md +++ b/README.md @@ -639,7 +639,7 @@ If your test name is handled by the test framework, and it is unique for sure, y File name will be trimmed to 255 characters to avoid long file names. Moreover, `space` character will be replaced by `_` and only characters alphabets, numbers, `-` (hyphen), `_` (underscore) are retained in the file name. -The trim regex is able to be customized by setting `SE_VIDEO_FILE_NAME_TRIM_REGEX` environment variable. The default value is `[:alnum:]-_`. The regex should be compatible with the `tr` command in bash. +The trim regex is able to be customized by setting `SE_VIDEO_FILE_NAME_TRIM_REGEX` environment variable. The default value is `[^a-zA-Z0-9-_]`. The regex should be compatible with Python `re.compile()` function. At deployment level, the recorder container is up always. In addition, you can disable video recording process via session capability `se:recordVideo`. For example in Python binding: diff --git a/Video/Dockerfile b/Video/Dockerfile index d80947dc81..5e12e87599 100644 --- a/Video/Dockerfile +++ b/Video/Dockerfile @@ -44,7 +44,8 @@ ENV DISPLAY_NUM="99" \ SE_PRESET="-preset ultrafast" \ VIDEO_FOLDER="/videos" \ SE_VIDEO_FILE_NAME="video.mp4" \ - SE_VIDEO_FILE_NAME_TRIM_REGEX="[:alnum:]-_" \ + SE_VIDEO_FILE_NAME_TRIM_REGEX="[^a-zA-Z0-9-_]" \ + SE_VIDEO_FILE_NAME_SUFFIX="true" \ # Environment variables for the uploader RCLONE_CONFIG="/opt/selenium/upload.conf" \ SE_VIDEO_UPLOAD_ENABLED="false" \ diff --git a/Video/video_nodeQuery.py b/Video/video_nodeQuery.py index db92d3757e..77910dbd66 100644 --- a/Video/video_nodeQuery.py +++ b/Video/video_nodeQuery.py @@ -4,8 +4,8 @@ import os import re import sys -from typing import List, Tuple +default_trim_pattern = "[^a-zA-Z0-9-_]" def main() -> None: """ @@ -26,7 +26,7 @@ def main() -> None: video_cap_name = os.environ.get("VIDEO_CAP_NAME", "se:recordVideo") test_name_cap = os.environ.get("TEST_NAME_CAP", "se:name") video_name_cap = os.environ.get("VIDEO_NAME_CAP", "se:videoName") - video_file_name_trim = os.environ.get("SE_VIDEO_FILE_NAME_TRIM_REGEX", "[:alnum:]-_") + video_file_name_trim = os.environ.get("SE_VIDEO_FILE_NAME_TRIM_REGEX", default_trim_pattern) video_file_name_suffix = os.environ.get("SE_VIDEO_FILE_NAME_SUFFIX", "true") # Initialize variables @@ -79,7 +79,7 @@ def normalize_filename(filename: str, trim_pattern: str) -> str: Args: filename: The original filename - trim_pattern: Pattern defining allowed characters (e.g., "[:alnum:]-_") + trim_pattern: Pattern defining allowed characters Returns: Normalized filename @@ -90,21 +90,10 @@ def normalize_filename(filename: str, trim_pattern: str) -> str: # Replace spaces with underscores normalized = filename.replace(" ", "_") - # Convert trim pattern to regex - # Handle character classes like [:alnum:] - posix_classes = { - "[:alnum:]": "a-zA-Z0-9", - "[:alpha:]": "a-zA-Z", - "[:digit:]": "0-9", - "[:space:]": " \t\n\r\f\v" - } - - allowed_chars = trim_pattern - for posix_class, replacement in posix_classes.items(): - if posix_class in allowed_chars: - allowed_chars = allowed_chars.replace(posix_class, replacement) - - pattern = f"[^{re.escape(allowed_chars)}]" + try: + pattern = re.compile(trim_pattern) + except re.error: + pattern = re.compile(default_trim_pattern) # Remove disallowed characters normalized = re.sub(pattern, "", normalized) diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index f6208f31e4..6c448407c8 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -672,6 +672,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | videoRecorder.uploader.configFileName | string | `"upload.conf"` | Uploader config file name | | videoRecorder.uploader.entryPointFileName | string | `"upload.sh"` | Uploader entry point file name | | videoRecorder.uploader.secrets | string | `nil` | For environment variables used in uploader which contains sensitive information, store in secret and refer envFrom Set config for rclone via ENV var with format: RCLONE_CONFIG_ + name of remote + _ + name of config file option (make it all uppercase) | +| videoRecorder.uploader.extraEnvFrom | list | `[]` | Custom environment variables by sourcing entire configMap, Secret, etc. for uploader | | videoRecorder.ports | list | `[9000]` | Video recording container port | | videoRecorder.resources.requests | object | `{"cpu":"0.1","memory":"128Mi"}` | Request resources for video recorder pods | | videoRecorder.resources.limits | object | `{"cpu":"0.5","memory":"1Gi"}` | Limit resources for video recorder pods | diff --git a/charts/selenium-grid/multiple-nodes-platform-version.yaml b/charts/selenium-grid/multiple-nodes-platform-version.yaml index e7272c7d21..24ceb8e40c 100644 --- a/charts/selenium-grid/multiple-nodes-platform-version.yaml +++ b/charts/selenium-grid/multiple-nodes-platform-version.yaml @@ -13,6 +13,26 @@ crossBrowsers: hpa: platformName: 'Linux' browserVersion: '' + - nameOverride: '{{ $.Release.Name }}-node-chrome-137' + imageTag: '137.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '137.0' + - nameOverride: '{{ $.Release.Name }}-node-chrome-136' + imageTag: '136.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '136.0' + - nameOverride: '{{ $.Release.Name }}-node-chrome-135' + imageTag: '135.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '135.0' + - nameOverride: '{{ $.Release.Name }}-node-chrome-134' + imageTag: '134.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '134.0' - nameOverride: '{{ $.Release.Name }}-node-chrome-133' imageTag: '133.0-20250707' hpa: @@ -204,6 +224,26 @@ crossBrowsers: hpa: platformName: 'Linux' browserVersion: '' + - nameOverride: '{{ $.Release.Name }}-node-firefox-139' + imageTag: '139.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '139.0' + - nameOverride: '{{ $.Release.Name }}-node-firefox-138' + imageTag: '138.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '138.0' + - nameOverride: '{{ $.Release.Name }}-node-firefox-137' + imageTag: '137.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '137.0' + - nameOverride: '{{ $.Release.Name }}-node-firefox-136' + imageTag: '136.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '136.0' - nameOverride: '{{ $.Release.Name }}-node-firefox-135' imageTag: '135.0-20250707' hpa: @@ -400,6 +440,26 @@ crossBrowsers: hpa: platformName: 'Linux' browserVersion: '' + - nameOverride: '{{ $.Release.Name }}-node-edge-137' + imageTag: '137.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '137.0' + - nameOverride: '{{ $.Release.Name }}-node-edge-136' + imageTag: '136.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '136.0' + - nameOverride: '{{ $.Release.Name }}-node-edge-135' + imageTag: '135.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '135.0' + - nameOverride: '{{ $.Release.Name }}-node-edge-134' + imageTag: '134.0-20250707' + hpa: + platformName: 'Linux' + browserVersion: '134.0' - nameOverride: '{{ $.Release.Name }}-node-edge-133' imageTag: '133.0-20250707' hpa: diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 02da2cda16..c875f35dd2 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -450,6 +450,15 @@ template: {{- with .recorder.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} + {{- if and .recorder.uploader.enabled (empty .recorder.uploader.name) }} + {{- with $.Values.uploaderConfigMap.secretVolumeMountName }} + - secretRef: + name: {{ tpl . $ }} + {{- end }} + {{- with .recorder.uploader.extraEnvFrom }} + {{- tpl (toYaml .) $ | nindent 10 }} + {{- end }} + {{- end }} {{- end }} ports: - containerPort: {{ .node.port }} @@ -580,13 +589,20 @@ template: name: {{ template "seleniumGrid.recorder.configmap.fullname" $ }} - configMapRef: name: {{ template "seleniumGrid.server.configmap.fullname" $ }} + - secretRef: + name: {{ template "seleniumGrid.common.secrets.fullname" $ }} {{- if $.Values.basicAuth.enabled }} - secretRef: name: {{ template "seleniumGrid.basicAuth.secrets.fullname" $ }} {{- end }} {{- if and .recorder.uploader.enabled (empty .recorder.uploader.name) }} + {{- with $.Values.uploaderConfigMap.secretVolumeMountName }} - secretRef: - name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) $.Values.uploaderConfigMap.secretVolumeMountName) $ }} + name: {{ tpl . $ }} + {{- end }} + {{- with .recorder.uploader.extraEnvFrom }} + {{- tpl (toYaml .) $ | nindent 10 }} + {{- end }} {{- end }} {{- with .recorder.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 10 }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 9025518a95..182a7889b2 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1968,6 +1968,8 @@ videoRecorder: # RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "xxx" # RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" # RCLONE_CONFIG_GS_NO_CHECK_BUCKET: "true" + # -- Custom environment variables by sourcing entire configMap, Secret, etc. for uploader + extraEnvFrom: [] # -- Video recording container port ports: - 9000 diff --git a/scripts/generate_list_env_vars/description.yaml b/scripts/generate_list_env_vars/description.yaml index 6158f0c78c..3253bfa34a 100644 --- a/scripts/generate_list_env_vars/description.yaml +++ b/scripts/generate_list_env_vars/description.yaml @@ -90,7 +90,7 @@ file name cli: '' - name: SE_VIDEO_FILE_NAME_TRIM_REGEX - description: Bash regex to trim the file name if it is too long + description: Python regex to trim the file name if it is too long cli: '' - name: SE_VIDEO_FILE_NAME_SUFFIX description: Append a suffix session id along with test metadata diff --git a/scripts/generate_list_env_vars/value.yaml b/scripts/generate_list_env_vars/value.yaml index ea9eb52e70..dc0b66897e 100644 --- a/scripts/generate_list_env_vars/value.yaml +++ b/scripts/generate_list_env_vars/value.yaml @@ -275,9 +275,9 @@ - name: SE_VIDEO_FILE_NAME default: video.mp4 - name: SE_VIDEO_FILE_NAME_SUFFIX - default: '' + default: 'true' - name: SE_VIDEO_FILE_NAME_TRIM_REGEX - default: '[:alnum:]-_' + default: '[^a-zA-Z0-9-_]' - name: SE_VIDEO_FILE_READY_WAIT_ATTEMPTS default: '' - name: SE_VIDEO_FOLDER diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 4e117b3519..c51a86ed5e 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -36,8 +36,8 @@ TEST_MULTIPLE_PLATFORMS = os.environ.get('TEST_MULTIPLE_PLATFORMS', 'false').lower() == 'true' TEST_MULTIPLE_PLATFORMS_RELAY = os.environ.get('TEST_MULTIPLE_PLATFORMS_RELAY', 'false').lower() == 'true' TEST_MULTIPLE_VERSIONS_EXPLICIT = os.environ.get('TEST_MULTIPLE_VERSIONS_EXPLICIT', 'true').lower() == 'true' -LIST_CHROMIUM_VERSIONS = ['130.0', '129.0', '128.0'] -LIST_FIREFOX_VERSIONS = ['132.0', '131.0', '130.0', '129.0', '128.0'] +LIST_CHROMIUM_VERSIONS = ['137.0', '136.0', '135.0', '134.0', '133.0', '132.0', '131.0'] +LIST_FIREFOX_VERSIONS = ['139.0', '138.0', '137.0', '136.0', '135.0', '134.0', '133.0'] LIST_PLATFORMS = ['Linux', None, 'Windows 11'] if not TEST_MULTIPLE_VERSIONS_EXPLICIT: diff --git a/tests/charts/refValues/README.md b/tests/charts/refValues/README.md index d0f1ad5f0e..4e96c469e6 100644 --- a/tests/charts/refValues/README.md +++ b/tests/charts/refValues/README.md @@ -5,8 +5,9 @@ * [Configure Kubernetes cluster in Docker Desktop](#configure-kubernetes-cluster-in-docker-desktop) * [Deploy Selenium Grid solution using Helm chart](#deploy-selenium-grid-solution-using-helm-chart) * [Deploy PVC for video recording and video manager storage.](#deploy-pvc-for-video-recording-and-video-manager-storage) + * [Deploy the secret for credentials to upload video recordings to AWS S3.](#deploy-the-secret-for-credentials-to-upload-video-recordings-to-aws-s3) * [Add Docker Selenium Helm chart repository](#add-docker-selenium-helm-chart-repository) - * [Install latest chart with reference values.](#install-latest-chart-with-reference-values) + * [Install the latest chart with reference values.](#install-the-latest-chart-with-reference-values) * [Verify Grid installation](#verify-grid-installation) * [Browser Nodes in autoscaling from zero mode.](#browser-nodes-in-autoscaling-from-zero-mode) * [Run a test in Grid](#run-a-test-in-grid) @@ -59,6 +60,22 @@ Checkout file [local-pvc-docker-desktop.yaml](local-pvc-docker-desktop.yaml) kubectl apply -f local-pvc-docker-desktop.yaml ``` +#### Deploy the secret for credentials to upload video recordings to AWS S3. + +Checkout file [aws-s3-upload-secret.yaml](aws-s3-upload-secret.yaml). Replace `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your AWS [access key](https://docs.aws.amazon.com/keyspaces/latest/devguide/aws.credentials.manage.html). + +```sh +kubectl apply -f aws-s3-upload-secret.yaml +``` + +Note: ++ You also can export AWS secret values to environment variables and use `envsubst` to substitute them in the YAML file. ++ `envsubst` is a command that substitutes environment variables in the YAML file. Install it following the instructions at [envsubst](https://github.com/a8m/envsubst?tab=readme-ov-file#linux-and-macos). + +```sh +envsubst