test: run legacy-cli e2e tests via bazel

Author: jbedard

.bazelrc

@@ -90,6 +90,10 @@ build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stam
 build:snapshot --stamp
 build:snapshot --//:enable_snapshot_repo_deps
 
+build:e2e --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=release"
+build:e2e --stamp
+test:e2e  --test_timeout=3600
+
 build:local --//:enable_package_json_tar_deps
 
 ###############################

.circleci/dynamic_config.yml

@@ -48,7 +48,9 @@ var_6: &only_pull_requests
       only:
         - /pull\/\d+/
 
+# All e2e test suites
 var_7: &all_e2e_subsets ['npm', 'esbuild', 'yarn']
+var_8: &all_e2e_build_types ['e2e', 'snapshot']
 
 # Executor Definitions
 # https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors
@@ -63,10 +65,20 @@ executors:
     working_directory: ~/ng
     resource_class: small
 
+  bazel-executor:
+    parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion
+    docker:
+      - image: cimg/node:<< parameters.nodeversion >>-browsers
+    working_directory: ~/ng
+    resource_class: xlarge
+
   windows-executor:
     # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named.
     working_directory: ~/ng
-    resource_class: windows.medium
+    resource_class: windows.large
     shell: powershell.exe -ExecutionPolicy Bypass
     machine:
       # Contents of this image:
@@ -116,7 +128,7 @@ commands:
       - initialize_env
       - run: nvm install 16.13
       - run: nvm use 16.13
-      - run: npm install -g yarn@1.22.10
+      - run: npm install -g yarn@1.22.10 @bazel/bazelisk@${BAZELISK_VERSION}
       - run: node --version
       - run: yarn --version
 
@@ -126,6 +138,7 @@ commands:
         type: env_var_name
         default: CIRCLE_PROJECT_REPONAME
     steps:
+      - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
       - devinfra/setup-bazel-remote-exec:
           bazelrc: ./.bazelrc.user
 
@@ -269,23 +282,24 @@ jobs:
           paths:
             - dist/_*.tgz
 
-  build-bazel-e2e:
-    executor: action-executor
-    resource_class: medium
+  bazel-build:
+    executor: bazel-executor
     steps:
       - custom_attach_workspace
-      - run: yarn bazel build //tests/legacy-cli/...
+      - setup_bazel_rbe
+      - run:
+          name: Bazel Build Packages
+          command: yarn bazel build //...
+      - fail_fast
 
-  unit-test:
-    executor: action-executor
-    resource_class: xlarge
+  bazel-test:
+    executor: bazel-executor
     parameters:
       nodeversion:
         type: string
         default: *default_nodeversion_major
     steps:
       - custom_attach_workspace
-      - browser-tools/install-chrome
       - setup_bazel_rbe
       - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
       - when:
@@ -311,6 +325,59 @@ jobs:
                 no_output_timeout: 40m
       - fail_fast
 
+  bazel-e2e-tests:
+    executor: bazel-executor
+    parallelism: 8
+    parameters:
+      build_type:
+        type: enum
+        enum: *all_e2e_build_types
+        default: 'e2e'
+      subset:
+        type: enum
+        enum: *all_e2e_subsets
+        default: 'npm'
+    steps:
+      - custom_attach_workspace
+      - initialize_env
+      - setup_bazel_rbe
+      - run: mkdir /mnt/ramdisk/e2e
+      - run:
+          name: Test << parameters.build_type >> << parameters.subset >>
+          command: yarn bazel test --define=E2E_TEMP=/mnt/ramdisk/e2e --define=E2E_SHARD_TOTAL=${CIRCLE_NODE_TOTAL} --define=E2E_SHARD_INDEX=${CIRCLE_NODE_INDEX} --config=<< parameters.build_type >> //tests/legacy-cli:e2e.<< parameters.subset >>
+          no_output_timeout: 40m
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - fail_fast
+
+  bazel-test-browsers:
+    executor: bazel-executor
+    steps:
+      - custom_attach_workspace
+      - initialize_env
+      - setup_bazel_rbe
+      - run:
+          name: Initialize Saucelabs
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Start Saucelabs Tunnel
+          command: ./scripts/saucelabs/start-tunnel.sh
+          background: true
+        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
+        # too early without Saucelabs not being ready.
+      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+      - run:
+          name: E2E Saucelabs Tests
+          command: yarn bazel test --config=saucelabs //tests/legacy-cli:e2e.saucelabs
+      - run: ./scripts/saucelabs/stop-tunnel.sh
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - fail_fast
+
   snapshot_publish:
     executor: action-executor
     resource_class: medium
@@ -382,6 +449,48 @@ jobs:
             node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main --ignore="tests/misc/browsers.ts"
       - fail_fast
 
+  bazel-e2e-cli-win:
+    executor: windows-executor
+    parallelism: 12
+    steps:
+      - checkout
+      - rebase_pr_win
+      - setup_windows
+      - restore_cache:
+          keys:
+            - *cache_key_win
+      - run:
+          # We use Arsenal Image Mounter (AIM) instead of ImDisk because of: https://github.com/nodejs/node/issues/6861
+          # Useful resources for AIM: http://reboot.pro/index.php?showtopic=22068
+          name: 'Arsenal Image Mounter (RAM Disk)'
+          command: |
+            pwsh ./.circleci/win-ram-disk.ps1
+      - run: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+      - save_cache:
+          key: *cache_key_win
+          paths:
+            - ~/.cache/yarn
+            # Path where Arsenal Image Mounter files are downloaded.
+            # Must match path in .circleci/win-ram-disk.ps1
+            - ./aim
+      - run:
+          name: Execute E2E Tests
+          environment:
+            # Required by `yarn ng-dev`
+            # See https://github.com/angular/angular/issues/46858
+            PWD: .
+          command: |
+            mkdir X:/ramdisk/e2e
+            bazel test --define=E2E_TEMP=X:/ramdisk/e2e --define=E2E_SHARD_TOTAL=$env:CIRCLE_NODE_TOTAL --define=E2E_SHARD_INDEX=$env:CIRCLE_NODE_INDEX --config=e2e //tests/legacy-cli:e2e.npm
+          # This timeout provides time for the actual tests to timeout and report status
+          # instead of CircleCI stopping the job without test failure information.
+          no_output_timeout: 40m
+      - fail_fast
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.npm
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.npm
+
 workflows:
   version: 2
   default_workflow:
@@ -457,23 +566,67 @@ workflows:
       # These jobs only really depend on Setup, but the build job is very quick to run (~35s) and
       # will catch any build errors before proceeding to the more lengthy and resource intensive
       # Bazel jobs.
-      - unit-test:
+      - bazel-test:
           name: test-node<< matrix.nodeversion >>
           matrix:
             parameters:
               nodeversion: *all_nodeversion_major
           requires:
-            - build
-
-      # Compile the e2e tests with bazel to ensure the non-runtime typescript
-      # compilation completes succesfully.
-      - build-bazel-e2e:
-          requires:
-            - build
+            - bazel-build
 
       # Windows jobs
       - e2e-cli-win
 
+      - bazel-e2e-cli-win
+
+      # Bazel jobs
+      - bazel-build:
+          requires:
+            - setup
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-cli-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          build_type: 'e2e'
+          filters:
+            branches:
+              ignore:
+                - main
+                - /\d+\.\d+\.x/
+          requires:
+            - bazel-build
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-snapshots-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          build_type: 'snapshot'
+          pre-steps:
+            - when:
+                condition:
+                  and:
+                    - not:
+                        equal: [main, << pipeline.git.branch >>]
+                    - not: << pipeline.parameters.snapshot_changed >>
+                steps:
+                  # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated.
+                  - run: circleci-agent step halt
+          requires:
+            - bazel-build
+          filters:
+            branches:
+              only:
+                - main
+                # This is needed to run this steps on Renovate PRs that amend the snapshots package.json
+                - /^pull\/.*/
+
+      - bazel-test-browsers:
+          requires:
+            - bazel-build
+
       # Publish jobs
       - snapshot_publish:
           <<: *only_release_branches

.circleci/env.sh

@@ -36,3 +36,8 @@ source $BASH_ENV;
 
 # Disable husky.
 setPublicVar HUSKY 0
+
+# Expose the Bazelisk version. We need to run Bazelisk globally since Windows has problems launching
+# Bazel from a node modules directoy that might be modified by the Bazel Yarn install then.
+setPublicVar BAZELISK_VERSION \
+    "$(cd ${PROJECT_ROOT}; node -p 'require("./package.json").devDependencies["@bazel/bazelisk"]')"

.circleci/win-ram-disk.ps1

@@ -26,5 +26,6 @@ if (-not (Test-Path -Path $aimContents)) {
 ./aim/cli/x64/aim_ll.exe --install ./aim/drivers
 
 # Setup RAM disk mount. Same parameters as ImDisk
+# Ensure size is large enough to support the bazel 'shard_count's such as for e2e tests.
 # See: https://support.circleci.com/hc/en-us/articles/4411520952091-Create-a-windows-RAM-disk
-./aim/cli/x64/aim_ll.exe -a -s 5G -m X: -p "/fs:ntfs /q /y"
+./aim/cli/x64/aim_ll.exe -a -s 12G -m X: -p "/fs:ntfs /q /y"

BUILD.bazel

@@ -3,6 +3,7 @@
 # Use of this source code is governed by an MIT-style license that can be
 # found in the LICENSE file at https://angular.io/license
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -16,6 +17,14 @@ exports_files([
     "package.json",
 ])
 
+# Files required by e2e tests
+copy_to_bin(
+    name = "config-files",
+    srcs = [
+        "package.json",
+    ],
+)
+
 # Detect if the build is running under --stamp
 config_setting(
     name = "stamp",

WORKSPACE

@@ -98,3 +98,14 @@ nodejs_register_toolchains(
     name = "node16",
     node_version = "16.13.1",
 )
+
+register_toolchains(
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_linux_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_x86_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain",
+)
+
+load("@npm//@angular/build-tooling/bazel/browsers:browser_repositories.bzl", "browser_repositories")
+
+browser_repositories()

docs/DEVELOPER.md

@@ -85,7 +85,7 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 - Compile the packages being tested: `yarn build`
 - Run all tests: `node tests/legacy-cli/run_e2e.js`
 - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
-- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...`
+- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.
 You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a

packages/angular/cli/src/utilities/version.ts

@@ -23,7 +23,7 @@ class Version {
   }
 }
 
-// TODO: Convert this to use build-time version stamping after flipping the build script to use bazel
+// TODO(bazel): Convert this to use build-time version stamping after flipping the build script to use bazel
 // export const VERSION = new Version('0.0.0-PLACEHOLDER');
 export const VERSION = new Version(
   (

tests/legacy-cli/BUILD.bazel

@@ -1,4 +1,5 @@
 load("//tools:defaults.bzl", "ts_library")
+load(":e2e.bzl", "e2e_suites")
 
 ts_library(
     name = "runner",
@@ -11,16 +12,25 @@ ts_library(
     deps = [
         "//packages/angular_devkit/core",
         "//packages/angular_devkit/core/node",
-        "//tests/legacy-cli/e2e/assets",
         "//tests/legacy-cli/e2e/utils",
         "@npm//@types/glob",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//yargs-parser",
+    ],
+)
 
+e2e_suites(
+    name = "e2e",
+    data = [
+        ":runner",
+
+        # Tests + setup
         # Loaded dynamically at runtime, not compiletime deps
+        "//tests/legacy-cli/e2e/assets",
         "//tests/legacy-cli/e2e/setup",
         "//tests/legacy-cli/e2e/initialize",
         "//tests/legacy-cli/e2e/tests",
     ],
+    runner = ":e2e_runner.ts",
 )