Auto merge of #29791 - sagudev:webgpu-cts, r=jdm

Vendoring machanism for webgpu cts & update

- Add `update-webgpu` command to mach to vendor webgpu cts.
- Update webgpu cts (480edec387)  & expectations (even though most are failing due to out of date impl)
- Ignore vendored path from tidy check

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes fix #27508

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because they are tests

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

python/servo/testing_commands.py

@@ -40,6 +40,9 @@ from servo.command_base import (
 )
 from servo_tidy_tests import test_tidy
 
+from servo.util import delete
+from distutils.dir_util import copy_tree
+
 SCRIPT_PATH = os.path.split(__file__)[0]
 PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
 WEB_PLATFORM_TESTS_PATH = os.path.join("tests", "wpt", "web-platform-tests")
@@ -820,6 +823,54 @@ testing/web-platform/mozilla/tests for Servo-only tests""" % reference_path)
         exec(compile(open(run_file).read(), run_file, 'exec'), run_globals)
         return run_globals["update_conformance"](version, dest_folder, None, patches_dir)
 
+    @Command('update-webgpu',
+             description='Update the WebGPU conformance test suite',
+             category='testing')
+    @CommandArgument(
+        '--repo', '-r', default="https://github.com/gpuweb/cts",
+        help='Repo to vendor cts from')
+    @CommandArgument(
+        '--checkout', '-c', default="main",
+        help='Branch or commit of repo')
+    def cts(self, repo="https://github.com/gpuweb/cts", checkout="main"):
+        tdir = path.join(self.context.topdir, "tests/wpt/webgpu/tests")
+        clone_dir = path.join(tdir, "cts_clone")
+        # clone
+        res = call(["git", "clone", "-n", repo, "cts_clone"], cwd=tdir)
+        if res != 0:
+            return res
+        # checkout
+        res = call(["git", "checkout", checkout], cwd=clone_dir)
+        if res != 0:
+            return res
+        # build
+        res = call(["npm", "ci"], cwd=clone_dir)
+        if res != 0:
+            return res
+        res = call(["npm", "run", "wpt"], cwd=clone_dir)
+        if res != 0:
+            return res
+        cts_html = path.join(clone_dir, "out-wpt", "cts.https.html")
+        # patch
+        with open(cts_html, 'r') as file:
+            filedata = file.read()
+        # files are mounted differently
+        filedata = filedata.replace('src=/webgpu/common/runtime/wpt.js', 'src=../webgpu/common/runtime/wpt.js')
+        # Write the file out again
+        with open(cts_html, 'w') as file:
+            file.write(filedata)
+        # copy
+        delete(path.join(tdir, "webgpu"))
+        copy_tree(path.join(clone_dir, "out-wpt"), path.join(tdir, "webgpu"))
+        # update commit
+        commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+        with open(path.join(tdir, "checkout_commit.txt"), 'w') as file:
+            file.write(commit)
+        # clean up
+        delete(clone_dir)
+        print("Updating manifest.")
+        return self.context.commands.dispatch("update-manifest", self.context)
+
     @Command('smoketest',
              description='Load a simple page in Servo and ensure that it closes properly',
              category='testing')

servo-tidy.toml

@@ -109,6 +109,8 @@ files = [
   "./support/android/openssl.sh",
   # Upstream code from Khronos/WebGL uses tabs for indentation
   "./tests/wpt/webgl/tests",
+  # Vendored from upstream
+  "./tests/wpt/webgpu/tests",
   # Our import script is not currently respecting the lint.
   "./tests/wpt/webgl/tools/import-conformance-tests.py",
   # Ignore those files since the issues reported are on purpose

tests/wpt/webgpu/meta/webgpu/webgpu/idl/exposed.http.html.ini

@@ -0,0 +1,2 @@
+[exposed.http.html]
+  expected: ERROR

tests/wpt/webgpu/meta/webgpu/webgpu/idl/exposed.https.html.ini

@@ -0,0 +1,2 @@
+[exposed.https.html]
+  expected: ERROR

tests/wpt/webgpu/meta/webgpu/webgpu/web-platform/reftests/canvas_clear.html.ini

@@ -1,2 +0,0 @@
-[canvas_clear.html]
-  expected: FAIL

tests/wpt/webgpu/meta/webgpu/webgpu/web-platform/reftests/canvas_complex_bgra8unorm.html.ini

@@ -1,2 +0,0 @@
-[canvas_complex_bgra8unorm.html]
-  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_clear.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_clear.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_bgra8unorm.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_colorspace_bgra8unorm.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba16float.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_colorspace_rgba16float.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba8unorm.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_colorspace_rgba8unorm.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_bgra8unorm_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_bgra8unorm_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_bgra8unorm_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_bgra8unorm_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba16float_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba16float_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba16float_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba16float_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba16float_store.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba16float_store.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba8unorm_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba8unorm_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba8unorm_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba8unorm_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_complex_rgba8unorm_store.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_complex_rgba8unorm_store.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_bgra8unorm_opaque_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_bgra8unorm_opaque_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_bgra8unorm_opaque_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_bgra8unorm_opaque_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_bgra8unorm_premultiplied_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_bgra8unorm_premultiplied_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_bgra8unorm_premultiplied_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_bgra8unorm_premultiplied_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba16float_opaque_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba16float_opaque_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba16float_opaque_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba16float_opaque_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba16float_premultiplied_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba16float_premultiplied_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba16float_premultiplied_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba16float_premultiplied_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba8unorm_opaque_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba8unorm_opaque_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba8unorm_opaque_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba8unorm_opaque_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba8unorm_premultiplied_copy.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba8unorm_premultiplied_copy.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_composite_alpha_rgba8unorm_premultiplied_draw.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_composite_alpha_rgba8unorm_premultiplied_draw.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_image_rendering.https.html.ini

@@ -0,0 +1,2 @@
+[canvas_image_rendering.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/resize_observer.https.html.ini

@@ -0,0 +1,2 @@
+[resize_observer.https.html]
+  expected: TIMEOUT

tests/wpt/webgpu/tests/checkout_commit.txt

@@ -0,0 +1 @@
+9ab2eade6a818ed58ac1a7b36b706858f3ba5eb3

tests/wpt/webgpu/tests/webgpu/common/framework/data_cache.js

@@ -0,0 +1,89 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+/** DataCache is an interface to a data store used to hold cached data */
+export class DataCache {
+  /** setDataStore() sets the backing data store used by the data cache */
+  setStore(dataStore) {
+    this.dataStore = dataStore;
+  }
+
+  /** setDebugLogger() sets the verbose logger */
+  setDebugLogger(logger) {
+    this.debugLogger = logger;
+  }
+
+  /**
+   * fetch() retrieves cacheable data from the data cache, first checking the
+   * in-memory cache, then the data store (if specified), then resorting to
+   * building the data and storing it in the cache.
+   */
+  async fetch(cacheable) {
+    // First check the in-memory cache
+    let data = this.cache.get(cacheable.path);
+    if (data !== undefined) {
+      this.log('in-memory cache hit');
+      return Promise.resolve(data);
+    }
+    this.log('in-memory cache miss');
+    // In in-memory cache miss.
+    // Next, try the data store.
+    if (this.dataStore !== null && !this.unavailableFiles.has(cacheable.path)) {
+      let serialized;
+      try {
+        serialized = await this.dataStore.load(cacheable.path);
+        this.log('loaded serialized');
+      } catch (err) {
+        // not found in data store
+        this.log(`failed to load (${cacheable.path}): ${err}`);
+        this.unavailableFiles.add(cacheable.path);
+      }
+      if (serialized !== undefined) {
+        this.log(`deserializing`);
+        data = cacheable.deserialize(serialized);
+        this.cache.set(cacheable.path, data);
+        return data;
+      }
+    }
+    // Not found anywhere. Build the data, and cache for future lookup.
+    this.log(`cache: building (${cacheable.path})`);
+    data = await cacheable.build();
+    this.cache.set(cacheable.path, data);
+    return data;
+  }
+
+  log(msg) {
+    if (this.debugLogger !== null) {
+      this.debugLogger(`DataCache: ${msg}`);
+    }
+  }
+
+  cache = new Map();
+  unavailableFiles = new Set();
+  dataStore = null;
+  debugLogger = null;
+}
+
+/** The data cache */
+export const dataCache = new DataCache();
+
+/** true if the current process is building the cache */
+let isBuildingDataCache = false;
+
+/** @returns true if the data cache is currently being built */
+export function getIsBuildingDataCache() {
+  return isBuildingDataCache;
+}
+
+/** Sets whether the data cache is currently being built */
+export function setIsBuildingDataCache(value = true) {
+  isBuildingDataCache = value;
+}
+
+/**
+ * Cacheable is the interface to something that can be stored into the
+ * DataCache.
+ * The 'npm run gen_cache' tool will look for module-scope variables of this
+ * interface, with the name `d`.
+ */

tests/wpt/webgpu/tests/webgpu/common/framework/fixture.js

@@ -1,64 +1,178 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { assert } from './util/util.js';
+ **/ import { assert, unreachable } from '../util/util.js';
 
 export class SkipTestCase extends Error {}
+export class UnexpectedPassError extends Error {}
 
-// A Fixture is a class used to instantiate each test case at run time.
-// A new instance of the Fixture is created for every single test case
-// (i.e. every time the test function is run).
-export class Fixture {
-  constructor(rec, params) {
-    _defineProperty(this, 'params', void 0);
-    _defineProperty(this, 'rec', void 0);
-    _defineProperty(this, 'eventualExpectations', []);
-    _defineProperty(this, 'numOutstandingAsyncExpectations', 0);
-    this.rec = rec;
+export { TestCaseRecorder } from '../internal/logging/test_case_recorder.js';
+
+/** The fully-general type for params passed to a test function invocation. */
+
+export class SubcaseBatchState {
+  constructor(
+    recorder,
+    /** The case parameters for this test fixture shared state. Subcase params are not included. */
+    params
+  ) {
+    this.recorder = recorder;
     this.params = params;
   }
 
-  // This has to be a member function instead of an async `createFixture` function, because
-  // we need to be able to ergonomically override it in subclasses.
+  /**
+   * Runs before the `.before()` function.
+   * @internal MAINTENANCE_TODO: Make this not visible to test code?
+   */
+  async init() {}
+  /**
+   * Runs between the `.before()` function and the subcases.
+   * @internal MAINTENANCE_TODO: Make this not visible to test code?
+   */
+  async postInit() {}
+  /**
+   * Runs after all subcases finish.
+   * @internal MAINTENANCE_TODO: Make this not visible to test code?
+   */
+  async finalize() {}
+}
+
+/**
+ * A Fixture is a class used to instantiate each test sub/case at run time.
+ * A new instance of the Fixture is created for every single test subcase
+ * (i.e. every time the test function is run).
+ */
+export class Fixture {
+  /**
+   * Interface for recording logs and test status.
+   *
+   * @internal
+   */
+
+  eventualExpectations = [];
+  numOutstandingAsyncExpectations = 0;
+  objectsToCleanUp = [];
+
+  static MakeSharedState(recorder, params) {
+    return new SubcaseBatchState(recorder, params);
+  }
+
+  /** @internal */
+  constructor(sharedState, rec, params) {
+    this._sharedState = sharedState;
+    this.rec = rec;
+    this._params = params;
+  }
+
+  /**
+   * Returns the (case+subcase) parameters for this test function invocation.
+   */
+  get params() {
+    return this._params;
+  }
+
+  /**
+   * Gets the test fixture's shared state. This object is shared between subcases
+   * within the same testcase.
+   */
+  get sharedState() {
+    return this._sharedState;
+  }
+
+  /**
+   * Override this to do additional pre-test-function work in a derived fixture.
+   * This has to be a member function instead of an async `createFixture` function, because
+   * we need to be able to ergonomically override it in subclasses.
+   *
+   * @internal MAINTENANCE_TODO: Make this not visible to test code?
+   */
   async init() {}
 
+  /**
+   * Override this to do additional post-test-function work in a derived fixture.
+   *
+   * Called even if init was unsuccessful.
+   *
+   * @internal MAINTENANCE_TODO: Make this not visible to test code?
+   */
+  async finalize() {
+    assert(
+      this.numOutstandingAsyncExpectations === 0,
+      'there were outstanding immediateAsyncExpectations (e.g. expectUncapturedError) at the end of the test'
+    );
+
+    // Loop to exhaust the eventualExpectations in case they chain off each other.
+    while (this.eventualExpectations.length) {
+      const p = this.eventualExpectations.shift();
+      try {
+        await p;
+      } catch (ex) {
+        this.rec.threw(ex);
+      }
+    }
+
+    // And clean up any objects now that they're done being used.
+    for (const o of this.objectsToCleanUp) {
+      if ('getExtension' in o) {
+        const WEBGL_lose_context = o.getExtension('WEBGL_lose_context');
+        if (WEBGL_lose_context) WEBGL_lose_context.loseContext();
+      } else if ('destroy' in o) {
+        o.destroy();
+      } else {
+        o.close();
+      }
+    }
+  }
+
+  /**
+   * Tracks an object to be cleaned up after the test finishes.
+   *
+   * MAINTENANCE_TODO: Use this in more places. (Will be easier once .destroy() is allowed on
+   * invalid objects.)
+   */
+  trackForCleanup(o) {
+    this.objectsToCleanUp.push(o);
+    return o;
+  }
+
+  /** Tracks an object, if it's destroyable, to be cleaned up after the test finishes. */
+  tryTrackForCleanup(o) {
+    if (typeof o === 'object' && o !== null) {
+      if (
+        'destroy' in o ||
+        'close' in o ||
+        o instanceof WebGLRenderingContext ||
+        o instanceof WebGL2RenderingContext
+      ) {
+        this.objectsToCleanUp.push(o);
+      }
+    }
+    return o;
+  }
+
+  /** Log a debug message. */
   debug(msg) {
     this.rec.debug(new Error(msg));
   }
 
+  /** Throws an exception marking the subcase as skipped. */
   skip(msg) {
     throw new SkipTestCase(msg);
   }
 
-  async finalize() {
-    assert(
-      this.numOutstandingAsyncExpectations === 0,
-      'there were outstanding asynchronous expectations (e.g. shouldReject) at the end of the test'
-    );
-
-    await Promise.all(this.eventualExpectations);
-  }
-
+  /** Log a warning and increase the result status to "Warn". */
   warn(msg) {
     this.rec.warn(new Error(msg));
   }
 
+  /** Log an error and increase the result status to "ExpectFailed". */
   fail(msg) {
     this.rec.expectationFailed(new Error(msg));
   }
 
+  /**
+   * Wraps an async function. Tracks its status to fail if the test tries to report a test status
+   * before the async work has finished.
+   */
   async immediateAsyncExpectation(fn) {
     this.numOutstandingAsyncExpectations++;
     const ret = await fn();
@@ -66,28 +180,32 @@ export class Fixture {
     return ret;
   }
 
+  /**
+   * Wraps an async function, passing it an `Error` object recording the original stack trace.
+   * The async work will be implicitly waited upon before reporting a test status.
+   */
   eventualAsyncExpectation(fn) {
     const promise = fn(new Error());
     this.eventualExpectations.push(promise);
-    return promise;
   }
 
-  expectErrorValue(expectedName, ex, niceStack) {
+  expectErrorValue(expectedError, ex, niceStack) {
     if (!(ex instanceof Error)) {
       niceStack.message = `THREW non-error value, of type ${typeof ex}: ${ex}`;
       this.rec.expectationFailed(niceStack);
       return;
     }
     const actualName = ex.name;
-    if (actualName !== expectedName) {
-      niceStack.message = `THREW ${actualName}, instead of ${expectedName}: ${ex}`;
+    if (expectedError !== true && actualName !== expectedError) {
+      niceStack.message = `THREW ${actualName}, instead of ${expectedError}: ${ex}`;
       this.rec.expectationFailed(niceStack);
     } else {
-      niceStack.message = `OK: threw ${actualName}${ex.message}`;
+      niceStack.message = `OK: threw ${actualName}: ${ex.message}`;
       this.rec.debug(niceStack);
     }
   }
 
+  /** Expect that the provided promise resolves (fulfills). */
   shouldResolve(p, msg) {
     this.eventualAsyncExpectation(async niceStack => {
       const m = msg ? ': ' + msg : '';
@@ -95,12 +213,16 @@ export class Fixture {
         await p;
         niceStack.message = 'resolved as expected' + m;
       } catch (ex) {
-        niceStack.message = `REJECTED${m}\n${ex.message}`;
+        niceStack.message = `REJECTED${m}`;
+        if (ex instanceof Error) {
+          niceStack.message += '\n' + ex.message;
+        }
         this.rec.expectationFailed(niceStack);
       }
     });
   }
 
+  /** Expect that the provided promise rejects, with the provided exception name. */
   shouldReject(expectedName, p, msg) {
     this.eventualAsyncExpectation(async niceStack => {
       const m = msg ? ': ' + msg : '';
@@ -115,16 +237,31 @@ export class Fixture {
     });
   }
 
-  shouldThrow(expectedName, fn, msg) {
+  /**
+   * Expect that the provided function throws (if `true` or `string`) or not (if `false`).
+   * If a string is provided, expect that the throw exception has that name.
+   *
+   * MAINTENANCE_TODO: Change to `string | false` so the exception name is always checked.
+   */
+  shouldThrow(expectedError, fn, msg) {
     const m = msg ? ': ' + msg : '';
     try {
       fn();
-      this.rec.expectationFailed(new Error('DID NOT THROW' + m));
+      if (expectedError === false) {
+        this.rec.debug(new Error('did not throw, as expected' + m));
+      } else {
+        this.rec.expectationFailed(new Error('unexpectedly did not throw' + m));
+      }
     } catch (ex) {
-      this.expectErrorValue(expectedName, ex, new Error(m));
+      if (expectedError === false) {
+        this.rec.expectationFailed(new Error('threw unexpectedly' + m));
+      } else {
+        this.expectErrorValue(expectedError, ex, new Error(m));
+      }
     }
   }
 
+  /** Expect that a condition is true. */
   expect(cond, msg) {
     if (cond) {
       const m = msg ? ': ' + msg : '';
@@ -134,4 +271,39 @@ export class Fixture {
     }
     return cond;
   }
+
+  /**
+   * If the argument is an `Error`, fail (or warn). If it's `undefined`, no-op.
+   * If the argument is an array, apply the above behavior on each of elements.
+   */
+  expectOK(error, { mode = 'fail', niceStack } = {}) {
+    const handleError = error => {
+      if (error instanceof Error) {
+        if (niceStack) {
+          error.stack = niceStack.stack;
+        }
+        if (mode === 'fail') {
+          this.rec.expectationFailed(error);
+        } else if (mode === 'warn') {
+          this.rec.warn(error);
+        } else {
+          unreachable();
+        }
+      }
+    };
+
+    if (Array.isArray(error)) {
+      for (const e of error) {
+        handleError(e);
+      }
+    } else {
+      handleError(error);
+    }
+  }
+
+  eventualExpectOK(error, { mode = 'fail' } = {}) {
+    this.eventualAsyncExpectation(async niceStack => {
+      this.expectOK(await error, { mode, niceStack });
+    });
+  }
 }

tests/wpt/webgpu/tests/webgpu/common/framework/gpu/device_pool.js

@@ -1,155 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { assert, raceWithRejectOnTimeout, unreachable, assertReject } from '../util/util.js';
-import { getGPU } from './implementation.js';
-
-class TestFailedButDeviceReusable extends Error {}
-export class TestOOMedShouldAttemptGC extends Error {}
-
-const kPopErrorScopeTimeoutMS = 5000;
-
-export class DevicePool {
-  constructor() {
-    _defineProperty(this, 'failed', false);
-    _defineProperty(this, 'holder', undefined);
-  } // undefined if "uninitialized" (not yet initialized, or lost)
-
-  async acquire() {
-    assert(!this.failed, 'WebGPU device previously failed to initialize; not retrying');
-
-    if (this.holder === undefined) {
-      try {
-        this.holder = await DevicePool.makeHolder();
-      } catch (ex) {
-        this.failed = true;
-        throw ex;
-      }
-    }
-    assert(!this.holder.acquired, 'Device was in use on DevicePool.acquire');
-    this.holder.acquired = true;
-
-    this.beginErrorScopes();
-    return this.holder.device;
-  }
-
-  // When a test is done using a device, it's released back into the pool.
-  // This waits for error scopes, checks their results, and checks for various error conditions.
-  async release(device) {
-    const holder = this.holder;
-    assert(holder !== undefined, 'trying to release a device while pool is uninitialized');
-    assert(holder.acquired, 'trying to release a device while already released');
-    assert(device === holder.device, 'Released device was the wrong device');
-
-    try {
-      // Time out if popErrorScope never completes. This could happen due to a browser bug - e.g.,
-      // as of this writing, on Chrome GPU process crash, popErrorScope just hangs.
-      await raceWithRejectOnTimeout(
-        this.endErrorScopes(),
-        kPopErrorScopeTimeoutMS,
-        'finalization popErrorScope timed out'
-      );
-
-      // (Hopefully if the device was lost, it has been reported by the time endErrorScopes()
-      // has finished (or timed out). If not, it could cause a finite number of extra test
-      // failures following this one (but should recover eventually).)
-      const lostReason = holder.lostReason;
-      if (lostReason !== undefined) {
-        // Fail the current test.
-        unreachable(`Device was lost: ${lostReason}`);
-      }
-    } catch (ex) {
-      // Any error that isn't explicitly TestFailedButDeviceReusable forces a new device to be
-      // created for the next test.
-      if (!(ex instanceof TestFailedButDeviceReusable)) {
-        this.holder = undefined;
-      }
-      throw ex;
-    } finally {
-      // TODO: device.destroy()
-
-      // Mark the holder as free. (This only has an effect if the pool still has the holder.)
-      // This could be done at the top but is done here to guard against async-races during release.
-      holder.acquired = false;
-    }
-  }
-
-  // Gets a device and creates a DeviceHolder.
-  // If the device is lost, DeviceHolder.lostReason gets set.
-  static async makeHolder() {
-    const gpu = getGPU();
-    const adapter = await gpu.requestAdapter();
-    assert(adapter !== null);
-    const device = await adapter.requestDevice();
-    assert(device !== null);
-
-    const holder = {
-      acquired: false,
-      device,
-      lostReason: undefined,
-    };
-
-    holder.device.lost.then(ev => {
-      holder.lostReason = ev.message;
-    });
-    return holder;
-  }
-
-  // Create error scopes that wrap the entire test.
-  beginErrorScopes() {
-    assert(this.holder !== undefined);
-    this.holder.device.pushErrorScope('out-of-memory');
-    this.holder.device.pushErrorScope('validation');
-  }
-
-  // End the whole-test error scopes. Check that there are no extra error scopes, and that no
-  // otherwise-uncaptured errors occurred during the test.
-  async endErrorScopes() {
-    assert(this.holder !== undefined);
-    let gpuValidationError;
-    let gpuOutOfMemoryError;
-
-    try {
-      // May reject if the device was lost.
-      gpuValidationError = await this.holder.device.popErrorScope();
-      gpuOutOfMemoryError = await this.holder.device.popErrorScope();
-    } catch (ex) {
-      assert(
-        this.holder.lostReason !== undefined,
-        "popErrorScope failed, but device.lost hasn't fired (yet)"
-      );
-
-      throw ex;
-    }
-
-    await assertReject(
-      this.holder.device.popErrorScope(),
-      'There was an extra error scope on the stack after a test'
-    );
-
-    if (gpuValidationError !== null) {
-      assert(gpuValidationError instanceof GPUValidationError);
-      // Allow the device to be reused.
-      throw new TestFailedButDeviceReusable(
-        `Unexpected validation error occurred: ${gpuValidationError.message}`
-      );
-    }
-    if (gpuOutOfMemoryError !== null) {
-      assert(gpuOutOfMemoryError instanceof GPUOutOfMemoryError);
-      // Don't allow the device to be reused; unexpected OOM could break the device.
-      throw new TestOOMedShouldAttemptGC('Unexpected out-of-memory error occurred');
-    }
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/gpu/implementation.js

@@ -1,19 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ /// <reference types="@webgpu/types" />
-import { assert } from '../util/util.js';
-let impl = undefined;
-
-export function getGPU() {
-  if (impl) {
-    return impl;
-  }
-
-  assert(
-    typeof navigator !== 'undefined' && navigator.gpu !== undefined,
-    'No WebGPU implementation found'
-  );
-
-  impl = navigator.gpu;
-  return impl;
-}

tests/wpt/webgpu/tests/webgpu/common/framework/logging/log_message.js

@@ -1,48 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { extractImportantStackTrace } from '../util/stack.js';
-export class LogMessageWithStack extends Error {
-  constructor(name, ex) {
-    super(ex.message);
-    _defineProperty(this, 'stackHidden', false);
-    _defineProperty(this, 'timesSeen', 1);
-
-    this.name = name;
-    this.stack = ex.stack;
-  }
-
-  /** Set a flag so the stack is not printed in toJSON(). */
-  setStackHidden() {
-    this.stackHidden = true;
-  }
-
-  /** Increment the "seen x times" counter. */
-  incrementTimesSeen() {
-    this.timesSeen++;
-  }
-
-  toJSON() {
-    let m = this.name;
-    if (this.message) m += ': ' + this.message;
-    if (!this.stackHidden && this.stack) {
-      m += '\n' + extractImportantStackTrace(this);
-    }
-    if (this.timesSeen > 1) {
-      m += `\n(seen ${this.timesSeen} times with identical stack)`;
-    }
-    return m;
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/logging/logger.js

@@ -1,35 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { version } from '../version.js';
-import { TestCaseRecorder } from './test_case_recorder.js';
-
-export class Logger {
-  constructor(debug) {
-    _defineProperty(this, 'debug', void 0);
-    _defineProperty(this, 'results', new Map());
-    this.debug = debug;
-  }
-
-  record(name) {
-    const result = { status: 'running', timems: -1 };
-    this.results.set(name, result);
-    return [new TestCaseRecorder(result, this.debug), result];
-  }
-
-  asJSON(space) {
-    return JSON.stringify({ version, results: Array.from(this.results) }, undefined, space);
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/logging/test_case_recorder.js

@@ -1,139 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { SkipTestCase } from '../fixture.js';
-import { now, assert } from '../util/util.js';
-import { LogMessageWithStack } from './log_message.js';
-var LogSeverity;
-(function (LogSeverity) {
-  LogSeverity[(LogSeverity['Pass'] = 0)] = 'Pass';
-  LogSeverity[(LogSeverity['Skip'] = 1)] = 'Skip';
-  LogSeverity[(LogSeverity['Warn'] = 2)] = 'Warn';
-  LogSeverity[(LogSeverity['ExpectFailed'] = 3)] = 'ExpectFailed';
-  LogSeverity[(LogSeverity['ValidationFailed'] = 4)] = 'ValidationFailed';
-  LogSeverity[(LogSeverity['ThrewException'] = 5)] = 'ThrewException';
-})(LogSeverity || (LogSeverity = {}));
-
-const kMaxLogStacks = 2;
-
-/** Holds onto a LiveTestCaseResult owned by the Logger, and writes the results into it. */
-export class TestCaseRecorder {
-  /** Used to dedup log messages which have identical stacks. */
-
-  constructor(result, debugging) {
-    _defineProperty(this, 'result', void 0);
-    _defineProperty(this, 'maxLogSeverity', LogSeverity.Pass);
-    _defineProperty(this, 'startTime', -1);
-    _defineProperty(this, 'logs', []);
-    _defineProperty(this, 'logLinesAtCurrentSeverity', 0);
-    _defineProperty(this, 'debugging', false);
-    _defineProperty(this, 'messagesForPreviouslySeenStacks', new Map());
-    this.result = result;
-    this.debugging = debugging;
-  }
-
-  start() {
-    assert(this.startTime < 0, 'TestCaseRecorder cannot be reused');
-    this.startTime = now();
-  }
-
-  finish() {
-    assert(this.startTime >= 0, 'finish() before start()');
-
-    const timeMilliseconds = now() - this.startTime;
-    // Round to next microsecond to avoid storing useless .xxxx00000000000002 in results.
-    this.result.timems = Math.ceil(timeMilliseconds * 1000) / 1000;
-
-    // Convert numeric enum back to string (but expose 'exception' as 'fail')
-    this.result.status =
-      this.maxLogSeverity === LogSeverity.Pass
-        ? 'pass'
-        : this.maxLogSeverity === LogSeverity.Skip
-        ? 'skip'
-        : this.maxLogSeverity === LogSeverity.Warn
-        ? 'warn'
-        : 'fail'; // Everything else is an error
-
-    this.result.logs = this.logs;
-  }
-
-  injectResult(injectedResult) {
-    Object.assign(this.result, injectedResult);
-  }
-
-  debug(ex) {
-    if (!this.debugging) {
-      return;
-    }
-    const logMessage = new LogMessageWithStack('DEBUG', ex);
-    logMessage.setStackHidden();
-    this.logImpl(LogSeverity.Pass, logMessage);
-  }
-
-  skipped(ex) {
-    this.logImpl(LogSeverity.Skip, new LogMessageWithStack('SKIP', ex));
-  }
-
-  warn(ex) {
-    this.logImpl(LogSeverity.Warn, new LogMessageWithStack('WARN', ex));
-  }
-
-  expectationFailed(ex) {
-    this.logImpl(LogSeverity.ExpectFailed, new LogMessageWithStack('EXPECTATION FAILED', ex));
-  }
-
-  validationFailed(ex) {
-    this.logImpl(LogSeverity.ValidationFailed, new LogMessageWithStack('VALIDATION FAILED', ex));
-  }
-
-  threw(ex) {
-    if (ex instanceof SkipTestCase) {
-      this.skipped(ex);
-      return;
-    }
-    this.logImpl(LogSeverity.ThrewException, new LogMessageWithStack('EXCEPTION', ex));
-  }
-
-  logImpl(level, logMessage) {
-    // Deduplicate errors with the exact same stack
-    if (logMessage.stack) {
-      const seen = this.messagesForPreviouslySeenStacks.get(logMessage.stack);
-      if (seen) {
-        seen.incrementTimesSeen();
-        return;
-      }
-      this.messagesForPreviouslySeenStacks.set(logMessage.stack, logMessage);
-    }
-
-    // Mark printStack=false for all logs except 2 at the highest severity
-    if (level > this.maxLogSeverity) {
-      this.logLinesAtCurrentSeverity = 0;
-      this.maxLogSeverity = level;
-      if (!this.debugging) {
-        // Go back and turn off printStack for everything of a lower log level
-        for (const log of this.logs) {
-          log.setStackHidden();
-        }
-      }
-    }
-    if (level < this.maxLogSeverity || this.logLinesAtCurrentSeverity >= kMaxLogStacks) {
-      if (!this.debugging) {
-        logMessage.setStackHidden();
-      }
-    }
-    this.logs.push(logMessage);
-    this.logLinesAtCurrentSeverity++;
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/params_builder.js

@@ -1,140 +1,213 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ let _Symbol$iterator;
-function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
+ **/ import { mergeParams } from '../internal/params_utils.js';
+import { stringifyPublicParams } from '../internal/query/stringify_params.js';
+import { assert, mapLazy } from '../util/util.js';
+
+// ================================================================
+// "Public" ParamsBuilder API / Documentation
+// ================================================================
+
+/**
+ * Provides doc comments for the methods of CaseParamsBuilder and SubcaseParamsBuilder.
+ * (Also enforces rough interface match between them.)
+ */
+
+/**
+ * Base class for `CaseParamsBuilder` and `SubcaseParamsBuilder`.
+ */
+export class ParamsBuilderBase {
+  constructor(cases) {
+    this.cases = cases;
   }
-  return obj;
-}
-import { publicParamsEquals } from './params_utils.js';
-import { assert } from './util/util.js';
-/** Forces a type to resolve its type definitions, to make it readable/debuggable. */
 
-function typeAssert() {}
-{
-  {
-    typeAssert();
-    typeAssert();
-    typeAssert();
-    typeAssert();
-    typeAssert();
-
-    typeAssert();
-
-    typeAssert();
-    typeAssert();
-    typeAssert();
-    typeAssert();
-    typeAssert();
-
-    // Unexpected test results - hopefully okay to ignore these
-    typeAssert();
-    typeAssert();
-  }
+  /**
+   * Hidden from test files. Use `builderIterateCasesWithSubcases` to access this.
+   */
 }
 
-export function poptions(name, values) {
-  const iter = makeReusableIterable(function* () {
-    for (const value of values) {
-      yield { [name]: value };
+/**
+ * Calls the (normally hidden) `iterateCasesWithSubcases()` method.
+ */
+export function builderIterateCasesWithSubcases(builder) {
+  return builder.iterateCasesWithSubcases();
+}
+
+/**
+ * Builder for combinatorial test **case** parameters.
+ *
+ * CaseParamsBuilder is immutable. Each method call returns a new, immutable object,
+ * modifying the list of cases according to the method called.
+ *
+ * This means, for example, that the `unit` passed into `TestBuilder.params()` can be reused.
+ */
+export class CaseParamsBuilder extends ParamsBuilderBase {
+  *iterateCasesWithSubcases() {
+    for (const a of this.cases()) {
+      yield [a, undefined];
     }
-  });
-
-  return iter;
-}
-
-export function pbool(name) {
-  return poptions(name, [false, true]);
-}
-
-export function params() {
-  return new ParamsBuilder();
-}
-_Symbol$iterator = Symbol.iterator;
-export class ParamsBuilder {
-  constructor() {
-    _defineProperty(this, 'paramSpecs', [{}]);
-  }
-  [_Symbol$iterator]() {
-    const iter = this.paramSpecs[Symbol.iterator]();
-    return iter;
   }
 
-  combine(newParams) {
-    const paramSpecs = this.paramSpecs;
-    this.paramSpecs = makeReusableIterable(function* () {
-      for (const a of paramSpecs) {
-        for (const b of newParams) {
-          yield mergeParams(a, b);
-        }
+  [Symbol.iterator]() {
+    return this.cases();
+  }
+
+  /** @inheritDoc */
+  expandWithParams(expander) {
+    const newGenerator = expanderGenerator(this.cases, expander);
+    return new CaseParamsBuilder(() => newGenerator({}));
+  }
+
+  /** @inheritDoc */
+  expand(key, expander) {
+    return this.expandWithParams(function* (p) {
+      for (const value of expander(p)) {
+        yield { [key]: value };
       }
     });
-
-    return this;
   }
 
-  expand(expander) {
-    const paramSpecs = this.paramSpecs;
-    this.paramSpecs = makeReusableIterable(function* () {
-      for (const a of paramSpecs) {
-        for (const b of expander(a)) {
-          yield mergeParams(a, b);
-        }
-      }
-    });
+  /** @inheritDoc */
+  combineWithParams(newParams) {
+    assertNotGenerator(newParams);
+    const seenValues = new Set();
+    for (const params of newParams) {
+      const paramsStr = stringifyPublicParams(params);
+      assert(!seenValues.has(paramsStr), `Duplicate entry in combine[WithParams]: ${paramsStr}`);
+      seenValues.add(paramsStr);
+    }
 
-    return this;
+    return this.expandWithParams(() => newParams);
   }
 
+  /** @inheritDoc */
+  combine(key, values) {
+    assertNotGenerator(values);
+    const mapped = mapLazy(values, v => ({ [key]: v }));
+    return this.combineWithParams(mapped);
+  }
+
+  /** @inheritDoc */
   filter(pred) {
-    const paramSpecs = this.paramSpecs;
-    this.paramSpecs = makeReusableIterable(function* () {
-      for (const p of paramSpecs) {
-        if (pred(p)) {
-          yield p;
-        }
-      }
-    });
-    return this;
+    const newGenerator = filterGenerator(this.cases, pred);
+    return new CaseParamsBuilder(() => newGenerator({}));
   }
 
+  /** @inheritDoc */
   unless(pred) {
     return this.filter(x => !pred(x));
   }
 
-  exclude(exclude) {
-    const excludeArray = Array.from(exclude);
-    const paramSpecs = this.paramSpecs;
-    this.paramSpecs = makeReusableIterable(function* () {
-      for (const p of paramSpecs) {
-        if (excludeArray.every(e => !publicParamsEquals(p, e))) {
-          yield p;
-        }
+  /**
+   * "Finalize" the list of cases and begin defining subcases.
+   * Returns a new SubcaseParamsBuilder. Methods called on SubcaseParamsBuilder
+   * generate new subcases instead of new cases.
+   */
+  beginSubcases() {
+    return new SubcaseParamsBuilder(
+      () => this.cases(),
+      function* () {
+        yield {};
+      }
+    );
+  }
+}
+
+/**
+ * The unit CaseParamsBuilder, representing a single case with no params: `[ {} ]`.
+ *
+ * `punit` is passed to every `.params()`/`.paramsSubcasesOnly()` call, so `kUnitCaseParamsBuilder`
+ * is only explicitly needed if constructing a ParamsBuilder outside of a test builder.
+ */
+export const kUnitCaseParamsBuilder = new CaseParamsBuilder(function* () {
+  yield {};
+});
+
+/**
+ * Builder for combinatorial test _subcase_ parameters.
+ *
+ * SubcaseParamsBuilder is immutable. Each method call returns a new, immutable object,
+ * modifying the list of subcases according to the method called.
+ */
+export class SubcaseParamsBuilder extends ParamsBuilderBase {
+  constructor(cases, generator) {
+    super(cases);
+    this.subcases = generator;
+  }
+
+  *iterateCasesWithSubcases() {
+    for (const caseP of this.cases()) {
+      const subcases = Array.from(this.subcases(caseP));
+      if (subcases.length) {
+        yield [caseP, subcases];
+      }
+    }
+  }
+
+  /** @inheritDoc */
+  expandWithParams(expander) {
+    return new SubcaseParamsBuilder(this.cases, expanderGenerator(this.subcases, expander));
+  }
+
+  /** @inheritDoc */
+  expand(key, expander) {
+    return this.expandWithParams(function* (p) {
+      for (const value of expander(p)) {
+        // TypeScript doesn't know here that NewPKey is always a single literal string type.
+        yield { [key]: value };
       }
     });
-    return this;
+  }
+
+  /** @inheritDoc */
+  combineWithParams(newParams) {
+    assertNotGenerator(newParams);
+    return this.expandWithParams(() => newParams);
+  }
+
+  /** @inheritDoc */
+  combine(key, values) {
+    assertNotGenerator(values);
+    return this.expand(key, () => values);
+  }
+
+  /** @inheritDoc */
+  filter(pred) {
+    return new SubcaseParamsBuilder(this.cases, filterGenerator(this.subcases, pred));
+  }
+
+  /** @inheritDoc */
+  unless(pred) {
+    return this.filter(x => !pred(x));
   }
 }
 
-// If you create an Iterable by calling a generator function (e.g. in IIFE), it is exhausted after
-// one use. This just wraps a generator function in an object so it be iterated multiple times.
-function makeReusableIterable(generatorFn) {
-  return { [Symbol.iterator]: generatorFn };
+function expanderGenerator(baseGenerator, expander) {
+  return function* (base) {
+    for (const a of baseGenerator(base)) {
+      for (const b of expander(mergeParams(base, a))) {
+        yield mergeParams(a, b);
+      }
+    }
+  };
 }
 
-// (keyof A & keyof B) is not empty, so they overlapped
+function filterGenerator(baseGenerator, pred) {
+  return function* (base) {
+    for (const a of baseGenerator(base)) {
+      if (pred(mergeParams(base, a))) {
+        yield a;
+      }
+    }
+  };
+}
 
-function mergeParams(a, b) {
-  for (const key of Object.keys(a)) {
-    assert(!(key in b), 'Duplicate key: ' + key);
+/** Assert an object is not a Generator (a thing returned from a generator function). */
+function assertNotGenerator(x) {
+  if ('constructor' in x) {
+    assert(
+      x.constructor !== (function* () {})().constructor,
+      'Argument must not be a generator, as generators are not reusable'
+    );
   }
-  return { ...a, ...b };
 }

tests/wpt/webgpu/tests/webgpu/common/framework/query/json_param_value.js

@@ -1,19 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { assert } from '../util/util.js';
-// JSON can't represent `undefined` and by default stores it as `null`.
-// Instead, store `undefined` as this magic string value in JSON.
-const jsUndefinedMagicValue = '_undef_';
-
-export function stringifyParamValue(value) {
-  return JSON.stringify(value, (k, v) => {
-    // Make sure no one actually uses the magic value as a parameter.
-    assert(v !== jsUndefinedMagicValue);
-
-    return v === undefined ? jsUndefinedMagicValue : v;
-  });
-}
-
-export function parseParamValue(s) {
-  return JSON.parse(s, (k, v) => (v === jsUndefinedMagicValue ? undefined : v));
-}

tests/wpt/webgpu/tests/webgpu/common/framework/query/query.js

@@ -1,127 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { assert } from '../util/util.js';
-import { encodeURIComponentSelectively } from './encode_selectively.js';
-import { kBigSeparator, kPathSeparator, kWildcard, kParamSeparator } from './separators.js';
-import { stringifyPublicParams } from './stringify_params.js';
-
-/**
- * Represents a test query of some level.
- *
- * TestQuery types are immutable.
- */
-
-// SingleCase
-
-/**
- * A multi-file test query, like `s:*` or `s:a,b,*`.
- *
- * Immutable (makes copies of constructor args).
- */
-export class TestQueryMultiFile {
-  constructor(suite, file) {
-    _defineProperty(this, 'level', 1);
-    _defineProperty(this, 'isMultiFile', true);
-    _defineProperty(this, 'suite', void 0);
-    _defineProperty(this, 'filePathParts', void 0);
-    this.suite = suite;
-    this.filePathParts = [...file];
-  }
-
-  toString() {
-    return encodeURIComponentSelectively(this.toStringHelper().join(kBigSeparator));
-  }
-
-  toStringHelper() {
-    return [this.suite, [...this.filePathParts, kWildcard].join(kPathSeparator)];
-  }
-}
-
-/**
- * A multi-test test query, like `s:f:*` or `s:f:a,b,*`.
- *
- * Immutable (makes copies of constructor args).
- */
-export class TestQueryMultiTest extends TestQueryMultiFile {
-  constructor(suite, file, test) {
-    super(suite, file);
-    _defineProperty(this, 'level', 2);
-    _defineProperty(this, 'isMultiFile', false);
-    _defineProperty(this, 'isMultiTest', true);
-    _defineProperty(this, 'testPathParts', void 0);
-    assert(file.length > 0, 'multi-test (or finer) query must have file-path');
-    this.testPathParts = [...test];
-  }
-
-  toStringHelper() {
-    return [
-      this.suite,
-      this.filePathParts.join(kPathSeparator),
-      [...this.testPathParts, kWildcard].join(kPathSeparator),
-    ];
-  }
-}
-
-/**
- * A multi-case test query, like `s:f:t:*` or `s:f:t:a,b,*`.
- *
- * Immutable (makes copies of constructor args), except for param values
- * (which aren't normally supposed to change; they're marked readonly in CaseParams).
- */
-export class TestQueryMultiCase extends TestQueryMultiTest {
-  constructor(suite, file, test, params) {
-    super(suite, file, test);
-    _defineProperty(this, 'level', 3);
-    _defineProperty(this, 'isMultiTest', false);
-    _defineProperty(this, 'isMultiCase', true);
-    _defineProperty(this, 'params', void 0);
-    assert(test.length > 0, 'multi-case (or finer) query must have test-path');
-    this.params = { ...params };
-  }
-
-  toStringHelper() {
-    const paramsParts = stringifyPublicParams(this.params);
-    return [
-      this.suite,
-      this.filePathParts.join(kPathSeparator),
-      this.testPathParts.join(kPathSeparator),
-      [...paramsParts, kWildcard].join(kParamSeparator),
-    ];
-  }
-}
-
-/**
- * A multi-case test query, like `s:f:t:` or `s:f:t:a=1,b=1`.
- *
- * Immutable (makes copies of constructor args).
- */
-export class TestQuerySingleCase extends TestQueryMultiCase {
-  constructor(...args) {
-    super(...args);
-    _defineProperty(this, 'level', 4);
-    _defineProperty(this, 'isMultiCase', false);
-  }
-
-  toStringHelper() {
-    const paramsParts = stringifyPublicParams(this.params);
-    return [
-      this.suite,
-      this.filePathParts.join(kPathSeparator),
-      this.testPathParts.join(kPathSeparator),
-      paramsParts.join(kParamSeparator),
-    ];
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/query/stringify_params.js

@@ -1,27 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { badParamValueChars, paramKeyIsPublic } from '../params_utils.js';
-import { assert } from '../util/util.js';
-
-import { stringifyParamValue } from './json_param_value.js';
-import { kParamKVSeparator } from './separators.js';
-
-export function stringifyPublicParams(p) {
-  return Object.keys(p)
-    .filter(k => paramKeyIsPublic(k))
-    .map(k => stringifySingleParam(k, p[k]));
-}
-
-export function stringifySingleParam(k, v) {
-  return `${k}${kParamKVSeparator}${stringifySingleParamValue(v)}`;
-}
-
-function stringifySingleParamValue(v) {
-  const s = stringifyParamValue(v);
-  assert(
-    !badParamValueChars.test(s),
-    `JSON.stringified param value must not match ${badParamValueChars} - was ${s}`
-  );
-
-  return s;
-}

tests/wpt/webgpu/tests/webgpu/common/framework/resources.js

@@ -0,0 +1,111 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ /**
+ * Base path for resources. The default value is correct for non-worker WPT, but standalone and
+ * workers must access resources using a different base path, so this is overridden in
+ * `test_worker-worker.ts` and `standalone.ts`.
+ */ let baseResourcePath = './resources';
+let crossOriginHost = '';
+
+function getAbsoluteBaseResourcePath(path) {
+  // Path is already an absolute one.
+  if (path[0] === '/') {
+    return path;
+  }
+
+  // Path is relative
+  const relparts = window.location.pathname.split('/');
+  relparts.pop();
+  const pathparts = path.split('/');
+
+  let i;
+  for (i = 0; i < pathparts.length; ++i) {
+    switch (pathparts[i]) {
+      case '':
+        break;
+      case '.':
+        break;
+      case '..':
+        relparts.pop();
+        break;
+      default:
+        relparts.push(pathparts[i]);
+        break;
+    }
+  }
+
+  return relparts.join('/');
+}
+
+function runningOnLocalHost() {
+  const hostname = window.location.hostname;
+  return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1';
+}
+
+/**
+ * Get a path to a resource in the `resources` directory relative to the current execution context
+ * (html file or worker .js file), for `fetch()`, `<img>`, `<video>`, etc but from cross origin host.
+ * Provide onlineUrl if the case running online.
+ * @internal MAINTENANCE_TODO: Cases may run in the LAN environment (not localhost but no internet
+ * access). We temporarily use `crossOriginHost` to configure the cross origin host name in that situation.
+ * But opening to  auto-detect mechanism or other solutions.
+ */
+export function getCrossOriginResourcePath(pathRelativeToResourcesDir, onlineUrl = '') {
+  // A cross origin host has been configured. Use this to load resource.
+  if (crossOriginHost !== '') {
+    return (
+      crossOriginHost +
+      getAbsoluteBaseResourcePath(baseResourcePath) +
+      '/' +
+      pathRelativeToResourcesDir
+    );
+  }
+
+  // Using 'localhost' and '127.0.0.1' trick to load cross origin resource. Set cross origin host name
+  // to 'localhost' if case is not running in 'localhost' domain. Otherwise, use '127.0.0.1'.
+  // host name to locahost unless the server running in
+  if (runningOnLocalHost()) {
+    let crossOriginHostName = '';
+    if (location.hostname === 'localhost') {
+      crossOriginHostName = 'http://127.0.0.1';
+    } else {
+      crossOriginHostName = 'http://localhost';
+    }
+
+    return (
+      crossOriginHostName +
+      ':' +
+      location.port +
+      getAbsoluteBaseResourcePath(baseResourcePath) +
+      '/' +
+      pathRelativeToResourcesDir
+    );
+  }
+
+  return onlineUrl;
+}
+
+/**
+ * Get a path to a resource in the `resources` directory, relative to the current execution context
+ * (html file or worker .js file), for `fetch()`, `<img>`, `<video>`, etc. Pass the cross origin host
+ * name if wants to load resoruce from cross origin host.
+ */
+export function getResourcePath(pathRelativeToResourcesDir) {
+  return baseResourcePath + '/' + pathRelativeToResourcesDir;
+}
+
+/**
+ * Set the base resource path (path to the `resources` directory relative to the current
+ * execution context).
+ */
+export function setBaseResourcePath(path) {
+  baseResourcePath = path;
+}
+
+/**
+ * Set the cross origin host and cases related to cross origin
+ * will load resource from the given host.
+ */
+export function setCrossOriginHost(host) {
+  crossOriginHost = host;
+}

tests/wpt/webgpu/tests/webgpu/common/framework/test_config.js

@@ -0,0 +1,10 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+export const globalTestConfig = {
+  maxSubcasesInFlight: 500,
+  testHeartbeatCallback: () => {},
+  noRaceWithRejectOnTimeout: false,
+  unrollConstEvalLoops: false,
+};

tests/wpt/webgpu/tests/webgpu/common/framework/test_group.js

@@ -1,162 +1,3 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { extractPublicParams, publicParamsEquals } from './params_utils.js';
-import { kPathSeparator } from './query/separators.js';
-import { stringifyPublicParams } from './query/stringify_params.js';
-import { validQueryPart } from './query/validQueryPart.js';
-import { assert } from './util/util.js';
-
-export function makeTestGroup(fixture) {
-  return new TestGroup(fixture);
-}
-
-// Interface for running tests
-
-export function makeTestGroupForUnitTesting(fixture) {
-  return new TestGroup(fixture);
-}
-
-class TestGroup {
-  constructor(fixture) {
-    _defineProperty(this, 'fixture', void 0);
-    _defineProperty(this, 'seen', new Set());
-    _defineProperty(this, 'tests', []);
-    this.fixture = fixture;
-  }
-
-  *iterate() {
-    for (const test of this.tests) {
-      yield* test.iterate();
-    }
-  }
-
-  checkName(name) {
-    assert(
-      // Shouldn't happen due to the rule above. Just makes sure that treated
-      // unencoded strings as encoded strings is OK.
-      name === decodeURIComponent(name),
-      `Not decodeURIComponent-idempotent: ${name} !== ${decodeURIComponent(name)}`
-    );
-
-    assert(!this.seen.has(name), `Duplicate test name: ${name}`);
-
-    this.seen.add(name);
-  }
-
-  // TODO: This could take a fixture, too, to override the one for the group.
-  test(name) {
-    this.checkName(name);
-
-    const parts = name.split(kPathSeparator);
-    for (const p of parts) {
-      assert(validQueryPart.test(p), `Invalid test name part ${p}; must match ${validQueryPart}`);
-    }
-
-    const test = new TestBuilder(parts, this.fixture);
-    this.tests.push(test);
-    return test;
-  }
-
-  checkCaseNamesAndDuplicates() {
-    for (const test of this.tests) {
-      test.checkCaseNamesAndDuplicates();
-    }
-  }
-}
-
-class TestBuilder {
-  constructor(testPath, fixture) {
-    _defineProperty(this, 'testPath', void 0);
-    _defineProperty(this, 'fixture', void 0);
-    _defineProperty(this, 'testFn', void 0);
-    _defineProperty(this, 'cases', undefined);
-    this.testPath = testPath;
-    this.fixture = fixture;
-  }
-
-  fn(fn) {
-    this.testFn = fn;
-  }
-
-  checkCaseNamesAndDuplicates() {
-    if (this.cases === undefined) {
-      return;
-    }
-
-    // This is n^2.
-    const seen = [];
-    for (const testcase of this.cases) {
-      // stringifyPublicParams also checks for invalid params values
-      const testcaseString = stringifyPublicParams(testcase);
-      assert(
-        !seen.some(x => publicParamsEquals(x, testcase)),
-        `Duplicate public test case params: ${testcaseString}`
-      );
-
-      seen.push(testcase);
-    }
-  }
-
-  params(casesIterable) {
-    assert(this.cases === undefined, 'test case is already parameterized');
-    this.cases = Array.from(casesIterable);
-
-    return this;
-  }
-
-  *iterate() {
-    assert(this.testFn !== undefined, 'No test function (.fn()) for test');
-    for (const params of this.cases || [{}]) {
-      yield new RunCaseSpecific(this.testPath, params, this.fixture, this.testFn);
-    }
-  }
-}
-
-class RunCaseSpecific {
-  constructor(testPath, params, fixture, fn) {
-    _defineProperty(this, 'id', void 0);
-    _defineProperty(this, 'params', void 0);
-    _defineProperty(this, 'fixture', void 0);
-    _defineProperty(this, 'fn', void 0);
-    this.id = { test: testPath, params: extractPublicParams(params) };
-    this.params = params;
-    this.fixture = fixture;
-    this.fn = fn;
-  }
-
-  async run(rec) {
-    rec.start();
-    try {
-      const inst = new this.fixture(rec, this.params || {});
-
-      try {
-        await inst.init();
-
-        await this.fn(inst);
-      } finally {
-        // Runs as long as constructor succeeded, even if initialization or the test failed.
-        await inst.finalize();
-      }
-    } catch (ex) {
-      // There was an exception from constructor, init, test, or finalize.
-      // An error from init or test may have been a SkipTestCase.
-      // An error from finalize may have been an eventualAsyncExpectation failure
-      // or unexpected validation/OOM error from the GPUDevice.
-      rec.threw(ex);
-    }
-    rec.finish();
-  }
-}
+ **/ export { makeTestGroup } from '../internal/test_group.js';

tests/wpt/webgpu/tests/webgpu/common/framework/util/async_mutex.js

@@ -1,38 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-export class AsyncMutex {
-  constructor() {
-    _defineProperty(this, 'newestQueueItem', void 0);
-  }
-
-  // Run an async function with a lock on this mutex.
-  // Waits until the mutex is available, locks it, runs the function, then releases it.
-  async with(fn) {
-    const p = (async () => {
-      // If the mutex is locked, wait for the last thing in the queue before running.
-      // (Everything in the queue runs in order, so this is after everything currently enqueued.)
-      if (this.newestQueueItem) {
-        await this.newestQueueItem;
-      }
-      return fn();
-    })();
-
-    // Push the newly-created Promise onto the queue by replacing the old "newest" item.
-    this.newestQueueItem = p;
-    // And return so the caller can wait on the result.
-    return p;
-  }
-}

tests/wpt/webgpu/tests/webgpu/common/framework/util/timeout.js

@@ -1,3 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ export const timeout = typeof step_timeout !== 'undefined' ? step_timeout : setTimeout;

tests/wpt/webgpu/tests/webgpu/common/framework/util/util.js

@@ -1,72 +0,0 @@
-/**
- * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { timeout } from './timeout.js';
-export function assert(condition, msg) {
-  if (!condition) {
-    throw new Error(msg && (typeof msg === 'string' ? msg : msg()));
-  }
-}
-
-export async function assertReject(p, msg) {
-  try {
-    await p;
-    unreachable(msg);
-  } catch (ex) {
-    // Assertion OK
-  }
-}
-
-export function unreachable(msg) {
-  throw new Error(msg);
-}
-
-// performance.now() is available in all browsers, but not in scope by default in Node.
-const perf = typeof performance !== 'undefined' ? performance : require('perf_hooks').performance;
-
-export function now() {
-  return perf.now();
-}
-
-export function resolveOnTimeout(ms) {
-  return new Promise(resolve => {
-    timeout(() => {
-      resolve();
-    }, ms);
-  });
-}
-
-export class PromiseTimeoutError extends Error {}
-
-export function rejectOnTimeout(ms, msg) {
-  return new Promise((_resolve, reject) => {
-    timeout(() => {
-      reject(new PromiseTimeoutError(msg));
-    }, ms);
-  });
-}
-
-export function raceWithRejectOnTimeout(p, ms, msg) {
-  return Promise.race([p, rejectOnTimeout(ms, msg)]);
-}
-
-export function objectEquals(x, y) {
-  if (typeof x !== 'object' || typeof y !== 'object') return x === y;
-  if (x === null || y === null) return x === y;
-  if (x.constructor !== y.constructor) return false;
-  if (x instanceof Function) return x === y;
-  if (x instanceof RegExp) return x === y;
-  if (x === y || x.valueOf() === y.valueOf()) return true;
-  if (Array.isArray(x) && Array.isArray(y) && x.length !== y.length) return false;
-  if (x instanceof Date) return false;
-  if (!(x instanceof Object)) return false;
-  if (!(y instanceof Object)) return false;
-
-  const x1 = x;
-  const y1 = y;
-  const p = Object.keys(x);
-  return Object.keys(y).every(i => p.indexOf(i) !== -1) && p.every(i => objectEquals(x1[i], y1[i]));
-}
-
-export function range(n, fn) {
-  return [...new Array(n)].map((_, i) => fn(i));
-}

tests/wpt/webgpu/tests/webgpu/common/framework/version.js

@@ -1,3 +0,0 @@
-// AUTO-GENERATED - DO NOT EDIT. See tools/gen_version.
-
-export const version = 'c1df7f4ff1adcde985384633e7cffa52d53e3535';

tests/wpt/webgpu/tests/webgpu/common/internal/file_loader.js

@@ -1,6 +1,7 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { parseQuery } from './query/parseQuery.js';
+ **/ import { assert } from '../util/util.js';
+import { parseQuery } from './query/parseQuery.js';
 
 import { loadTreeForQuery } from './tree.js';
 
@@ -9,17 +10,30 @@ import { loadTreeForQuery } from './tree.js';
 // - `out/webgpu/listing.js` (which is pre-baked, has a TestSuiteListing)
 
 // Base class for DefaultTestFileLoader and FakeTestFileLoader.
-export class TestFileLoader {
-  importSpecFile(suite, path) {
-    return this.import(`${suite}/${path.join('/')}.spec.js`);
+export class TestFileLoader extends EventTarget {
+  async importSpecFile(suite, path) {
+    const url = `${suite}/${path.join('/')}.spec.js`;
+    this.dispatchEvent(new MessageEvent('import', { data: { url } }));
+
+    const ret = await this.import(url);
+    this.dispatchEvent(new MessageEvent('imported', { data: { url } }));
+
+    return ret;
   }
 
   async loadTree(query, subqueriesToExpand = []) {
-    return loadTreeForQuery(
+    const tree = await loadTreeForQuery(
       this,
       query,
-      subqueriesToExpand.map(q => parseQuery(q))
+      subqueriesToExpand.map(s => {
+        const q = parseQuery(s);
+        assert(q.level >= 2, () => `subqueriesToExpand entries should not be multi-file:\n  ${q}`);
+        return q;
+      })
     );
+
+    this.dispatchEvent(new MessageEvent('finish'));
+    return tree;
   }
 
   async loadCases(query) {

tests/wpt/webgpu/tests/webgpu/common/internal/logging/log_message.js

@@ -0,0 +1,42 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { extractImportantStackTrace } from '../stack.js';
+export class LogMessageWithStack extends Error {
+  stackHiddenMessage = undefined;
+
+  constructor(name, ex) {
+    super(ex.message);
+
+    this.name = name;
+    this.stack = ex.stack;
+    if ('extra' in ex) {
+      this.extra = ex.extra;
+    }
+  }
+
+  /** Set a flag so the stack is not printed in toJSON(). */
+  setStackHidden(stackHiddenMessage) {
+    this.stackHiddenMessage ??= stackHiddenMessage;
+  }
+
+  toJSON() {
+    let m = this.name;
+    if (this.message) m += ': ' + this.message;
+    if (this.stack) {
+      if (this.stackHiddenMessage === undefined) {
+        m += '\n' + extractImportantStackTrace(this);
+      } else if (this.stackHiddenMessage) {
+        m += `\n  at (elided: ${this.stackHiddenMessage})`;
+      }
+    }
+    return m;
+  }
+}
+
+/**
+ * Returns a string, nicely indented, for debug logs.
+ * This is used in the cmdline and wpt runtimes. In WPT, it shows up in the `*-actual.txt` file.
+ */
+export function prettyPrintLog(log) {
+  return '  - ' + log.toJSON().replace(/\n/g, '\n    ');
+}

tests/wpt/webgpu/tests/webgpu/common/internal/logging/logger.js

@@ -0,0 +1,27 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { version } from '../version.js';
+import { TestCaseRecorder } from './test_case_recorder.js';
+
+export class Logger {
+  static globalDebugMode = false;
+
+  results = new Map();
+
+  constructor({ overrideDebugMode } = {}) {
+    this.overriddenDebugMode = overrideDebugMode;
+  }
+
+  record(name) {
+    const result = { status: 'running', timems: -1 };
+    this.results.set(name, result);
+    return [
+      new TestCaseRecorder(result, this.overriddenDebugMode ?? Logger.globalDebugMode),
+      result,
+    ];
+  }
+
+  asJSON(space) {
+    return JSON.stringify({ version, results: Array.from(this.results) }, undefined, space);
+  }
+}

tests/wpt/webgpu/tests/webgpu/common/internal/logging/result.js

@@ -1,3 +1,3 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/
+ **/ export {};

tests/wpt/webgpu/tests/webgpu/common/internal/logging/test_case_recorder.js

@@ -0,0 +1,158 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { SkipTestCase, UnexpectedPassError } from '../../framework/fixture.js';
+import { globalTestConfig } from '../../framework/test_config.js';
+import { now, assert } from '../../util/util.js';
+
+import { LogMessageWithStack } from './log_message.js';
+var LogSeverity;
+(function (LogSeverity) {
+  LogSeverity[(LogSeverity['Pass'] = 0)] = 'Pass';
+  LogSeverity[(LogSeverity['Skip'] = 1)] = 'Skip';
+  LogSeverity[(LogSeverity['Warn'] = 2)] = 'Warn';
+  LogSeverity[(LogSeverity['ExpectFailed'] = 3)] = 'ExpectFailed';
+  LogSeverity[(LogSeverity['ValidationFailed'] = 4)] = 'ValidationFailed';
+  LogSeverity[(LogSeverity['ThrewException'] = 5)] = 'ThrewException';
+})(LogSeverity || (LogSeverity = {}));
+
+const kMaxLogStacks = 2;
+const kMinSeverityForStack = LogSeverity.Warn;
+
+/** Holds onto a LiveTestCaseResult owned by the Logger, and writes the results into it. */
+export class TestCaseRecorder {
+  inSubCase = false;
+  subCaseStatus = LogSeverity.Pass;
+  finalCaseStatus = LogSeverity.Pass;
+  hideStacksBelowSeverity = kMinSeverityForStack;
+  startTime = -1;
+  logs = [];
+  logLinesAtCurrentSeverity = 0;
+  debugging = false;
+  /** Used to dedup log messages which have identical stacks. */
+  messagesForPreviouslySeenStacks = new Map();
+
+  constructor(result, debugging) {
+    this.result = result;
+    this.debugging = debugging;
+  }
+
+  start() {
+    assert(this.startTime < 0, 'TestCaseRecorder cannot be reused');
+    this.startTime = now();
+  }
+
+  finish() {
+    assert(this.startTime >= 0, 'finish() before start()');
+
+    const timeMilliseconds = now() - this.startTime;
+    // Round to next microsecond to avoid storing useless .xxxx00000000000002 in results.
+    this.result.timems = Math.ceil(timeMilliseconds * 1000) / 1000;
+
+    // Convert numeric enum back to string (but expose 'exception' as 'fail')
+    this.result.status =
+      this.finalCaseStatus === LogSeverity.Pass
+        ? 'pass'
+        : this.finalCaseStatus === LogSeverity.Skip
+        ? 'skip'
+        : this.finalCaseStatus === LogSeverity.Warn
+        ? 'warn'
+        : 'fail'; // Everything else is an error
+
+    this.result.logs = this.logs;
+  }
+
+  beginSubCase() {
+    this.subCaseStatus = LogSeverity.Pass;
+    this.inSubCase = true;
+  }
+
+  endSubCase(expectedStatus) {
+    try {
+      if (expectedStatus === 'fail') {
+        if (this.subCaseStatus <= LogSeverity.Warn) {
+          throw new UnexpectedPassError();
+        } else {
+          this.subCaseStatus = LogSeverity.Pass;
+        }
+      }
+    } finally {
+      this.inSubCase = false;
+      if (this.subCaseStatus > this.finalCaseStatus) {
+        this.finalCaseStatus = this.subCaseStatus;
+      }
+    }
+  }
+
+  injectResult(injectedResult) {
+    Object.assign(this.result, injectedResult);
+  }
+
+  debug(ex) {
+    if (!this.debugging) return;
+    this.logImpl(LogSeverity.Pass, 'DEBUG', ex);
+  }
+
+  info(ex) {
+    this.logImpl(LogSeverity.Pass, 'INFO', ex);
+  }
+
+  skipped(ex) {
+    this.logImpl(LogSeverity.Skip, 'SKIP', ex);
+  }
+
+  warn(ex) {
+    this.logImpl(LogSeverity.Warn, 'WARN', ex);
+  }
+
+  expectationFailed(ex) {
+    this.logImpl(LogSeverity.ExpectFailed, 'EXPECTATION FAILED', ex);
+  }
+
+  validationFailed(ex) {
+    this.logImpl(LogSeverity.ValidationFailed, 'VALIDATION FAILED', ex);
+  }
+
+  threw(ex) {
+    if (ex instanceof SkipTestCase) {
+      this.skipped(ex);
+      return;
+    }
+    this.logImpl(LogSeverity.ThrewException, 'EXCEPTION', ex);
+  }
+
+  logImpl(level, name, baseException) {
+    assert(baseException instanceof Error, 'test threw a non-Error object');
+    globalTestConfig.testHeartbeatCallback();
+    const logMessage = new LogMessageWithStack(name, baseException);
+
+    // Final case status should be the "worst" of all log entries.
+    if (this.inSubCase) {
+      if (level > this.subCaseStatus) this.subCaseStatus = level;
+    } else {
+      if (level > this.finalCaseStatus) this.finalCaseStatus = level;
+    }
+
+    // setFirstLineOnly for all logs except `kMaxLogStacks` stacks at the highest severity
+    if (level > this.hideStacksBelowSeverity) {
+      this.logLinesAtCurrentSeverity = 0;
+      this.hideStacksBelowSeverity = level;
+
+      // Go back and setFirstLineOnly for everything of a lower log level
+      for (const log of this.logs) {
+        log.setStackHidden('below max severity');
+      }
+    }
+    if (level === this.hideStacksBelowSeverity) {
+      this.logLinesAtCurrentSeverity++;
+    } else if (level < kMinSeverityForStack) {
+      logMessage.setStackHidden('');
+    } else if (level < this.hideStacksBelowSeverity) {
+      logMessage.setStackHidden('below max severity');
+    }
+    if (this.logLinesAtCurrentSeverity > kMaxLogStacks) {
+      logMessage.setStackHidden(`only ${kMaxLogStacks} shown`);
+    }
+
+    this.logs.push(logMessage);
+  }
+}

tests/wpt/webgpu/tests/webgpu/common/internal/params_utils.js

@@ -1,11 +1,9 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { comparePublicParamsPaths, Ordering } from './query/compare.js';
+ **/ import { assert } from '../util/util.js';
+
+import { comparePublicParamsPaths, Ordering } from './query/compare.js';
 import { kWildcard, kParamSeparator, kParamKVSeparator } from './query/separators.js';
-// Consider adding more types here if needed
-//
-// TODO: This type isn't actually used to constrain what you're allowed to do in `.params()`, so
-// it's not really serving its purpose. Figure out how to fix that?
 
 export function paramKeyIsPublic(key) {
   return !key.startsWith('_');
@@ -28,3 +26,33 @@ export const badParamValueChars = new RegExp(
 export function publicParamsEquals(x, y) {
   return comparePublicParamsPaths(x, y) === Ordering.Equal;
 }
+
+function typeAssert() {}
+{
+  {
+    typeAssert();
+    typeAssert();
+    typeAssert();
+    typeAssert();
+    typeAssert();
+
+    typeAssert();
+
+    typeAssert();
+    typeAssert();
+    typeAssert();
+    typeAssert();
+    typeAssert();
+
+    // Unexpected test results - hopefully okay to ignore these
+    typeAssert();
+    typeAssert();
+  }
+}
+
+export function mergeParams(a, b) {
+  for (const key of Object.keys(a)) {
+    assert(!(key in b), 'Duplicate key: ' + key);
+  }
+  return { ...a, ...b };
+}

tests/wpt/webgpu/tests/webgpu/common/internal/query/compare.js

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { paramKeyIsPublic } from '../params_utils.js';
-import { assert, objectEquals } from '../util/util.js';
+ **/ import { assert, objectEquals } from '../../util/util.js';
+import { paramKeyIsPublic } from '../params_utils.js';
 
 export let Ordering;
 

tests/wpt/webgpu/tests/webgpu/common/internal/query/json_param_value.js

@@ -0,0 +1,103 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { assert, sortObjectByKey } from '../../util/util.js';
+// JSON can't represent various values and by default stores them as `null`.
+// Instead, storing them as a magic string values in JSON.
+const jsUndefinedMagicValue = '_undef_';
+const jsNaNMagicValue = '_nan_';
+const jsPositiveInfinityMagicValue = '_posinfinity_';
+const jsNegativeInfinityMagicValue = '_neginfinity_';
+
+// -0 needs to be handled separately, because -0 === +0 returns true. Not
+// special casing +0/0, since it behaves intuitively. Assuming that if -0 is
+// being used, the differentiation from +0 is desired.
+const jsNegativeZeroMagicValue = '_negzero_';
+
+// bigint values are not defined in JSON, so need to wrap them up as strings
+const jsBigIntMagicPattern = /^(\d+)n$/;
+
+const toStringMagicValue = new Map([
+  [undefined, jsUndefinedMagicValue],
+  [NaN, jsNaNMagicValue],
+  [Number.POSITIVE_INFINITY, jsPositiveInfinityMagicValue],
+  [Number.NEGATIVE_INFINITY, jsNegativeInfinityMagicValue],
+  // No -0 handling because it is special cased.
+]);
+
+const fromStringMagicValue = new Map([
+  [jsUndefinedMagicValue, undefined],
+  [jsNaNMagicValue, NaN],
+  [jsPositiveInfinityMagicValue, Number.POSITIVE_INFINITY],
+  [jsNegativeInfinityMagicValue, Number.NEGATIVE_INFINITY],
+  // -0 is handled in this direction because there is no comparison issue.
+  [jsNegativeZeroMagicValue, -0],
+]);
+
+function stringifyFilter(k, v) {
+  // Make sure no one actually uses a magic value as a parameter.
+  if (typeof v === 'string') {
+    assert(
+      !fromStringMagicValue.has(v),
+      `${v} is a magic value for stringification, so cannot be used`
+    );
+
+    assert(
+      v !== jsNegativeZeroMagicValue,
+      `${v} is a magic value for stringification, so cannot be used`
+    );
+
+    assert(
+      v.match(jsBigIntMagicPattern) === null,
+      `${v} matches bigint magic pattern for stringification, so cannot be used`
+    );
+  }
+
+  if (Object.is(v, -0)) {
+    return jsNegativeZeroMagicValue;
+  }
+
+  if (typeof v === 'bigint') {
+    return `${v}n`;
+  }
+
+  return toStringMagicValue.has(v) ? toStringMagicValue.get(v) : v;
+}
+
+export function stringifyParamValue(value) {
+  return JSON.stringify(value, stringifyFilter);
+}
+
+/**
+ * Like stringifyParamValue but sorts dictionaries by key, for hashing.
+ */
+export function stringifyParamValueUniquely(value) {
+  return JSON.stringify(value, (k, v) => {
+    if (typeof v === 'object' && v !== null) {
+      return sortObjectByKey(v);
+    }
+
+    return stringifyFilter(k, v);
+  });
+}
+
+// 'any' is part of the JSON.parse reviver interface, so cannot be avoided.
+
+function parseParamValueReviver(k, v) {
+  if (fromStringMagicValue.has(v)) {
+    return fromStringMagicValue.get(v);
+  }
+
+  if (typeof v === 'string') {
+    const match = v.match(jsBigIntMagicPattern);
+    if (match !== null) {
+      // [0] is the entire match, and following entries are the capture groups
+      return BigInt(match[1]);
+    }
+  }
+
+  return v;
+}
+
+export function parseParamValue(s) {
+  return JSON.parse(s, parseParamValueReviver);
+}

tests/wpt/webgpu/tests/webgpu/common/internal/query/parseQuery.js

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { badParamValueChars, paramKeyIsPublic } from '../params_utils.js';
-import { assert } from '../util/util.js';
+ **/ import { assert } from '../../util/util.js';
+import { badParamValueChars, paramKeyIsPublic } from '../params_utils.js';
 
 import { parseParamValue } from './json_param_value.js';
 import {
@@ -17,7 +17,9 @@ export function parseQuery(s) {
   try {
     return parseQueryImpl(s);
   } catch (ex) {
-    ex.message += '\n  on: ' + s;
+    if (ex instanceof Error) {
+      ex.message += '\n  on: ' + s;
+    }
     throw ex;
   }
 }
@@ -91,7 +93,7 @@ function parseQueryImpl(s) {
   const params = {};
   for (const paramPart of paramsParts) {
     const [k, v] = parseSingleParam(paramPart);
-    assert(validQueryPart.test(k), 'param key names must match ' + validQueryPart);
+    assert(validQueryPart.test(k), `param key names must match ${validQueryPart}`);
     params[k] = v;
   }
   if (paramsHasWildcard) {

tests/wpt/webgpu/tests/webgpu/common/internal/query/query.js

@@ -0,0 +1,237 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { optionEnabled } from '../../runtime/helper/options.js';
+import { assert, unreachable } from '../../util/util.js';
+
+import { compareQueries, Ordering } from './compare.js';
+import { encodeURIComponentSelectively } from './encode_selectively.js';
+import { parseQuery } from './parseQuery.js';
+import { kBigSeparator, kPathSeparator, kWildcard } from './separators.js';
+import { stringifyPublicParams } from './stringify_params.js';
+
+/**
+ * Represents a test query of some level.
+ *
+ * TestQuery types are immutable.
+ */
+
+/**
+ * A multi-file test query, like `s:*` or `s:a,b,*`.
+ *
+ * Immutable (makes copies of constructor args).
+ */
+export class TestQueryMultiFile {
+  level = 1;
+  isMultiFile = true;
+
+  constructor(suite, file) {
+    this.suite = suite;
+    this.filePathParts = [...file];
+  }
+
+  get depthInLevel() {
+    return this.filePathParts.length;
+  }
+
+  toString() {
+    return encodeURIComponentSelectively(this.toStringHelper().join(kBigSeparator));
+  }
+
+  toStringHelper() {
+    return [this.suite, [...this.filePathParts, kWildcard].join(kPathSeparator)];
+  }
+}
+
+/**
+ * A multi-test test query, like `s:f:*` or `s:f:a,b,*`.
+ *
+ * Immutable (makes copies of constructor args).
+ */
+export class TestQueryMultiTest extends TestQueryMultiFile {
+  level = 2;
+  isMultiFile = false;
+  isMultiTest = true;
+
+  constructor(suite, file, test) {
+    super(suite, file);
+    assert(file.length > 0, 'multi-test (or finer) query must have file-path');
+    this.testPathParts = [...test];
+  }
+
+  get depthInLevel() {
+    return this.testPathParts.length;
+  }
+
+  toStringHelper() {
+    return [
+      this.suite,
+      this.filePathParts.join(kPathSeparator),
+      [...this.testPathParts, kWildcard].join(kPathSeparator),
+    ];
+  }
+}
+
+/**
+ * A multi-case test query, like `s:f:t:*` or `s:f:t:a,b,*`.
+ *
+ * Immutable (makes copies of constructor args), except for param values
+ * (which aren't normally supposed to change; they're marked readonly in TestParams).
+ */
+export class TestQueryMultiCase extends TestQueryMultiTest {
+  level = 3;
+  isMultiTest = false;
+  isMultiCase = true;
+
+  constructor(suite, file, test, params) {
+    super(suite, file, test);
+    assert(test.length > 0, 'multi-case (or finer) query must have test-path');
+    this.params = { ...params };
+  }
+
+  get depthInLevel() {
+    return Object.keys(this.params).length;
+  }
+
+  toStringHelper() {
+    return [
+      this.suite,
+      this.filePathParts.join(kPathSeparator),
+      this.testPathParts.join(kPathSeparator),
+      stringifyPublicParams(this.params, true),
+    ];
+  }
+}
+
+/**
+ * A multi-case test query, like `s:f:t:` or `s:f:t:a=1,b=1`.
+ *
+ * Immutable (makes copies of constructor args).
+ */
+export class TestQuerySingleCase extends TestQueryMultiCase {
+  level = 4;
+  isMultiCase = false;
+
+  get depthInLevel() {
+    return 0;
+  }
+
+  toStringHelper() {
+    return [
+      this.suite,
+      this.filePathParts.join(kPathSeparator),
+      this.testPathParts.join(kPathSeparator),
+      stringifyPublicParams(this.params),
+    ];
+  }
+}
+
+/**
+ * Parse raw expectations input into TestQueryWithExpectation[], filtering so that only
+ * expectations that are relevant for the provided query and wptURL.
+ *
+ * `rawExpectations` should be @type {{ query: string, expectation: Expectation }[]}
+ *
+ * The `rawExpectations` are parsed and validated that they are in the correct format.
+ * If `wptURL` is passed, the query string should be of the full path format such
+ * as `path/to/cts.https.html?worker=0&q=suite:test_path:test_name:foo=1;bar=2;*`.
+ * If `wptURL` is `undefined`, the query string should be only the query
+ * `suite:test_path:test_name:foo=1;bar=2;*`.
+ */
+export function parseExpectationsForTestQuery(
+  rawExpectations,
+
+  query,
+  wptURL
+) {
+  if (!Array.isArray(rawExpectations)) {
+    unreachable('Expectations should be an array');
+  }
+  const expectations = [];
+  for (const entry of rawExpectations) {
+    assert(typeof entry === 'object');
+    const rawExpectation = entry;
+    assert(rawExpectation.query !== undefined, 'Expectation missing query string');
+    assert(rawExpectation.expectation !== undefined, 'Expectation missing expectation string');
+
+    let expectationQuery;
+    if (wptURL !== undefined) {
+      const expectationURL = new URL(`${wptURL.origin}/${entry.query}`);
+      if (expectationURL.pathname !== wptURL.pathname) {
+        continue;
+      }
+      assert(
+        expectationURL.pathname === wptURL.pathname,
+        `Invalid expectation path ${expectationURL.pathname}
+Expectation should be of the form path/to/cts.https.html?worker=0&q=suite:test_path:test_name:foo=1;bar=2;...
+        `
+      );
+
+      const params = expectationURL.searchParams;
+      if (optionEnabled('worker', params) !== optionEnabled('worker', wptURL.searchParams)) {
+        continue;
+      }
+
+      const qs = params.getAll('q');
+      assert(qs.length === 1, 'currently, there must be exactly one ?q= in the expectation string');
+      expectationQuery = parseQuery(qs[0]);
+    } else {
+      expectationQuery = parseQuery(entry.query);
+    }
+
+    // Strip params from multicase expectations so that an expectation of foo=2;*
+    // is stored if the test query is bar=3;*
+    const queryForFilter =
+      expectationQuery instanceof TestQueryMultiCase
+        ? new TestQueryMultiCase(
+            expectationQuery.suite,
+            expectationQuery.filePathParts,
+            expectationQuery.testPathParts,
+            {}
+          )
+        : expectationQuery;
+
+    if (compareQueries(query, queryForFilter) === Ordering.Unordered) {
+      continue;
+    }
+
+    switch (entry.expectation) {
+      case 'pass':
+      case 'skip':
+      case 'fail':
+        break;
+      default:
+        unreachable(`Invalid expectation ${entry.expectation}`);
+    }
+
+    expectations.push({
+      query: expectationQuery,
+      expectation: entry.expectation,
+    });
+  }
+  return expectations;
+}
+
+/**
+ * For display purposes only, produces a "relative" query string from parent to child.
+ * Used in the wpt runtime to reduce the verbosity of logs.
+ */
+export function relativeQueryString(parent, child) {
+  const ordering = compareQueries(parent, child);
+  if (ordering === Ordering.Equal) {
+    return '';
+  } else if (ordering === Ordering.StrictSuperset) {
+    const parentString = parent.toString();
+    assert(parentString.endsWith(kWildcard));
+    const childString = child.toString();
+    assert(
+      childString.startsWith(parentString.substring(0, parentString.length - 2)),
+      'impossible?: childString does not start with parentString[:-2]'
+    );
+
+    return childString.substring(parentString.length - 2);
+  } else {
+    unreachable(
+      `relativeQueryString arguments have invalid ordering ${ordering}:\n${parent}\n${child}`
+    );
+  }
+}

tests/wpt/webgpu/tests/webgpu/common/internal/query/stringify_params.js

@@ -0,0 +1,46 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { assert } from '../../util/util.js';
+import { badParamValueChars, paramKeyIsPublic } from '../params_utils.js';
+
+import { stringifyParamValue, stringifyParamValueUniquely } from './json_param_value.js';
+import { kParamKVSeparator, kParamSeparator, kWildcard } from './separators.js';
+
+export function stringifyPublicParams(p, addWildcard = false) {
+  const parts = Object.keys(p)
+    .filter(k => paramKeyIsPublic(k))
+    .map(k => stringifySingleParam(k, p[k]));
+
+  if (addWildcard) parts.push(kWildcard);
+
+  return parts.join(kParamSeparator);
+}
+
+/**
+ * An _approximately_ unique string representing a CaseParams value.
+ */
+export function stringifyPublicParamsUniquely(p) {
+  const keys = Object.keys(p).sort();
+  return keys
+    .filter(k => paramKeyIsPublic(k))
+    .map(k => stringifySingleParamUniquely(k, p[k]))
+    .join(kParamSeparator);
+}
+
+export function stringifySingleParam(k, v) {
+  return `${k}${kParamKVSeparator}${stringifySingleParamValue(v)}`;
+}
+
+function stringifySingleParamUniquely(k, v) {
+  return `${k}${kParamKVSeparator}${stringifyParamValueUniquely(v)}`;
+}
+
+function stringifySingleParamValue(v) {
+  const s = stringifyParamValue(v);
+  assert(
+    !badParamValueChars.test(s),
+    `JSON.stringified param value must not match ${badParamValueChars} - was ${s}`
+  );
+
+  return s;
+}

tests/wpt/webgpu/tests/webgpu/common/internal/stack.js

@@ -3,17 +3,23 @@
  **/ // Returns the stack trace of an Error, but without the extra boilerplate at the bottom
 // (e.g. RunCaseSpecific, processTicksAndRejections, etc.), for logging.
 export function extractImportantStackTrace(e) {
-  if (!e.stack) {
+  let stack = e.stack;
+  if (!stack) {
     return '';
   }
-  const lines = e.stack.split('\n');
+  const redundantMessage = 'Error: ' + e.message + '\n';
+  if (stack.startsWith(redundantMessage)) {
+    stack = stack.substring(redundantMessage.length);
+  }
+
+  const lines = stack.split('\n');
   for (let i = lines.length - 1; i >= 0; --i) {
     const line = lines[i];
     if (line.indexOf('.spec.') !== -1) {
       return lines.slice(0, i + 1).join('\n');
     }
   }
-  return e.stack;
+  return stack;
 }
 
 // *** Examples ***

tests/wpt/webgpu/tests/webgpu/common/internal/test_group.js

@@ -0,0 +1,471 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { SkipTestCase, UnexpectedPassError } from '../framework/fixture.js';
+import {
+  builderIterateCasesWithSubcases,
+  kUnitCaseParamsBuilder,
+} from '../framework/params_builder.js';
+import { globalTestConfig } from '../framework/test_config.js';
+
+import { TestCaseRecorder } from '../internal/logging/test_case_recorder.js';
+import { extractPublicParams, mergeParams } from '../internal/params_utils.js';
+import { compareQueries, Ordering } from '../internal/query/compare.js';
+import { TestQuerySingleCase } from '../internal/query/query.js';
+import { kPathSeparator } from '../internal/query/separators.js';
+import {
+  stringifyPublicParams,
+  stringifyPublicParamsUniquely,
+} from '../internal/query/stringify_params.js';
+import { validQueryPart } from '../internal/query/validQueryPart.js';
+import { assert, unreachable } from '../util/util.js';
+
+export function makeTestGroup(fixture) {
+  return new TestGroup(fixture);
+}
+
+// Interfaces for running tests
+
+export function makeTestGroupForUnitTesting(fixture) {
+  return new TestGroup(fixture);
+}
+
+export class TestGroup {
+  seen = new Set();
+  tests = [];
+
+  constructor(fixture) {
+    this.fixture = fixture;
+  }
+
+  iterate() {
+    return this.tests;
+  }
+
+  checkName(name) {
+    assert(
+      // Shouldn't happen due to the rule above. Just makes sure that treating
+      // unencoded strings as encoded strings is OK.
+      name === decodeURIComponent(name),
+      `Not decodeURIComponent-idempotent: ${name} !== ${decodeURIComponent(name)}`
+    );
+
+    assert(!this.seen.has(name), `Duplicate test name: ${name}`);
+
+    this.seen.add(name);
+  }
+
+  test(name) {
+    const testCreationStack = new Error(`Test created: ${name}`);
+
+    this.checkName(name);
+
+    const parts = name.split(kPathSeparator);
+    for (const p of parts) {
+      assert(validQueryPart.test(p), `Invalid test name part ${p}; must match ${validQueryPart}`);
+    }
+
+    const test = new TestBuilder(parts, this.fixture, testCreationStack);
+    this.tests.push(test);
+    return test;
+  }
+
+  validate() {
+    for (const test of this.tests) {
+      test.validate();
+    }
+  }
+}
+
+class TestBuilder {
+  testCases = undefined;
+  batchSize = 0;
+
+  constructor(testPath, fixture, testCreationStack) {
+    this.testPath = testPath;
+    this.isUnimplemented = false;
+    this.fixture = fixture;
+    this.testCreationStack = testCreationStack;
+  }
+
+  desc(description) {
+    this.description = description.trim();
+    return this;
+  }
+
+  specURL(url) {
+    return this;
+  }
+
+  beforeAllSubcases(fn) {
+    assert(this.beforeFn === undefined);
+    this.beforeFn = fn;
+    return this;
+  }
+
+  fn(fn) {
+    // MAINTENANCE_TODO: add "TODO" if there's no description? (and make sure it only ends up on
+    // actual tests, not on test parents in the tree, which is what happens if you do it here, not
+    // sure why)
+    assert(this.testFn === undefined);
+    this.testFn = fn;
+  }
+
+  batch(b) {
+    this.batchSize = b;
+    return this;
+  }
+
+  unimplemented() {
+    assert(this.testFn === undefined);
+
+    this.description =
+      (this.description ? this.description + '\n\n' : '') + 'TODO: .unimplemented()';
+    this.isUnimplemented = true;
+
+    this.testFn = () => {
+      throw new SkipTestCase('test unimplemented');
+    };
+  }
+
+  validate() {
+    const testPathString = this.testPath.join(kPathSeparator);
+    assert(this.testFn !== undefined, () => {
+      let s = `Test is missing .fn(): ${testPathString}`;
+      if (this.testCreationStack.stack) {
+        s += `\n-> test created at:\n${this.testCreationStack.stack}`;
+      }
+      return s;
+    });
+
+    if (this.testCases === undefined) {
+      return;
+    }
+
+    const seen = new Set();
+    for (const [caseParams, subcases] of builderIterateCasesWithSubcases(this.testCases)) {
+      for (const subcaseParams of subcases ?? [{}]) {
+        const params = mergeParams(caseParams, subcaseParams);
+        assert(this.batchSize === 0 || !('batch__' in params));
+
+        // stringifyPublicParams also checks for invalid params values
+        const testcaseString = stringifyPublicParams(params);
+
+        // A (hopefully) unique representation of a params value.
+        const testcaseStringUnique = stringifyPublicParamsUniquely(params);
+        assert(
+          !seen.has(testcaseStringUnique),
+          `Duplicate public test case params for test ${testPathString}: ${testcaseString}`
+        );
+
+        seen.add(testcaseStringUnique);
+      }
+    }
+  }
+
+  params(cases) {
+    assert(this.testCases === undefined, 'test case is already parameterized');
+    if (cases instanceof Function) {
+      this.testCases = cases(kUnitCaseParamsBuilder);
+    } else {
+      this.testCases = cases;
+    }
+    return this;
+  }
+
+  paramsSimple(cases) {
+    assert(this.testCases === undefined, 'test case is already parameterized');
+    this.testCases = kUnitCaseParamsBuilder.combineWithParams(cases);
+    return this;
+  }
+
+  paramsSubcasesOnly(subcases) {
+    if (subcases instanceof Function) {
+      return this.params(subcases(kUnitCaseParamsBuilder.beginSubcases()));
+    } else {
+      return this.params(kUnitCaseParamsBuilder.beginSubcases().combineWithParams(subcases));
+    }
+  }
+
+  *iterate() {
+    assert(this.testFn !== undefined, 'No test function (.fn()) for test');
+    this.testCases ??= kUnitCaseParamsBuilder;
+    for (const [caseParams, subcases] of builderIterateCasesWithSubcases(this.testCases)) {
+      if (this.batchSize === 0 || subcases === undefined) {
+        yield new RunCaseSpecific(
+          this.testPath,
+          caseParams,
+          this.isUnimplemented,
+          subcases,
+          this.fixture,
+          this.testFn,
+          this.beforeFn,
+          this.testCreationStack
+        );
+      } else {
+        const subcaseArray = Array.from(subcases);
+        if (subcaseArray.length <= this.batchSize) {
+          yield new RunCaseSpecific(
+            this.testPath,
+            caseParams,
+            this.isUnimplemented,
+            subcaseArray,
+            this.fixture,
+            this.testFn,
+            this.beforeFn,
+            this.testCreationStack
+          );
+        } else {
+          for (let i = 0; i < subcaseArray.length; i = i + this.batchSize) {
+            yield new RunCaseSpecific(
+              this.testPath,
+              { ...caseParams, batch__: i / this.batchSize },
+              this.isUnimplemented,
+              subcaseArray.slice(i, Math.min(subcaseArray.length, i + this.batchSize)),
+              this.fixture,
+              this.testFn,
+              this.beforeFn,
+              this.testCreationStack
+            );
+          }
+        }
+      }
+    }
+  }
+}
+
+class RunCaseSpecific {
+  constructor(
+    testPath,
+    params,
+    isUnimplemented,
+    subcases,
+    fixture,
+    fn,
+    beforeFn,
+    testCreationStack
+  ) {
+    this.id = { test: testPath, params: extractPublicParams(params) };
+    this.isUnimplemented = isUnimplemented;
+    this.params = params;
+    this.subcases = subcases;
+    this.fixture = fixture;
+    this.fn = fn;
+    this.beforeFn = beforeFn;
+    this.testCreationStack = testCreationStack;
+  }
+
+  async runTest(rec, sharedState, params, throwSkip, expectedStatus) {
+    try {
+      rec.beginSubCase();
+      if (expectedStatus === 'skip') {
+        throw new SkipTestCase('Skipped by expectations');
+      }
+
+      const inst = new this.fixture(sharedState, rec, params);
+      try {
+        await inst.init();
+        await this.fn(inst);
+      } finally {
+        // Runs as long as constructor succeeded, even if initialization or the test failed.
+        await inst.finalize();
+      }
+    } catch (ex) {
+      // There was an exception from constructor, init, test, or finalize.
+      // An error from init or test may have been a SkipTestCase.
+      // An error from finalize may have been an eventualAsyncExpectation failure
+      // or unexpected validation/OOM error from the GPUDevice.
+      if (throwSkip && ex instanceof SkipTestCase) {
+        throw ex;
+      }
+      rec.threw(ex);
+    } finally {
+      try {
+        rec.endSubCase(expectedStatus);
+      } catch (ex) {
+        assert(ex instanceof UnexpectedPassError);
+        ex.message = `Testcase passed unexpectedly.`;
+        ex.stack = this.testCreationStack.stack;
+        rec.warn(ex);
+      }
+    }
+  }
+
+  async run(rec, selfQuery, expectations) {
+    const getExpectedStatus = selfQueryWithSubParams => {
+      let didSeeFail = false;
+      for (const exp of expectations) {
+        const ordering = compareQueries(exp.query, selfQueryWithSubParams);
+        if (ordering === Ordering.Unordered || ordering === Ordering.StrictSubset) {
+          continue;
+        }
+
+        switch (exp.expectation) {
+          // Skip takes precedence. If there is any expectation indicating a skip,
+          // signal it immediately.
+          case 'skip':
+            return 'skip';
+          case 'fail':
+            // Otherwise, indicate that we might expect a failure.
+            didSeeFail = true;
+            break;
+          default:
+            unreachable();
+        }
+      }
+      return didSeeFail ? 'fail' : 'pass';
+    };
+
+    const { testHeartbeatCallback, maxSubcasesInFlight } = globalTestConfig;
+    try {
+      rec.start();
+      const sharedState = this.fixture.MakeSharedState(rec, this.params);
+      try {
+        await sharedState.init();
+        if (this.beforeFn) {
+          await this.beforeFn(sharedState);
+        }
+        await sharedState.postInit();
+        testHeartbeatCallback();
+
+        let allPreviousSubcasesFinalizedPromise = Promise.resolve();
+        if (this.subcases) {
+          let totalCount = 0;
+          let skipCount = 0;
+
+          // If there are too many subcases in flight, starting the next subcase will register
+          // `resolvePromiseBlockingSubcase` and wait until `subcaseFinishedCallback` is called.
+          let subcasesInFlight = 0;
+          let resolvePromiseBlockingSubcase = undefined;
+          const subcaseFinishedCallback = () => {
+            subcasesInFlight -= 1;
+            // If there is any subcase waiting on a previous subcase to finish,
+            // unblock it now, and clear the resolve callback.
+            if (resolvePromiseBlockingSubcase) {
+              resolvePromiseBlockingSubcase();
+              resolvePromiseBlockingSubcase = undefined;
+            }
+          };
+
+          for (const subParams of this.subcases) {
+            // Make a recorder that will defer all calls until `allPreviousSubcasesFinalizedPromise`
+            // resolves. Waiting on `allPreviousSubcasesFinalizedPromise` ensures that
+            // logs from all the previous subcases have been flushed before flushing new logs.
+            const subcasePrefix = 'subcase: ' + stringifyPublicParams(subParams);
+            const subRec = new Proxy(rec, {
+              get: (target, k) => {
+                const prop = TestCaseRecorder.prototype[k];
+                if (typeof prop === 'function') {
+                  testHeartbeatCallback();
+                  return function (...args) {
+                    void allPreviousSubcasesFinalizedPromise.then(() => {
+                      // Prepend the subcase name to all error messages.
+                      for (const arg of args) {
+                        if (arg instanceof Error) {
+                          try {
+                            arg.message = subcasePrefix + '\n' + arg.message;
+                          } catch {
+                            // If that fails (e.g. on DOMException), try to put it in the stack:
+                            let stack = subcasePrefix;
+                            if (arg.stack) stack += '\n' + arg.stack;
+                            try {
+                              arg.stack = stack;
+                            } catch {
+                              // If that fails too, just silence it.
+                            }
+                          }
+                        }
+                      }
+
+                      const rv = prop.apply(target, args);
+                      // Because this proxy executes functions in a deferred manner,
+                      // it should never be used for functions that need to return a value.
+                      assert(rv === undefined);
+                    });
+                  };
+                }
+                return prop;
+              },
+            });
+
+            const params = mergeParams(this.params, subParams);
+            const subcaseQuery = new TestQuerySingleCase(
+              selfQuery.suite,
+              selfQuery.filePathParts,
+              selfQuery.testPathParts,
+              params
+            );
+
+            // Limit the maximum number of subcases in flight.
+            if (subcasesInFlight >= maxSubcasesInFlight) {
+              await new Promise(resolve => {
+                // There should only be one subcase waiting at a time.
+                assert(resolvePromiseBlockingSubcase === undefined);
+                resolvePromiseBlockingSubcase = resolve;
+              });
+            }
+
+            subcasesInFlight += 1;
+            // Runs async without waiting so that subsequent subcases can start.
+            // All finalization steps will be waited on at the end of the testcase.
+            const finalizePromise = this.runTest(
+              subRec,
+              sharedState,
+              params,
+              /* throwSkip */ true,
+              getExpectedStatus(subcaseQuery)
+            )
+              .then(() => {
+                subRec.info(new Error('OK'));
+              })
+              .catch(ex => {
+                if (ex instanceof SkipTestCase) {
+                  // Convert SkipTestCase to info messages
+                  ex.message = 'subcase skipped: ' + ex.message;
+                  subRec.info(ex);
+                  ++skipCount;
+                } else {
+                  // Since we are catching all error inside runTest(), this should never happen
+                  subRec.threw(ex);
+                }
+              })
+              .finally(subcaseFinishedCallback);
+
+            allPreviousSubcasesFinalizedPromise = allPreviousSubcasesFinalizedPromise.then(
+              () => finalizePromise
+            );
+
+            ++totalCount;
+          }
+
+          // Wait for all subcases to finalize and report their results.
+          await allPreviousSubcasesFinalizedPromise;
+
+          if (skipCount === totalCount) {
+            rec.skipped(new SkipTestCase('all subcases were skipped'));
+          }
+        } else {
+          await this.runTest(
+            rec,
+            sharedState,
+            this.params,
+            /* throwSkip */ false,
+            getExpectedStatus(selfQuery)
+          );
+        }
+      } finally {
+        testHeartbeatCallback();
+        // Runs as long as the shared state constructor succeeded, even if initialization or a test failed.
+        await sharedState.finalize();
+        testHeartbeatCallback();
+      }
+    } catch (ex) {
+      // There was an exception from sharedState/fixture constructor, init, beforeFn, or test.
+      // An error from beforeFn may have been SkipTestCase.
+      // An error from finalize may have been an eventualAsyncExpectation failure
+      // or unexpected validation/OOM error from the GPUDevice.
+      rec.threw(ex);
+    } finally {
+      rec.finish();
+    }
+  }
+}

tests/wpt/webgpu/tests/webgpu/common/internal/test_suite_listing.js

@@ -1,3 +1,3 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/
+ **/ export {};

tests/wpt/webgpu/tests/webgpu/common/internal/tree.js

@@ -1,18 +1,8 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
+ **/ import { globalTestConfig } from '../framework/test_config.js';
+import { assert, now } from '../util/util.js';
+
 import { compareQueries, Ordering } from './query/compare.js';
 import {
   TestQueryMultiCase,
@@ -22,8 +12,7 @@ import {
 } from './query/query.js';
 import { kBigSeparator, kWildcard, kPathSeparator, kParamSeparator } from './query/separators.js';
 import { stringifySingleParam } from './query/stringify_params.js';
-
-import { assert } from './util/util.js';
+import { StacklessError } from './util.js';
 
 // `loadTreeForQuery()` loads a TestTree for a given queryToLoad.
 // The resulting tree is a linked-list all the way from `suite:*` to queryToLoad,
@@ -33,13 +22,13 @@ import { assert } from './util/util.js';
 // A node is considered "collapsible" if none of the subqueriesToExpand is a StrictSubset
 // of that node.
 //
-// In WebKit/Blink-style web_tests, an expectation file marks individual cts.html "variants" as
-// "Failure", "Crash", etc.
-// By passing in the list of expectations as the subqueriesToExpand, we can programmatically
-// subdivide the cts.html "variants" list to be able to implement arbitrarily-fine suppressions
-// (instead of having to suppress entire test files, which would lose a lot of coverage).
+// In WebKit/Blink-style web_tests, an expectation file marks individual cts.https.html "variants
+// as "Failure", "Crash", etc. By passing in the list of expectations as the subqueriesToExpand,
+// we can programmatically subdivide the cts.https.html "variants" list to be able to implement
+// arbitrarily-fine suppressions (instead of having to suppress entire test files, which would
+// lose a lot of coverage).
 //
-// `iterateCollapsedQueries()` produces the list of queries for the variants list.
+// `iterateCollapsedNodes()` produces the list of queries for the variants list.
 //
 // Though somewhat complicated, this system has important benefits:
 //   - Avoids having to suppress entire test files, which would cause large test coverage loss.
@@ -50,13 +39,42 @@ import { assert } from './util/util.js';
 //     about expectation granularity.
 
 export class TestTree {
-  constructor(root) {
-    _defineProperty(this, 'root', void 0);
+  /**
+   * The `queryToLoad` that this test tree was created for.
+   * Test trees are always rooted at `suite:*`, but they only contain nodes that fit
+   * within `forQuery`.
+   *
+   * This is used for `iterateCollapsedNodes` which only starts collapsing at the next
+   * `TestQueryLevel` after `forQuery`.
+   */
+
+  constructor(forQuery, root) {
+    this.forQuery = forQuery;
+    TestTree.propagateCounts(root);
     this.root = root;
+    assert(
+      root.query.level === 1 && root.query.depthInLevel === 0,
+      'TestTree root must be the root (suite:*)'
+    );
   }
 
-  iterateCollapsedQueries() {
-    return TestTree.iterateSubtreeCollapsedQueries(this.root);
+  /**
+   * Iterate through the leaves of a version of the tree which has been pruned to exclude
+   * subtrees which:
+   * - are at a deeper `TestQueryLevel` than `this.forQuery`, and
+   * - were not a `Ordering.StrictSubset` of any of the `subqueriesToExpand` during tree creation.
+   */
+  iterateCollapsedNodes({
+    includeIntermediateNodes = false,
+    includeEmptySubtrees = false,
+    alwaysExpandThroughLevel,
+  }) {
+    const expandThroughLevel = Math.max(this.forQuery.level, alwaysExpandThroughLevel);
+    return TestTree.iterateSubtreeNodes(this.root, {
+      includeIntermediateNodes,
+      includeEmptySubtrees,
+      expandThroughLevel,
+    });
   }
 
   iterateLeaves() {
@@ -64,15 +82,14 @@ export class TestTree {
   }
 
   /**
-   * If a parent and its child are at different levels, then
-   * generally the parent has only one child, i.e.:
+   * Dissolve nodes which have only one child, e.g.:
    *   a,* { a,b,* { a,b:* { ... } } }
-   * Collapse that down into:
+   * collapses down into:
    *   a,* { a,b:* { ... } }
    * which is less needlessly verbose when displaying the tree in the standalone runner.
    */
-  dissolveLevelBoundaries() {
-    const newRoot = dissolveLevelBoundaries(this.root);
+  dissolveSingleChildTrees() {
+    const newRoot = dissolveSingleChildTrees(this.root);
     assert(newRoot === this.root);
   }
 
@@ -80,12 +97,24 @@ export class TestTree {
     return TestTree.subtreeToString('(root)', this.root, '');
   }
 
-  static *iterateSubtreeCollapsedQueries(subtree) {
+  static *iterateSubtreeNodes(subtree, opts) {
+    if (opts.includeIntermediateNodes) {
+      yield subtree;
+    }
+
     for (const [, child] of subtree.children) {
-      if ('children' in child && !child.collapsible) {
-        yield* TestTree.iterateSubtreeCollapsedQueries(child);
+      if ('children' in child) {
+        // Is a subtree
+        const collapsible = child.collapsible && child.query.level > opts.expandThroughLevel;
+        if (child.children.size > 0 && !collapsible) {
+          yield* TestTree.iterateSubtreeNodes(child, opts);
+        } else if (child.children.size > 0 || opts.includeEmptySubtrees) {
+          // Don't yield empty subtrees (e.g. files with no tests) unless includeEmptySubtrees
+          yield child;
+        }
       } else {
-        yield child.query;
+        // Is a leaf
+        yield child;
       }
     }
   }
@@ -100,9 +129,33 @@ export class TestTree {
     }
   }
 
+  /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */
+  static propagateCounts(subtree) {
+    subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 };
+    for (const [, child] of subtree.children) {
+      if ('children' in child) {
+        const counts = TestTree.propagateCounts(child);
+        subtree.subtreeCounts.tests += counts.tests;
+        subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO;
+      }
+    }
+    return subtree.subtreeCounts;
+  }
+
+  /** Displays counts in the format `(Nodes with TODOs) / (Total test count)`. */
+  static countsToString(tree) {
+    if (tree.subtreeCounts) {
+      return `${tree.subtreeCounts.nodesWithTODO} / ${tree.subtreeCounts.tests}`;
+    } else {
+      return '';
+    }
+  }
+
   static subtreeToString(name, tree, indent) {
     const collapsible = 'run' in tree ? '>' : tree.collapsible ? '+' : '-';
-    let s = indent + `${collapsible} ${JSON.stringify(name)} => ${tree.query}`;
+    let s =
+      indent +
+      `${collapsible} ${TestTree.countsToString(tree)} ${JSON.stringify(name)} => ${tree.query}`;
     if ('children' in tree) {
       if (tree.description !== undefined) {
         s += `\n${indent}  | ${JSON.stringify(tree.description)}`;
@@ -116,7 +169,8 @@ export class TestTree {
   }
 }
 
-// TODO: Consider having subqueriesToExpand actually impact the depth-order of params in the tree.
+// MAINTENANCE_TODO: Consider having subqueriesToExpand actually impact the depth-order of params
+// in the tree.
 export async function loadTreeForQuery(loader, queryToLoad, subqueriesToExpand) {
   const suite = queryToLoad.suite;
   const specs = await loader.listing(suite);
@@ -140,13 +194,14 @@ export async function loadTreeForQuery(loader, queryToLoad, subqueriesToExpand)
   // L3 =  case-level, e.g. suite:a,b:c,d:
   let foundCase = false;
   // L0 is suite:*
-  const subtreeL0 = makeTreeForSuite(suite);
-  isCollapsible(subtreeL0.query); // mark seenSubqueriesToExpand
+  const subtreeL0 = makeTreeForSuite(suite, isCollapsible);
+
+  const imports_start = now();
+  const pEntriesWithImports = []; // Promise<entry with importedSpec>[]
   for (const entry of specs) {
     if (entry.file.length === 0 && 'readme' in entry) {
       // Suite-level readme.
-      assert(subtreeL0.description === undefined);
-      subtreeL0.description = entry.readme.trim();
+      setSubtreeDescriptionAndCountTODOs(subtreeL0, entry.readme);
       continue;
     }
 
@@ -159,6 +214,34 @@ export async function loadTreeForQuery(loader, queryToLoad, subqueriesToExpand)
       }
     }
 
+    // We're going to be fetching+importing a bunch of things, so do it in async.
+    const pEntryWithImport = (async () => {
+      if ('readme' in entry) {
+        return entry;
+      } else {
+        return {
+          ...entry,
+          importedSpec: await loader.importSpecFile(queryToLoad.suite, entry.file),
+        };
+      }
+    })();
+
+    const kForceSerialImporting = false;
+    if (kForceSerialImporting) {
+      await pEntryWithImport;
+    }
+    pEntriesWithImports.push(pEntryWithImport);
+  }
+
+  const entriesWithImports = await Promise.all(pEntriesWithImports);
+  if (globalTestConfig.frameworkDebugLog) {
+    const imported_time = performance.now() - imports_start;
+    globalTestConfig.frameworkDebugLog(
+      `Imported importedSpecFiles[${entriesWithImports.length}] in ${imported_time}ms.`
+    );
+  }
+
+  for (const entry of entriesWithImports) {
     if ('readme' in entry) {
       // Entry is a README that is an ancestor or descendant of the query.
       // (It's included for display in the standalone runner.)
@@ -166,64 +249,102 @@ export async function loadTreeForQuery(loader, queryToLoad, subqueriesToExpand)
       // readmeSubtree is suite:a,b,*
       // (This is always going to dedup with a file path, if there are any test spec files under
       // the directory that has the README).
-      const readmeSubtree = addSubtreeForDirPath(subtreeL0, entry.file);
+      const readmeSubtree = addSubtreeForDirPath(subtreeL0, entry.file, isCollapsible);
 
-      assert(readmeSubtree.description === undefined);
-      readmeSubtree.description = entry.readme.trim();
+      setSubtreeDescriptionAndCountTODOs(readmeSubtree, entry.readme);
       continue;
     }
+
     // Entry is a spec file.
-
-    const spec = await loader.importSpecFile(queryToLoad.suite, entry.file);
-    const description = spec.description.trim();
+    const spec = entry.importedSpec;
     // subtreeL1 is suite:a,b:*
-    const subtreeL1 = addSubtreeForFilePath(subtreeL0, entry.file, description, isCollapsible);
+    const subtreeL1 = addSubtreeForFilePath(subtreeL0, entry.file, isCollapsible);
 
-    // TODO: If tree generation gets too slow, avoid actually iterating the cases in a file
-    // if there's no need to (based on the subqueriesToExpand).
+    setSubtreeDescriptionAndCountTODOs(subtreeL1, spec.description);
+
+    let groupHasTests = false;
     for (const t of spec.g.iterate()) {
+      groupHasTests = true;
       {
-        const queryL3 = new TestQuerySingleCase(suite, entry.file, t.id.test, t.id.params);
-        const orderingL3 = compareQueries(queryL3, queryToLoad);
-        if (orderingL3 === Ordering.Unordered || orderingL3 === Ordering.StrictSuperset) {
-          // Case is not matched by this query.
+        const queryL2 = new TestQueryMultiCase(suite, entry.file, t.testPath, {});
+        const orderingL2 = compareQueries(queryL2, queryToLoad);
+        if (orderingL2 === Ordering.Unordered) {
+          // Test path is not matched by this query.
           continue;
         }
       }
 
       // subtreeL2 is suite:a,b:c,d:*
-      const subtreeL2 = addSubtreeForTestPath(subtreeL1, t.id.test, isCollapsible);
+      const subtreeL2 = addSubtreeForTestPath(
+        subtreeL1,
+        t.testPath,
+        t.testCreationStack,
+        isCollapsible
+      );
 
-      // Leaf for case is suite:a,b:c,d:x=1;y=2
-      addLeafForCase(subtreeL2, t, isCollapsible);
+      // This is 1 test. Set tests=1 then count TODOs.
+      subtreeL2.subtreeCounts ??= { tests: 1, nodesWithTODO: 0 };
+      if (t.description) setSubtreeDescriptionAndCountTODOs(subtreeL2, t.description);
 
-      foundCase = true;
+      // MAINTENANCE_TODO: If tree generation gets too slow, avoid actually iterating the cases in a
+      // file if there's no need to (based on the subqueriesToExpand).
+      for (const c of t.iterate()) {
+        {
+          const queryL3 = new TestQuerySingleCase(suite, entry.file, c.id.test, c.id.params);
+          const orderingL3 = compareQueries(queryL3, queryToLoad);
+          if (orderingL3 === Ordering.Unordered || orderingL3 === Ordering.StrictSuperset) {
+            // Case is not matched by this query.
+            continue;
+          }
+        }
+
+        // Leaf for case is suite:a,b:c,d:x=1;y=2
+        addLeafForCase(subtreeL2, c, isCollapsible);
+
+        foundCase = true;
+      }
+    }
+    if (!groupHasTests && !subtreeL1.subtreeCounts) {
+      throw new StacklessError(
+        `${subtreeL1.query} has no tests - it must have "TODO" in its description`
+      );
     }
   }
 
   for (const [i, sq] of subqueriesToExpandEntries) {
-    const seen = seenSubqueriesToExpand[i];
-    assert(
-      seen,
-      `subqueriesToExpand entry did not match anything \
-(can happen due to overlap with another subquery): ${sq.toString()}`
-    );
+    const subquerySeen = seenSubqueriesToExpand[i];
+    if (!subquerySeen) {
+      throw new StacklessError(
+        `subqueriesToExpand entry did not match anything \
+(could be wrong, or could be redundant with a previous subquery):\n  ${sq.toString()}`
+      );
+    }
   }
-  assert(foundCase, 'Query does not match any cases');
+  assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`);
 
-  return new TestTree(subtreeL0);
+  return new TestTree(queryToLoad, subtreeL0);
 }
 
-function makeTreeForSuite(suite) {
+function setSubtreeDescriptionAndCountTODOs(subtree, description) {
+  assert(subtree.description === undefined);
+  subtree.description = description.trim();
+  subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 };
+  if (subtree.description.indexOf('TODO') !== -1) {
+    subtree.subtreeCounts.nodesWithTODO++;
+  }
+}
+
+function makeTreeForSuite(suite, isCollapsible) {
+  const query = new TestQueryMultiFile(suite, []);
   return {
     readableRelativeName: suite + kBigSeparator,
-    query: new TestQueryMultiFile(suite, []),
+    query,
     children: new Map(),
-    collapsible: false,
+    collapsible: isCollapsible(query),
   };
 }
 
-function addSubtreeForDirPath(tree, file) {
+function addSubtreeForDirPath(tree, file, isCollapsible) {
   const subqueryFile = [];
   // To start, tree is suite:*
   // This loop goes from that -> suite:a,* -> suite:a,b,*
@@ -231,16 +352,20 @@ function addSubtreeForDirPath(tree, file) {
     subqueryFile.push(part);
     tree = getOrInsertSubtree(part, tree, () => {
       const query = new TestQueryMultiFile(tree.query.suite, subqueryFile);
-      return { readableRelativeName: part + kPathSeparator + kWildcard, query, collapsible: false };
+      return {
+        readableRelativeName: part + kPathSeparator + kWildcard,
+        query,
+        collapsible: isCollapsible(query),
+      };
     });
   }
   return tree;
 }
 
-function addSubtreeForFilePath(tree, file, description, checkCollapsible) {
+function addSubtreeForFilePath(tree, file, isCollapsible) {
   // To start, tree is suite:*
   // This goes from that -> suite:a,* -> suite:a,b,*
-  tree = addSubtreeForDirPath(tree, file);
+  tree = addSubtreeForDirPath(tree, file, isCollapsible);
   // This goes from that -> suite:a,b:*
   const subtree = getOrInsertSubtree('', tree, () => {
     const query = new TestQueryMultiTest(tree.query.suite, tree.query.filePathParts, []);
@@ -248,14 +373,13 @@ function addSubtreeForFilePath(tree, file, description, checkCollapsible) {
     return {
       readableRelativeName: file[file.length - 1] + kBigSeparator + kWildcard,
       query,
-      description,
-      collapsible: checkCollapsible(query),
+      collapsible: isCollapsible(query),
     };
   });
   return subtree;
 }
 
-function addSubtreeForTestPath(tree, test, isCollapsible) {
+function addSubtreeForTestPath(tree, test, testCreationStack, isCollapsible) {
   const subqueryTest = [];
   // To start, tree is suite:a,b:*
   // This loop goes from that -> suite:a,b:c,* -> suite:a,b:c,d,*
@@ -289,6 +413,7 @@ function addSubtreeForTestPath(tree, test, isCollapsible) {
       readableRelativeName: subqueryTest[subqueryTest.length - 1] + kBigSeparator + kWildcard,
       kWildcard,
       query,
+      testCreationStack,
       collapsible: isCollapsible(query),
     };
   });
@@ -347,32 +472,34 @@ function getOrInsertSubtree(key, parent, createSubtree) {
 }
 
 function insertLeaf(parent, query, t) {
-  const key = '';
   const leaf = {
     readableRelativeName: readableNameForCase(query),
     query,
-    run: rec => t.run(rec),
+    run: (rec, expectations) => t.run(rec, query, expectations || []),
+    isUnimplemented: t.isUnimplemented,
   };
 
-  assert(!parent.children.has(key));
+  // This is a leaf (e.g. s:f:t:x=1;* -> s:f:t:x=1). The key is always ''.
+  const key = '';
+  assert(!parent.children.has(key), `Duplicate testcase: ${query}`);
   parent.children.set(key, leaf);
 }
 
-function dissolveLevelBoundaries(tree) {
+function dissolveSingleChildTrees(tree) {
   if ('children' in tree) {
-    if (tree.children.size === 1 && tree.description === undefined) {
+    const shouldDissolveThisTree =
+      tree.children.size === 1 && tree.query.depthInLevel !== 0 && tree.description === undefined;
+    if (shouldDissolveThisTree) {
       // Loops exactly once
       for (const [, child] of tree.children) {
-        if (child.query.level > tree.query.level) {
-          const newtree = dissolveLevelBoundaries(child);
-
-          return newtree;
-        }
+        // Recurse on child
+        return dissolveSingleChildTrees(child);
       }
     }
 
     for (const [k, child] of tree.children) {
-      const newChild = dissolveLevelBoundaries(child);
+      // Recurse on each child
+      const newChild = dissolveSingleChildTrees(child);
       if (newChild !== child) {
         tree.children.set(k, newChild);
       }

tests/wpt/webgpu/tests/webgpu/common/internal/util.js

@@ -0,0 +1,11 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ /**
+ * Error without a stack, which can be used to fatally exit from `tool/` scripts with a
+ * user-friendly message (and no confusing stack).
+ */ export class StacklessError extends Error {
+  constructor(message) {
+    super(message);
+    this.stack = undefined;
+  }
+}

tests/wpt/webgpu/tests/webgpu/common/internal/version.js

@@ -0,0 +1,3 @@
+// AUTO-GENERATED - DO NOT EDIT. See tools/gen_version.
+
+export const version = '480edec387e8cd5bf5934680050c59a3f7a01438';

tests/wpt/webgpu/tests/webgpu/common/runtime/helper/options.js

@@ -1,7 +1,18 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ const url = new URL(window.location.toString());
-export function optionEnabled(opt) {
-  const val = url.searchParams.get(opt);
+ **/ let windowURL = undefined;
+function getWindowURL() {
+  if (windowURL === undefined) {
+    windowURL = new URL(window.location.toString());
+  }
+  return windowURL;
+}
+
+export function optionEnabled(opt, searchParams = getWindowURL().searchParams) {
+  const val = searchParams.get(opt);
   return val !== null && val !== '0';
 }
+
+export function optionString(opt, searchParams = getWindowURL().searchParams) {
+  return searchParams.get(opt) || '';
+}

tests/wpt/webgpu/tests/webgpu/common/runtime/helper/test_worker-worker.js

@@ -1,26 +1,36 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { DefaultTestFileLoader } from '../../framework/file_loader.js';
-import { Logger } from '../../framework/logging/logger.js';
-import { parseQuery } from '../../framework/query/parseQuery.js';
-import { assert } from '../../framework/util/util.js';
+ **/ import { setBaseResourcePath } from '../../framework/resources.js';
+import { DefaultTestFileLoader } from '../../internal/file_loader.js';
+import { Logger } from '../../internal/logging/logger.js';
+import { parseQuery } from '../../internal/query/parseQuery.js';
 
-// should be DedicatedWorkerGlobalScope
+import { setDefaultRequestAdapterOptions } from '../../util/navigator_gpu.js';
+import { assert } from '../../util/util.js';
+
+// Should be DedicatedWorkerGlobalScope, but importing lib "webworker" conflicts with lib "dom".
 
 const loader = new DefaultTestFileLoader();
 
+setBaseResourcePath('../../../resources');
+
 self.onmessage = async ev => {
   const query = ev.data.query;
+  const expectations = ev.data.expectations;
+  const defaultRequestAdapterOptions = ev.data.defaultRequestAdapterOptions;
   const debug = ev.data.debug;
 
-  const log = new Logger(debug);
+  setDefaultRequestAdapterOptions(defaultRequestAdapterOptions);
+
+  Logger.globalDebugMode = debug;
+  const log = new Logger();
 
   const testcases = Array.from(await loader.loadCases(parseQuery(query)));
   assert(testcases.length === 1, 'worker query resulted in != 1 cases');
 
   const testcase = testcases[0];
   const [rec, result] = log.record(testcase.query.toString());
-  await testcase.run(rec);
+  await testcase.run(rec, expectations);
 
   self.postMessage({ query, result });
 };

tests/wpt/webgpu/tests/webgpu/common/runtime/helper/test_worker.js

@@ -1,25 +1,13 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ function _defineProperty(obj, key, value) {
-  if (key in obj) {
-    Object.defineProperty(obj, key, {
-      value: value,
-      enumerable: true,
-      configurable: true,
-      writable: true,
-    });
-  } else {
-    obj[key] = value;
-  }
-  return obj;
-}
-import { LogMessageWithStack } from '../../framework/logging/log_message.js';
+ **/ import { LogMessageWithStack } from '../../internal/logging/log_message.js';
+
+import { getDefaultRequestAdapterOptions } from '../../util/navigator_gpu.js';
 
 export class TestWorker {
+  resolvers = new Map();
+
   constructor(debug) {
-    _defineProperty(this, 'debug', void 0);
-    _defineProperty(this, 'worker', void 0);
-    _defineProperty(this, 'resolvers', new Map());
     this.debug = debug;
 
     const selfPath = import.meta.url;
@@ -36,13 +24,18 @@ export class TestWorker {
       }
       this.resolvers.get(query)(result);
 
-      // TODO(kainino0x): update the Logger with this result (or don't have a logger and update the
-      // entire results JSON somehow at some point).
+      // MAINTENANCE_TODO(kainino0x): update the Logger with this result (or don't have a logger and
+      // update the entire results JSON somehow at some point).
     };
   }
 
-  async run(rec, query) {
-    this.worker.postMessage({ query, debug: this.debug });
+  async run(rec, query, expectations = []) {
+    this.worker.postMessage({
+      query,
+      expectations,
+      debug: this.debug,
+      defaultRequestAdapterOptions: getDefaultRequestAdapterOptions(),
+    });
     const workerResult = await new Promise(resolve => {
       this.resolvers.set(query, resolve);
     });

tests/wpt/webgpu/tests/webgpu/common/runtime/wpt.js

@@ -1,58 +1,73 @@
 /**
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
- **/ import { DefaultTestFileLoader } from '../framework/file_loader.js';
-import { Logger } from '../framework/logging/logger.js';
-import { parseQuery } from '../framework/query/parseQuery.js';
-
-import { AsyncMutex } from '../framework/util/async_mutex.js';
-import { assert } from '../framework/util/util.js';
+ **/ // Implements the wpt-embedded test runner (see also: wpt/cts.https.html).
+import { globalTestConfig } from '../framework/test_config.js';
+import { DefaultTestFileLoader } from '../internal/file_loader.js';
+import { prettyPrintLog } from '../internal/logging/log_message.js';
+import { Logger } from '../internal/logging/logger.js';
+import { parseQuery } from '../internal/query/parseQuery.js';
+import { parseExpectationsForTestQuery, relativeQueryString } from '../internal/query/query.js';
+import { assert } from '../util/util.js';
 
 import { optionEnabled } from './helper/options.js';
 import { TestWorker } from './helper/test_worker.js';
 
-(async () => {
+// testharness.js API (https://web-platform-tests.org/writing-tests/testharness-api.html)
+
+setup({
+  // It's convenient for us to asynchronously add tests to the page. Prevent done() from being
+  // called implicitly when the page is finished loading.
+  explicit_done: true,
+});
+
+void (async () => {
+  const workerEnabled = optionEnabled('worker');
+  const worker = workerEnabled ? new TestWorker(false) : undefined;
+
+  globalTestConfig.unrollConstEvalLoops = optionEnabled('unroll_const_eval_loops');
+
+  const failOnWarnings =
+    typeof shouldWebGPUCTSFailOnWarnings !== 'undefined' && (await shouldWebGPUCTSFailOnWarnings);
+
   const loader = new DefaultTestFileLoader();
   const qs = new URLSearchParams(window.location.search).getAll('q');
   assert(qs.length === 1, 'currently, there must be exactly one ?q=');
-  const testcases = await loader.loadCases(parseQuery(qs[0]));
+  const filterQuery = parseQuery(qs[0]);
+  const testcases = await loader.loadCases(filterQuery);
 
-  await addWPTTests(testcases);
-})();
+  const expectations =
+    typeof loadWebGPUExpectations !== 'undefined'
+      ? parseExpectationsForTestQuery(
+          await loadWebGPUExpectations,
+          filterQuery,
+          new URL(window.location.href)
+        )
+      : [];
 
-// Note: `async_test`s must ALL be added within the same task. This function *must not* be async.
-function addWPTTests(testcases) {
-  const worker = optionEnabled('worker') ? new TestWorker(false) : undefined;
-
-  const log = new Logger(false);
-  const mutex = new AsyncMutex();
-  const running = [];
+  const log = new Logger();
 
   for (const testcase of testcases) {
     const name = testcase.query.toString();
-    const wpt_fn = function () {
-      const p = mutex.with(async () => {
-        const [rec, res] = log.record(name);
-        if (worker) {
-          await worker.run(rec, name);
-        } else {
-          await testcase.run(rec);
-        }
+    // For brevity, display the case name "relative" to the ?q= path.
+    const shortName = relativeQueryString(filterQuery, testcase.query) || '(case)';
 
-        this.step(() => {
-          // Unfortunately, it seems not possible to surface any logs for warn/skip.
-          if (res.status === 'fail') {
-            throw (res.logs || []).map(s => s.toJSON()).join('\n\n');
-          }
-        });
-        this.done();
-      });
+    const wpt_fn = async () => {
+      const [rec, res] = log.record(name);
+      if (worker) {
+        await worker.run(rec, name, expectations);
+      } else {
+        await testcase.run(rec, expectations);
+      }
 
-      running.push(p);
-      return p;
+      // Unfortunately, it seems not possible to surface any logs for warn/skip.
+      if (res.status === 'fail' || (res.status === 'warn' && failOnWarnings)) {
+        const logs = (res.logs ?? []).map(prettyPrintLog);
+        assert_unreached('\n' + logs.join('\n') + '\n');
+      }
     };
 
-    async_test(wpt_fn, name);
+    promise_test(wpt_fn, shortName);
   }
 
-  return Promise.all(running).then(() => log);
-}
+  done();
+})();

tests/wpt/webgpu/tests/webgpu/common/util/collect_garbage.js

@@ -2,8 +2,13 @@
  * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
  **/ import { resolveOnTimeout } from './util.js';
 
+/**
+ * Attempts to trigger JavaScript garbage collection, either using explicit methods if exposed
+ * (may be available in testing environments with special browser runtime flags set), or using
+ * some weird tricks to incur GC pressure. Adopted from the WebGL CTS.
+ */
 export async function attemptGarbageCollection() {
-  const w = self;
+  const w = globalThis;
   if (w.GCController) {
     w.GCController.collect();
     return;
@@ -19,8 +24,9 @@ export async function attemptGarbageCollection() {
       .getInterface(Components.interfaces.nsIDOMWindowUtils)
       .garbageCollect();
     return;
-  } catch (e) {}
-
+  } catch (e) {
+    // ignore any failure
+  }
   if (w.gc) {
     w.gc();
     return;
@@ -34,7 +40,9 @@ export async function attemptGarbageCollection() {
   let i;
   function gcRec(n) {
     if (n < 1) return;
+
     let temp = { i: 'ab' + i + i / 100000 };
+
     temp = temp + 'foo';
     temp; // dummy use of unused variable
     gcRec(n - 1);

tests/wpt/webgpu/tests/webgpu/common/util/colors.js

@@ -0,0 +1,63 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+/**
+ * The interface used for formatting strings with color metadata.
+ *
+ * Currently Colors will use the 'ansi-colors' module if it can be loaded.
+ * If it cannot be loaded, then the Colors implementation is a straight pass-through.
+ *
+ * Colors may also be a no-op if the current environment does not support colors.
+ */
+export let Colors;
+
+try {
+  Colors = require('ansi-colors');
+} catch {
+  const passthrough = s => s;
+  passthrough.enabled = false;
+  passthrough.reset = passthrough;
+  passthrough.bold = passthrough;
+  passthrough.dim = passthrough;
+  passthrough.italic = passthrough;
+  passthrough.underline = passthrough;
+  passthrough.inverse = passthrough;
+  passthrough.hidden = passthrough;
+  passthrough.strikethrough = passthrough;
+  passthrough.black = passthrough;
+  passthrough.red = passthrough;
+  passthrough.green = passthrough;
+  passthrough.yellow = passthrough;
+  passthrough.blue = passthrough;
+  passthrough.magenta = passthrough;
+  passthrough.cyan = passthrough;
+  passthrough.white = passthrough;
+  passthrough.gray = passthrough;
+  passthrough.grey = passthrough;
+  passthrough.blackBright = passthrough;
+  passthrough.redBright = passthrough;
+  passthrough.greenBright = passthrough;
+  passthrough.yellowBright = passthrough;
+  passthrough.blueBright = passthrough;
+  passthrough.magentaBright = passthrough;
+  passthrough.cyanBright = passthrough;
+  passthrough.whiteBright = passthrough;
+  passthrough.bgBlack = passthrough;
+  passthrough.bgRed = passthrough;
+  passthrough.bgGreen = passthrough;
+  passthrough.bgYellow = passthrough;
+  passthrough.bgBlue = passthrough;
+  passthrough.bgMagenta = passthrough;
+  passthrough.bgCyan = passthrough;
+  passthrough.bgWhite = passthrough;
+  passthrough.bgBlackBright = passthrough;
+  passthrough.bgRedBright = passthrough;
+  passthrough.bgGreenBright = passthrough;
+  passthrough.bgYellowBright = passthrough;
+  passthrough.bgBlueBright = passthrough;
+  passthrough.bgMagentaBright = passthrough;
+  passthrough.bgCyanBright = passthrough;
+  passthrough.bgWhiteBright = passthrough;
+  Colors = passthrough;
+}

tests/wpt/webgpu/tests/webgpu/common/util/data_tables.js

@@ -0,0 +1,29 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+export function keysOf(obj) {
+  return Object.keys(obj);
+}
+
+export function numericKeysOf(obj) {
+  return Object.keys(obj).map(n => Number(n));
+}
+
+/**
+ * Creates an info lookup object from a more nicely-formatted table. See below for examples.
+ *
+ * Note: Using `as const` on the arguments to this function is necessary to infer the correct type.
+ */
+export function makeTable(members, defaults, table) {
+  const result = {};
+  for (const [k, v] of Object.entries(table)) {
+    const item = {};
+    for (let i = 0; i < members.length; ++i) {
+      item[members[i]] = v[i] ?? defaults[i];
+    }
+    result[k] = item;
+  }
+
+  return result;
+}

tests/wpt/webgpu/tests/webgpu/common/util/navigator_gpu.js

@@ -0,0 +1,79 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+import { ErrorWithExtra, assert } from './util.js';
+
+/**
+ * Finds and returns the `navigator.gpu` object (or equivalent, for non-browser implementations).
+ * Throws an exception if not found.
+ */
+function defaultGPUProvider() {
+  assert(
+    typeof navigator !== 'undefined' && navigator.gpu !== undefined,
+    'No WebGPU implementation found'
+  );
+
+  return navigator.gpu;
+}
+
+/**
+ * GPUProvider is a function that creates and returns a new GPU instance.
+ * May throw an exception if a GPU cannot be created.
+ */
+
+let gpuProvider = defaultGPUProvider;
+
+/**
+ * Sets the function to create and return a new GPU instance.
+ */
+export function setGPUProvider(provider) {
+  assert(impl === undefined, 'setGPUProvider() should not be after getGPU()');
+  gpuProvider = provider;
+}
+
+let impl = undefined;
+
+let defaultRequestAdapterOptions;
+
+export function setDefaultRequestAdapterOptions(options) {
+  if (impl) {
+    throw new Error('must call setDefaultRequestAdapterOptions before getGPU');
+  }
+  defaultRequestAdapterOptions = { ...options };
+}
+
+export function getDefaultRequestAdapterOptions() {
+  return defaultRequestAdapterOptions;
+}
+
+/**
+ * Finds and returns the `navigator.gpu` object (or equivalent, for non-browser implementations).
+ * Throws an exception if not found.
+ */
+export function getGPU(recorder) {
+  if (impl) {
+    return impl;
+  }
+
+  impl = gpuProvider();
+
+  if (defaultRequestAdapterOptions) {
+    const oldFn = impl.requestAdapter;
+    impl.requestAdapter = function (options) {
+      const promise = oldFn.call(this, { ...defaultRequestAdapterOptions, ...options });
+      if (recorder) {
+        void promise.then(async adapter => {
+          if (adapter) {
+            const info = await adapter.requestAdapterInfo();
+            const infoString = `Adapter: ${info.vendor} / ${info.architecture} / ${info.device}`;
+            recorder.debug(new ErrorWithExtra(infoString, () => ({ adapterInfo: info })));
+          }
+        });
+      }
+      return promise;
+    };
+  }
+
+  return impl;
+}

tests/wpt/webgpu/tests/webgpu/common/util/preprocessor.js

@@ -0,0 +1,142 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { assert } from './util.js'; // The state of the preprocessor is a stack of States.
+var State;
+
+// Have already seen a passing condition; now skipping the rest
+
+// The transitions in the state space are the following preprocessor directives:
+// - Sibling elif
+// - Sibling else
+// - Sibling endif
+// - Child if
+(function (State) {
+  State[(State['Seeking'] = 0)] = 'Seeking';
+  State[(State['Passing'] = 1)] = 'Passing';
+  State[(State['Skipping'] = 2)] = 'Skipping';
+})(State || (State = {}));
+class Directive {
+  constructor(depth) {
+    this.depth = depth;
+  }
+
+  checkDepth(stack) {
+    assert(
+      stack.length === this.depth,
+      `Number of "$"s must match nesting depth, currently ${stack.length} (e.g. $if $$if $$endif $endif)`
+    );
+  }
+}
+
+class If extends Directive {
+  constructor(depth, predicate) {
+    super(depth);
+    this.predicate = predicate;
+  }
+
+  applyTo(stack) {
+    this.checkDepth(stack);
+    const parentState = stack[stack.length - 1].state;
+    stack.push({
+      allowsFollowingElse: true,
+      state:
+        parentState !== State.Passing
+          ? State.Skipping
+          : this.predicate
+          ? State.Passing
+          : State.Seeking,
+    });
+  }
+}
+
+class ElseIf extends If {
+  applyTo(stack) {
+    assert(stack.length >= 1);
+    const { allowsFollowingElse, state: siblingState } = stack.pop();
+    this.checkDepth(stack);
+    assert(allowsFollowingElse, 'pp.elif after pp.else');
+    if (siblingState !== State.Seeking) {
+      stack.push({ allowsFollowingElse: true, state: State.Skipping });
+    } else {
+      super.applyTo(stack);
+    }
+  }
+}
+
+class Else extends Directive {
+  applyTo(stack) {
+    assert(stack.length >= 1);
+    const { allowsFollowingElse, state: siblingState } = stack.pop();
+    this.checkDepth(stack);
+    assert(allowsFollowingElse, 'pp.else after pp.else');
+    stack.push({
+      allowsFollowingElse: false,
+      state: siblingState === State.Seeking ? State.Passing : State.Skipping,
+    });
+  }
+}
+
+class EndIf extends Directive {
+  applyTo(stack) {
+    stack.pop();
+    this.checkDepth(stack);
+  }
+}
+
+/**
+ * A simple template-based, non-line-based preprocessor implementing if/elif/else/endif.
+ *
+ * @example
+ * ```
+ *     const shader = pp`
+ * ${pp._if(expr)}
+ *   const x: ${type} = ${value};
+ * ${pp._elif(expr)}
+ * ${pp.__if(expr)}
+ * ...
+ * ${pp.__else}
+ * ...
+ * ${pp.__endif}
+ * ${pp._endif}`;
+ * ```
+ *
+ * @param strings - The array of constant string chunks of the template string.
+ * @param ...values - The array of interpolated `${}` values within the template string.
+ */
+export function pp(strings, ...values) {
+  let result = '';
+  const stateStack = [{ allowsFollowingElse: false, state: State.Passing }];
+
+  for (let i = 0; i < values.length; ++i) {
+    const passing = stateStack[stateStack.length - 1].state === State.Passing;
+    if (passing) {
+      result += strings[i];
+    }
+
+    const value = values[i];
+    if (value instanceof Directive) {
+      value.applyTo(stateStack);
+    } else {
+      if (passing) {
+        result += value;
+      }
+    }
+  }
+  assert(stateStack.length === 1, 'Unterminated preprocessor condition at end of file');
+  result += strings[values.length];
+
+  return result;
+}
+pp._if = predicate => new If(1, predicate);
+pp._elif = predicate => new ElseIf(1, predicate);
+pp._else = new Else(1);
+pp._endif = new EndIf(1);
+pp.__if = predicate => new If(2, predicate);
+pp.__elif = predicate => new ElseIf(2, predicate);
+pp.__else = new Else(2);
+pp.__endif = new EndIf(2);
+pp.___if = predicate => new If(3, predicate);
+pp.___elif = predicate => new ElseIf(3, predicate);
+pp.___else = new Else(3);
+pp.___endif = new EndIf(3);
+// Add more if needed.

tests/wpt/webgpu/tests/webgpu/common/util/timeout.js

@@ -0,0 +1,7 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+/**
+ * Equivalent of `setTimeout`, but redirects to WPT's `step_timeout` when it is defined.
+ */
+export const timeout = typeof step_timeout !== 'undefined' ? step_timeout : setTimeout;

tests/wpt/webgpu/tests/webgpu/common/util/types.js

@@ -0,0 +1,13 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/
+
+export function assertTypeTrue() {}
+
+/**
+ * Computes the intersection of a set of types, given the union of those types.
+ *
+ * From: https://stackoverflow.com/a/56375136
+ */
+
+// K exhausted

tests/wpt/webgpu/tests/webgpu/common/util/util.js

@@ -0,0 +1,309 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { Float16Array } from '../../external/petamoriken/float16/float16.js';
+import { SkipTestCase } from '../framework/fixture.js';
+import { globalTestConfig } from '../framework/test_config.js';
+import { Logger } from '../internal/logging/logger.js';
+
+import { keysOf } from './data_tables.js';
+import { timeout } from './timeout.js';
+
+/**
+ * Error with arbitrary `extra` data attached, for debugging.
+ * The extra data is omitted if not running the test in debug mode (`?debug=1`).
+ */
+export class ErrorWithExtra extends Error {
+  /**
+   * `extra` function is only called if in debug mode.
+   * If an `ErrorWithExtra` is passed, its message is used and its extras are passed through.
+   */
+
+  constructor(baseOrMessage, newExtra) {
+    const message = typeof baseOrMessage === 'string' ? baseOrMessage : baseOrMessage.message;
+    super(message);
+
+    const oldExtras = baseOrMessage instanceof ErrorWithExtra ? baseOrMessage.extra : {};
+    this.extra = Logger.globalDebugMode
+      ? { ...oldExtras, ...newExtra() }
+      : { omitted: 'pass ?debug=1' };
+  }
+}
+
+/**
+ * Asserts `condition` is true. Otherwise, throws an `Error` with the provided message.
+ */
+export function assert(condition, msg) {
+  if (!condition) {
+    throw new Error(msg && (typeof msg === 'string' ? msg : msg()));
+  }
+}
+
+/** If the argument is an Error, throw it. Otherwise, pass it back. */
+export function assertOK(value) {
+  if (value instanceof Error) {
+    throw value;
+  }
+  return value;
+}
+
+/**
+ * Resolves if the provided promise rejects; rejects if it does not.
+ */
+export async function assertReject(p, msg) {
+  try {
+    await p;
+    unreachable(msg);
+  } catch (ex) {
+    // Assertion OK
+  }
+}
+
+/**
+ * Assert this code is unreachable. Unconditionally throws an `Error`.
+ */
+export function unreachable(msg) {
+  throw new Error(msg);
+}
+
+/**
+ * Throw a `SkipTestCase` exception, which skips the test case.
+ */
+export function skipTestCase(msg) {
+  throw new SkipTestCase(msg);
+}
+
+/**
+ * The `performance` interface.
+ * It is available in all browsers, but it is not in scope by default in Node.
+ */
+const perf = typeof performance !== 'undefined' ? performance : require('perf_hooks').performance;
+
+/**
+ * Calls the appropriate `performance.now()` depending on whether running in a browser or Node.
+ */
+export function now() {
+  return perf.now();
+}
+
+/**
+ * Returns a promise which resolves after the specified time.
+ */
+export function resolveOnTimeout(ms) {
+  return new Promise(resolve => {
+    timeout(() => {
+      resolve();
+    }, ms);
+  });
+}
+
+export class PromiseTimeoutError extends Error {}
+
+/**
+ * Returns a promise which rejects after the specified time.
+ */
+export function rejectOnTimeout(ms, msg) {
+  return new Promise((_resolve, reject) => {
+    timeout(() => {
+      reject(new PromiseTimeoutError(msg));
+    }, ms);
+  });
+}
+
+/**
+ * Takes a promise `p`, and returns a new one which rejects if `p` takes too long,
+ * and otherwise passes the result through.
+ */
+export function raceWithRejectOnTimeout(p, ms, msg) {
+  if (globalTestConfig.noRaceWithRejectOnTimeout) {
+    return p;
+  }
+  // Setup a promise that will reject after `ms` milliseconds. We cancel this timeout when
+  // `p` is finalized, so the JavaScript VM doesn't hang around waiting for the timer to
+  // complete, once the test runner has finished executing the tests.
+  const timeoutPromise = new Promise((_resolve, reject) => {
+    const handle = timeout(() => {
+      reject(new PromiseTimeoutError(msg));
+    }, ms);
+    p = p.finally(() => clearTimeout(handle));
+  });
+  return Promise.race([p, timeoutPromise]);
+}
+
+/**
+ * Takes a promise `p` and returns a new one which rejects if `p` resolves or rejects,
+ * and otherwise resolves after the specified time.
+ */
+export function assertNotSettledWithinTime(p, ms, msg) {
+  // Rejects regardless of whether p resolves or rejects.
+  const rejectWhenSettled = p.then(() => Promise.reject(new Error(msg)));
+  // Resolves after `ms` milliseconds.
+  const timeoutPromise = new Promise(resolve => {
+    const handle = timeout(() => {
+      resolve(undefined);
+    }, ms);
+    p.finally(() => clearTimeout(handle));
+  });
+  return Promise.race([rejectWhenSettled, timeoutPromise]);
+}
+
+/**
+ * Returns a `Promise.reject()`, but also registers a dummy `.catch()` handler so it doesn't count
+ * as an uncaught promise rejection in the runtime.
+ */
+export function rejectWithoutUncaught(err) {
+  const p = Promise.reject(err);
+  // Suppress uncaught promise rejection.
+  p.catch(() => {});
+  return p;
+}
+
+/**
+ * Makes a copy of a JS `object`, with the keys reordered into sorted order.
+ */
+export function sortObjectByKey(v) {
+  const sortedObject = {};
+  for (const k of Object.keys(v).sort()) {
+    sortedObject[k] = v[k];
+  }
+  return sortedObject;
+}
+
+/**
+ * Determines whether two JS values are equal, recursing into objects and arrays.
+ * NaN is treated specially, such that `objectEquals(NaN, NaN)`.
+ */
+export function objectEquals(x, y) {
+  if (typeof x !== 'object' || typeof y !== 'object') {
+    if (typeof x === 'number' && typeof y === 'number' && Number.isNaN(x) && Number.isNaN(y)) {
+      return true;
+    }
+    return x === y;
+  }
+  if (x === null || y === null) return x === y;
+  if (x.constructor !== y.constructor) return false;
+  if (x instanceof Function) return x === y;
+  if (x instanceof RegExp) return x === y;
+  if (x === y || x.valueOf() === y.valueOf()) return true;
+  if (Array.isArray(x) && Array.isArray(y) && x.length !== y.length) return false;
+  if (x instanceof Date) return false;
+  if (!(x instanceof Object)) return false;
+  if (!(y instanceof Object)) return false;
+
+  const x1 = x;
+  const y1 = y;
+  const p = Object.keys(x);
+  return Object.keys(y).every(i => p.indexOf(i) !== -1) && p.every(i => objectEquals(x1[i], y1[i]));
+}
+
+/**
+ * Generates a range of values `fn(0)..fn(n-1)`.
+ */
+export function range(n, fn) {
+  return [...new Array(n)].map((_, i) => fn(i));
+}
+
+/**
+ * Generates a range of values `fn(0)..fn(n-1)`.
+ */
+export function* iterRange(n, fn) {
+  for (let i = 0; i < n; ++i) {
+    yield fn(i);
+  }
+}
+
+/** Creates a (reusable) iterable object that maps `f` over `xs`, lazily. */
+export function mapLazy(xs, f) {
+  return {
+    *[Symbol.iterator]() {
+      for (const x of xs) {
+        yield f(x);
+      }
+    },
+  };
+}
+
+const ReorderOrders = {
+  forward: true,
+  backward: true,
+  shiftByHalf: true,
+};
+
+export const kReorderOrderKeys = keysOf(ReorderOrders);
+
+/**
+ * Creates a new array from the given array with the first half
+ * swapped with the last half.
+ */
+export function shiftByHalf(arr) {
+  const len = arr.length;
+  const half = (len / 2) | 0;
+  const firstHalf = arr.splice(0, half);
+  return [...arr, ...firstHalf];
+}
+
+/**
+ * Creates a reordered array from the input array based on the Order
+ */
+export function reorder(order, arr) {
+  switch (order) {
+    case 'forward':
+      return arr.slice();
+    case 'backward':
+      return arr.slice().reverse();
+    case 'shiftByHalf': {
+      // should this be pseudo random?
+      return shiftByHalf(arr);
+    }
+  }
+}
+
+const TypedArrayBufferViewInstances = [
+  new Uint8Array(),
+  new Uint8ClampedArray(),
+  new Uint16Array(),
+  new Uint32Array(),
+  new Int8Array(),
+  new Int16Array(),
+  new Int32Array(),
+  new Float16Array(),
+  new Float32Array(),
+  new Float64Array(),
+];
+
+export const kTypedArrayBufferViews = {
+  ...(() => {
+    const result = {};
+    for (const v of TypedArrayBufferViewInstances) {
+      result[v.constructor.name] = v.constructor;
+    }
+    return result;
+  })(),
+};
+export const kTypedArrayBufferViewKeys = keysOf(kTypedArrayBufferViews);
+export const kTypedArrayBufferViewConstructors = Object.values(kTypedArrayBufferViews);
+
+function subarrayAsU8(buf, { start = 0, length }) {
+  if (buf instanceof ArrayBuffer) {
+    return new Uint8Array(buf, start, length);
+  } else if (buf instanceof Uint8Array || buf instanceof Uint8ClampedArray) {
+    // Don't wrap in new views if we don't need to.
+    if (start === 0 && (length === undefined || length === buf.byteLength)) {
+      return buf;
+    }
+  }
+  const byteOffset = buf.byteOffset + start * buf.BYTES_PER_ELEMENT;
+  const byteLength =
+    length !== undefined
+      ? length * buf.BYTES_PER_ELEMENT
+      : buf.byteLength - (byteOffset - buf.byteOffset);
+  return new Uint8Array(buf.buffer, byteOffset, byteLength);
+}
+
+/**
+ * Copy a range of bytes from one ArrayBuffer or TypedArray to another.
+ *
+ * `start`/`length` are in elements (or in bytes, if ArrayBuffer).
+ */
+export function memcpy(src, dst) {
+  subarrayAsU8(dst.dst, dst).set(subarrayAsU8(src.src, src));
+}

tests/wpt/webgpu/tests/webgpu/common/util/wpt_reftest_wait.js

@@ -0,0 +1,24 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ import { timeout } from './timeout.js'; // Copied from https://github.com/web-platform-tests/wpt/blob/master/common/reftest-wait.js
+
+/**
+ * Remove the `reftest-wait` class on the document element.
+ * The reftest runner will wait with taking a screenshot while
+ * this class is present.
+ *
+ * See https://web-platform-tests.org/writing-tests/reftests.html#controlling-when-comparison-occurs
+ */
+export function takeScreenshot() {
+  document.documentElement.classList.remove('reftest-wait');
+}
+
+/**
+ * Call `takeScreenshot()` after a delay of at least `ms` milliseconds.
+ * @param {number} ms - milliseconds
+ */
+export function takeScreenshotDelayed(ms) {
+  timeout(() => {
+    takeScreenshot();
+  }, ms);
+}

tests/wpt/webgpu/tests/webgpu/cts.html

@@ -1,63 +0,0 @@
-<!-- AUTO-GENERATED - DO NOT EDIT. See WebGPU CTS: tools/gen_wpt_cts_html. -->
-<!--
-    This test suite is built from the TypeScript sources at:
-    https://github.com/gpuweb/cts
-
-    If you are debugging WebGPU conformance tests, it's highly recommended that
-    you use the standalone interactive runner in that repository, which
-    provides tools for easier debugging and editing (source maps, debug
-    logging, warn/skip functionality, etc.)
-
-    NOTE:
-    The WPT version of this file is generated with *one variant per test spec
-    file*. If your harness needs more fine-grained suppressions, you'll need to
-    generate your own variants list from your suppression list.
-    See `tools/gen_wpt_cts_html` to do this.
-
-    When run under browser CI, the original cts.html should be skipped, and
-    this alternate version should be run instead, under a non-exported WPT test
-    directory (e.g. Chromium's wpt_internal).
--->
-
-<!doctype html>
-<title>WebGPU CTS</title>
-<meta charset=utf-8>
-<link rel=help href='https://gpuweb.github.io/gpuweb/'>
-
-<script src=/resources/testharness.js></script>
-<script src=/resources/testharnessreport.js></script>
-<script type=module src=../webgpu/common/runtime/wpt.js></script>
-<!--<meta name=variant content='?q=webgpu:api,operation,buffers,map:*'>-->
-<meta name=variant content='?q=webgpu:api,operation,buffers,map_detach:*'>
-<meta name=variant content='?q=webgpu:api,operation,buffers,map_oom:*'>
-<meta name=variant content='?q=webgpu:api,operation,command_buffer,basic:*'>
-<meta name=variant content='?q=webgpu:api,operation,command_buffer,copies:*'>
-<meta name=variant content='?q=webgpu:api,operation,command_buffer,render,basic:*'>
-<!--<meta name=variant content='?q=webgpu:api,operation,copyBetweenLinearDataAndTexture:*'>-->
-<meta name=variant content='?q=webgpu:api,operation,fences:*'>
-<!--<meta name=variant content='?q=webgpu:api,operation,render_pass,storeOp:*'>-->
-<!--<meta name=variant content='?q=webgpu:api,operation,resource_init,copied_texture_clear:*'>-->
-<meta name=variant content='?q=webgpu:api,validation,copyBufferToBuffer:*'>
-<meta name=variant content='?q=webgpu:api,validation,copy_between_linear_data_and_texture,copyBetweenLinearDataAndTexture_dataRelated:*'>
-<meta name=variant content='?q=webgpu:api,validation,copy_between_linear_data_and_texture,copyBetweenLinearDataAndTexture_textureRelated:*'>
-<meta name=variant content='?q=webgpu:api,validation,createBindGroup:*'>
-<!--<meta name=variant content='?q=webgpu:api,validation,createBindGroupLayout:*'>-->
-<meta name=variant content='?q=webgpu:api,validation,createPipelineLayout:*'>
-<!--<meta name=variant content='?q=webgpu:api,validation,createTexture:*'>-->
-<!--<meta name=variant content='?q=webgpu:api,validation,createView:*'>-->
-<meta name=variant content='?q=webgpu:api,validation,error_scope:*'>
-<meta name=variant content='?q=webgpu:api,validation,fences:*'>
-<meta name=variant content='?q=webgpu:api,validation,queue_submit:*'>
-<meta name=variant content='?q=webgpu:api,validation,render_pass,resolve:*'>
-<meta name=variant content='?q=webgpu:api,validation,render_pass,storeOp:*'>
-<meta name=variant content='?q=webgpu:api,validation,render_pass_descriptor:*'>
-<meta name=variant content='?q=webgpu:api,validation,resource_usages,textureUsageInRender:*'>
-<meta name=variant content='?q=webgpu:api,validation,setBindGroup:*'>
-<meta name=variant content='?q=webgpu:api,validation,setBlendColor:*'>
-<meta name=variant content='?q=webgpu:api,validation,setScissorRect:*'>
-<meta name=variant content='?q=webgpu:api,validation,setStencilReference:*'>
-<meta name=variant content='?q=webgpu:api,validation,setViewport:*'>
-<meta name=variant content='?q=webgpu:examples:*'>
-<meta name=variant content='?q=webgpu:idl,constants,flags:*'>
-<meta name=variant content='?q=webgpu:web-platform,canvas,context_creation:*'>
-<meta name=variant content='?q=webgpu:web-platform,copyImageBitmapToTexture:*'>

tests/wpt/webgpu/tests/webgpu/external/petamoriken/float16/float16.d.js

@@ -0,0 +1,3 @@
+/**
+ * AUTO-GENERATED - DO NOT EDIT. Source: https://github.com/gpuweb/cts
+ **/ export {};

tests/wpt/webgpu/tests/webgpu/resources/README.md

@@ -0,0 +1,15 @@
+Always use `getResourcePath()` to get the appropriate path to these resources depending
+on the context (WPT, standalone, worker, etc.)
+
+
+The test video files were generated with the ffmpeg cmds below:
+ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx -pix_fmt yuv420p -frames 500 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp8-bt601.webm
+ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libtheora -pix_fmt yuv420p -frames 500 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-theora-bt601.ogv
+ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 500 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-h264-bt601.mp4
+ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 500 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp9-bt601.webm
+ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 500 -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv -vf scale=out_color_matrix=bt709:out_range=tv four-colors-vp9-bt709.webm
+
+These rotation test files are copies of four-colors-h264-bt601.mp4 with metadata changes.
+ffmpeg.exe -i .\four-colors-h264-bt601.mp4 -c copy -metadata:s:v rotate=90 four-colors-h264-bt601-rotate-90.mp4
+ffmpeg.exe -i .\four-colors-h264-bt601.mp4 -c copy -metadata:s:v rotate=180 four-colors-h264-bt601-rotate-180.mp4
+ffmpeg.exe -i .\four-colors-h264-bt601.mp4 -c copy -metadata:s:v rotate=270 four-colors-h264-bt601-rotate-270.mp4