Add ability to install custom JS module loaders.

- Adds `JSModuleLoaderFactory` interface for ESM loader hook.
- Adds `CommonJSResolverHook` interface for CJS resolver hook.
- Adds `js.module-loader-factory=handler` setting to enable.
- Adjusts `JSEngine` to retain the installed factory and/or resolver.
- Adjusts `JSRealm` to use `JSEngine` to create the module loader.
- Adjusts `NpmCompatibleESModuleLoader` to be extensible.
- Adjusts `CommonJSResolution` to use the resolver hook if present.

Relates to oracle/graal#9177

Signed-off-by: Sam Gammon <sam@elide.dev>

Author: sgammon

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/builtins/commonjs/CommonJSRequireBuiltin.java

@@ -46,6 +46,7 @@
 import static com.oracle.truffle.js.builtins.commonjs.CommonJSResolution.MJS_EXT;
 import static com.oracle.truffle.js.builtins.commonjs.CommonJSResolution.NODE_EXT;
 import static com.oracle.truffle.js.builtins.commonjs.CommonJSResolution.getCoreModuleReplacement;
+import static com.oracle.truffle.js.runtime.JSContextOptions.ModuleLoaderFactoryMode.HANDLER;
 
 import java.util.Map;
 import java.util.Objects;
@@ -64,6 +65,7 @@
 import com.oracle.truffle.js.runtime.Errors;
 import com.oracle.truffle.js.runtime.JSArguments;
 import com.oracle.truffle.js.runtime.JSContext;
+import com.oracle.truffle.js.runtime.JSEngine;
 import com.oracle.truffle.js.runtime.JSErrorType;
 import com.oracle.truffle.js.runtime.JSException;
 import com.oracle.truffle.js.runtime.JSRealm;
@@ -169,6 +171,21 @@ protected static Object fallback(@SuppressWarnings("unused") Object function, Ob
     @TruffleBoundary
     private Object requireImpl(String moduleIdentifier, TruffleFile entryPath, JSRealm realm) {
         log("required module '", moduleIdentifier, "' from path ", entryPath);
+        // 1.1 (Non-spec): If a module resolver hook has been installed, give it a chance to resolve the module, but
+        // only if `handler` mode is enabled for JS module resolution.
+        if (realm.getContextOptions().getModuleLoaderFactoryMode().equals(HANDLER)) {
+            var resolver = JSEngine.getCjsResolverHook();
+            if (resolver != null) {
+                log("custom import hook is active; there is a resolver. loading module '", moduleIdentifier, "'");
+                var maybeResolved = resolver.resolveModule(realm, moduleIdentifier, entryPath);
+                if (maybeResolved != null) {
+                    log("custom handler returned module impl for '", moduleIdentifier, "'");
+                    return maybeResolved;
+                } else if (LOG_REQUIRE_PATH_RESOLUTION) {
+                    log("custom handler returned null; falling back for module '", moduleIdentifier, "'");
+                }
+            }
+        }
         String moduleReplacementName = getCoreModuleReplacement(realm, moduleIdentifier);
         if (moduleReplacementName != null) {
             log("using module replacement for module '", moduleIdentifier, "' with ", moduleReplacementName);

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/builtins/commonjs/CommonJSResolution.java

@@ -59,13 +59,15 @@
 import com.oracle.truffle.js.lang.JavaScriptLanguage;
 import com.oracle.truffle.js.runtime.Errors;
 import com.oracle.truffle.js.runtime.JSArguments;
+import com.oracle.truffle.js.runtime.JSEngine;
 import com.oracle.truffle.js.runtime.JSRealm;
 import com.oracle.truffle.js.runtime.Strings;
 import com.oracle.truffle.js.runtime.builtins.JSFunction;
 import com.oracle.truffle.js.runtime.builtins.JSFunctionObject;
 import com.oracle.truffle.js.runtime.objects.JSDynamicObject;
 import com.oracle.truffle.js.runtime.objects.JSObject;
 import com.oracle.truffle.js.runtime.objects.Undefined;
+import static com.oracle.truffle.js.runtime.JSContextOptions.ModuleLoaderFactoryMode.HANDLER;
 
 public final class CommonJSResolution {
 

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/builtins/commonjs/NpmCompatibleESModuleLoader.java

@@ -87,7 +87,7 @@
 import com.oracle.truffle.js.runtime.objects.ScriptOrModule;
 import com.oracle.truffle.js.runtime.objects.Undefined;
 
-public final class NpmCompatibleESModuleLoader extends DefaultESModuleLoader {
+public class NpmCompatibleESModuleLoader extends DefaultESModuleLoader {
 
     private static final URI TryCommonJS = URI.create("custom:///try-common-js-token");
     private static final URI TryCustomESM = URI.create("custom:///try-custom-esm-token");
@@ -106,7 +106,7 @@ public static NpmCompatibleESModuleLoader create(JSRealm realm) {
         return new NpmCompatibleESModuleLoader(realm);
     }
 
-    private NpmCompatibleESModuleLoader(JSRealm realm) {
+    protected NpmCompatibleESModuleLoader(JSRealm realm) {
         super(realm);
     }
 

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/runtime/CommonJSResolverHook.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.truffle.js.runtime;
+
+import com.oracle.truffle.api.TruffleFile;
+
+/**
+ * Allows GraalJs users to hook into the JavaScript CJS loading process.
+ */
+public interface CommonJSResolverHook {
+    /**
+     * Resolve a CommonJS module identifier to a file.
+     *
+     * <p>Return types which are valid include:
+     * <ul>
+     *     <li>{@link TruffleFile}: Will be interpreted as normal (i.e. as a CJS file)</li>
+     *     <li>Guest-compatible value: Returned as the module itself, without evaluation</li>
+     * </ul>
+     * </p>
+     *
+     * @param realm            the realm in which the module is being resolved
+     * @param moduleIdentifier the CommonJS module identifier
+     * @param entryPath        the path of the module that is importing the module
+     * @return Optional wrapping the type which should be returned for this module implementation; supported types
+     *   are listed above.
+     */
+    Object resolveModule(JSRealm realm, String moduleIdentifier, TruffleFile entryPath);
+}

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/runtime/JSContextOptions.java

@@ -638,6 +638,25 @@ public String toString() {
     public static final OptionKey<UnhandledRejectionsTrackingMode> UNHANDLED_REJECTIONS = new OptionKey<>(UnhandledRejectionsTrackingMode.NONE);
     @CompilationFinal private UnhandledRejectionsTrackingMode unhandledRejectionsMode;
 
+    public enum ModuleLoaderFactoryMode {
+        DEFAULT,
+        HANDLER;
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+    }
+
+    public static final String MODULE_LOADER_FACTORY_NAME = JS_OPTION_PREFIX + "module-loader-factory";
+
+    @Option(name = MODULE_LOADER_FACTORY_NAME, category = OptionCategory.USER, stability = OptionStability.EXPERIMENTAL, sandbox = SandboxPolicy.TRUSTED, help = """
+                    Configure a factory for overriding the JavaScript module loader. Accepted values: \
+                    'default', default behavior applies for CommonJS and ESM. \
+                    'handler', the handler function set with JSEngine.setModuleLoaderFactory will be called to satisfy CJS or ESM imports.""") //
+    public static final OptionKey<ModuleLoaderFactoryMode> MODULE_LOADER_FACTORY_MODE = new OptionKey<>(ModuleLoaderFactoryMode.DEFAULT);
+    @CompilationFinal private ModuleLoaderFactoryMode moduleLoaderFactoryMode;
+
     public static final String OPERATOR_OVERLOADING_NAME = JS_OPTION_PREFIX + "operator-overloading";
     @Option(name = OPERATOR_OVERLOADING_NAME, category = OptionCategory.USER, help = "Enable operator overloading") //
     public static final OptionKey<Boolean> OPERATOR_OVERLOADING = new OptionKey<>(false);
@@ -810,7 +829,8 @@ private void cacheOptions(SandboxPolicy sandboxPolicy) {
         this.useUTCForLegacyDates = USE_UTC_FOR_LEGACY_DATES.hasBeenSet(optionValues) ? readBooleanOption(USE_UTC_FOR_LEGACY_DATES) : !v8CompatibilityMode;
         this.webAssembly = readBooleanOption(WEBASSEMBLY);
         this.unhandledRejectionsMode = readUnhandledRejectionsMode();
-        this.newSetMethods = readBooleanOption(NEW_SET_METHODS, JSConfig.ECMAScript2025);
+        this.moduleLoaderFactoryMode = readModuleLoaderFactoryMode();
+	this.newSetMethods = readBooleanOption(NEW_SET_METHODS, JSConfig.ECMAScript2025);
         this.atomicsWaitAsync = readBooleanOption(ATOMICS_WAIT_ASYNC, JSConfig.ECMAScript2024);
         this.asyncIteratorHelpers = getEcmaScriptVersion() >= JSConfig.ECMAScript2018 && readBooleanOption(ASYNC_ITERATOR_HELPERS);
         this.iteratorHelpers = getEcmaScriptVersion() >= JSConfig.ECMAScript2018 && (this.asyncIteratorHelpers || readBooleanOption(ITERATOR_HELPERS, JSConfig.ECMAScript2025));
@@ -845,6 +865,10 @@ private UnhandledRejectionsTrackingMode readUnhandledRejectionsMode() {
         return UNHANDLED_REJECTIONS.getValue(optionValues);
     }
 
+    private ModuleLoaderFactoryMode readModuleLoaderFactoryMode() {
+        return MODULE_LOADER_FACTORY_MODE.getValue(optionValues);
+    }
+
     private boolean readBooleanOption(OptionKey<Boolean> key) {
         return key.getValue(optionValues);
     }
@@ -1336,4 +1360,7 @@ public boolean isWorker() {
         return worker;
     }
 
+    public ModuleLoaderFactoryMode getModuleLoaderFactoryMode() {
+        return moduleLoaderFactoryMode;
+    }
 }

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/runtime/JSEngine.java

@@ -40,14 +40,22 @@
  */
 package com.oracle.truffle.js.runtime;
 
+import java.util.Optional;
 import java.util.ServiceLoader;
 
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.TruffleLanguage;
 import com.oracle.truffle.js.lang.JavaScriptLanguage;
 
 public final class JSEngine {
     private static final JSEngine INSTANCE = new JSEngine();
 
+    @CompilerDirectives.CompilationFinal
+    private static volatile JSModuleLoaderFactory MODULE_LOADER_FACTORY = null;
+
+    @CompilerDirectives.CompilationFinal
+    private static volatile CommonJSResolverHook CJS_RESOLVER_HOOK = null;
+
     private final Evaluator parser;
 
     private JSEngine() {
@@ -71,6 +79,22 @@ private JSContext createContext(JavaScriptLanguage language, TruffleLanguage.Env
         return JSContext.createContext(parser, language, env);
     }
 
+    public static void setModuleLoaderFactory(JSModuleLoaderFactory factory) {
+        MODULE_LOADER_FACTORY = factory;
+    }
+
+    public static void setCjsResolverHook(CommonJSResolverHook hook) {
+        CJS_RESOLVER_HOOK = hook;
+    }
+
+    public static JSModuleLoaderFactory getModuleLoaderFactory() {
+        return MODULE_LOADER_FACTORY;
+    }
+
+    public static CommonJSResolverHook getCjsResolverHook() {
+        return CJS_RESOLVER_HOOK;
+    }
+
     public static JSContext createJSContext(JavaScriptLanguage language, TruffleLanguage.Env env) {
         return JSEngine.getInstance().createContext(language, env);
     }

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/runtime/JSModuleLoaderFactory.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.truffle.js.runtime;
+
+import com.oracle.truffle.js.runtime.objects.JSModuleLoader;
+
+/**
+ * Allows GraalJs users to hook into the JavaScript ESM loading process.
+ */
+public interface JSModuleLoaderFactory {
+    /**
+     * Create an instance of the JavaScript module loader.
+     *
+     * @param realm JavaScript realm which intends to own this loader.
+     * @return Loader instance.
+     */
+    JSModuleLoader createLoader(JSRealm realm);
+}

graal-js/src/com.oracle.truffle.js/src/com/oracle/truffle/js/runtime/JSRealm.java

@@ -55,6 +55,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.SplittableRandom;
 import java.util.WeakHashMap;
 
@@ -2859,12 +2860,33 @@ public JSModuleLoader getModuleLoader() {
     @TruffleBoundary
     private synchronized void createModuleLoader() {
         if (moduleLoader == null) {
-            if (getContextOptions().isCommonJSRequire()) {
-                moduleLoader = NpmCompatibleESModuleLoader.create(this);
-            } else {
-                moduleLoader = DefaultESModuleLoader.create(this);
+            JSModuleLoader loader = null;
+            switch (getContextOptions().getModuleLoaderFactoryMode()) {
+                case HANDLER -> loader = loadCustomModuleLoaderOrFallBack();
+                case DEFAULT -> loader = createStandardModuleLoader(this);
             }
+            assert loader != null;
+            moduleLoader = loader;
+        }
+    }
+
+    private JSModuleLoader loadCustomModuleLoaderOrFallBack() {
+        JSModuleLoaderFactory fac = JSEngine.getModuleLoaderFactory();
+        if (fac == null) {
+            return createStandardModuleLoader(this);
+        }
+        var loader = fac.createLoader(this);
+        if (loader == null) {
+            return createStandardModuleLoader(this);
+        }
+        return loader;
+    }
+
+    private static JSModuleLoader createStandardModuleLoader(JSRealm realm) {
+        if (realm.getContextOptions().isCommonJSRequire()) {
+            return NpmCompatibleESModuleLoader.create(realm);
         }
+        return DefaultESModuleLoader.create(realm);
     }
 
     public final JSAgent getAgent() {