Recompile cljs.loader in REPL

The cljs.loader namespace has to be compiled last, because all inputs
need to be known to populate the `MODULE_INFOS` and `MODULE_URIS`
vars.

Loading a file or evaluating a form in the REPL could trigger another
compilation of the cljs.loader namespace, without the information for
the `MODULE_INFOS` and `MODULE_URIS` available. This compilation
erased the previously populated `MODULE_INFOS` and `MODULE_URIS` vars
and corrupted the module loader.

This patch modifies the loader compilation pass to swap and merge the
`MODULE_INFOS` and `MODULE_URIS` into the compiler state, to make this
information available to compilation stages happening later.

The patch also changes the REPL to recompile and load the cljs.loader
namespace when a file gets loaded, or a namespace form gets
evaluated. A new module graph will be calculated and merged into the
previously calculated one. Recompilation of the cljs.loader namespace
will only happen if the file or form require the cljs.loader
namespace.

Since the :aot-cache is populated at a point where the information for
the `MODULE_INFOS` and `MODULE_URIS` is not yet available, this patch
also makes sure the cljs.loader namespace is never cached.

Author: r0man

src/main/clojure/cljs/closure.clj

@@ -634,6 +634,10 @@
                 (util/mkdirs target)
                 (spit target (slurp f))
                 (.setLastModified target (util/last-modified jar-file))))))))
+    ;; Files that don't require compilation (cljs.loader for example)
+    ;; need to be copied from JAR to disk.
+    (when-not (.exists out-file)
+      (jar-file-to-disk jar-file (util/output-directory opts) opts))
     ;; have to call compile-file as it includes more IJavaScript
     ;; information than ana/parse-ns for now
     (compile-file
@@ -1140,6 +1144,11 @@
       :depends-on #{:core}}})
   )
 
+(defn- const-expr-form
+  "Returns the :const-expr form for `sym` from `compiler-state`."
+  [compiler-state sym]
+  (get-in compiler-state [::ana/namespaces (symbol (namespace sym)) :defs (symbol (name sym)) :const-expr :form]))
+
 (defn compile-loader
   "Special compilation pass for cljs.loader namespace. cljs.loader must be
   compiled last after all inputs. This is because all inputs must be known and
@@ -1153,14 +1162,15 @@
                       first)]
     (let [module-uris  (module-graph/modules->module-uris modules inputs opts)
           module-infos (module-graph/modules->module-infos modules)]
-      (env/with-compiler-env
-        (ana/add-consts @env/*compiler*
-          {'cljs.core/MODULE_URIS  module-uris
-           'cljs.core/MODULE_INFOS module-infos})
-        (-compile (:source-file loader)
-          (merge opts
-            {:cache-key   (util/content-sha (pr-str module-uris))
-             :output-file (comp/rename-to-js (util/ns->relpath (:ns loader)))})))))
+      (swap! env/*compiler* ana/add-consts
+             {'cljs.core/MODULE_INFOS
+              (merge (const-expr-form @env/*compiler* 'cljs.core/MODULE_INFOS) module-infos)
+              'cljs.core/MODULE_URIS
+              (merge (const-expr-form @env/*compiler* 'cljs.core/MODULE_URIS) module-uris)})
+      (-compile (:source-file loader)
+        (merge opts
+          {:cache-key   (util/content-sha (pr-str module-uris))
+           :output-file (comp/rename-to-js (util/ns->relpath (:ns loader)))}))))
   inputs)
 
 (defn build-modules

src/main/clojure/cljs/compiler.cljc

@@ -1480,22 +1480,24 @@
           (:options @env/*compiler*))))
      ([^File src ^File dest opts]
       (let [{:keys [ns requires]} (ana/parse-ns src)]
-        (ensure
-         (or (not (.exists dest))
-             (util/changed? src dest)
-             (let [version' (util/compiled-by-version dest)
-                   version (util/clojurescript-version)]
-               (and version (not= version version')))
-             (and opts
-                  (not (and (io/resource "cljs/core.aot.js") (= 'cljs.core ns)))
-                  (not= (ana/build-affecting-options opts)
-                        (ana/build-affecting-options (util/build-options dest))))
-             (and opts (:source-map opts)
-                  (if (= (:optimizations opts) :none)
-                    (not (.exists (io/file (str (.getPath dest) ".map"))))
-                    (not (get-in @env/*compiler* [::compiled-cljs (.getAbsolutePath dest)]))))
-             (when-let [recompiled' (and *recompiled* @*recompiled*)]
-               (some requires recompiled'))))))))
+        (if (and (= 'cljs.loader ns) (not (contains? opts :cache-key)))
+          false
+          (ensure
+           (or (not (.exists dest))
+               (util/changed? src dest)
+               (let [version' (util/compiled-by-version dest)
+                     version (util/clojurescript-version)]
+                 (and version (not= version version')))
+               (and opts
+                    (not (and (io/resource "cljs/core.aot.js") (= 'cljs.core ns)))
+                    (not= (ana/build-affecting-options opts)
+                          (ana/build-affecting-options (util/build-options dest))))
+               (and opts (:source-map opts)
+                    (if (= (:optimizations opts) :none)
+                      (not (.exists (io/file (str (.getPath dest) ".map"))))
+                      (not (get-in @env/*compiler* [::compiled-cljs (.getAbsolutePath dest)]))))
+               (when-let [recompiled' (and *recompiled* @*recompiled*)]
+                 (some requires recompiled')))))))))
 
 #?(:clj
    (defn compile-file

src/main/clojure/cljs/repl.cljc

@@ -206,10 +206,34 @@
 (defn compilable? [input]
   (contains? input :source-file))
 
+(defn- load-sources
+  "Load the compiled `sources` into the REPL."
+  [repl-env sources opts]
+  (if (:output-dir opts)
+    ;; REPLs that read from :output-dir just need to add deps,
+    ;; environment will handle actual loading - David
+    (let [sb (StringBuffer.)]
+      (doseq [source sources]
+        (with-open [rdr (io/reader (:url source))]
+          (.append sb (cljsc/add-dep-string opts source))))
+      (when (:repl-verbose opts)
+        (println (.toString sb)))
+      (-evaluate repl-env "<cljs repl>" 1 (.toString sb)))
+    ;; REPLs that stream must manually load each dep - David
+    (doseq [{:keys [url provides]} sources]
+      (-load repl-env provides url))))
+
+(defn- load-cljs-loader
+  "Compile and load the cljs.loader namespace if it's present in `sources`."
+  [repl-env sources opts]
+  (when-let [source (first (filter #(= (:ns %) 'cljs.loader) sources))]
+    (cljsc/compile-loader sources opts)
+    (load-sources repl-env [source] opts)))
+
 (defn load-namespace
   "Load a namespace and all of its dependencies into the evaluation environment.
-  The environment is responsible for ensuring that each namespace is loaded once and
-  only once."
+  The environment is responsible for ensuring that each namespace is
+  loaded once and only once. Returns the compiled sources."
   ([repl-env ns] (load-namespace repl-env ns nil))
   ([repl-env ns opts]
    (let [ns      (if (and (seq? ns) (= (first ns) 'quote)) (second ns) ns)
@@ -221,29 +245,18 @@
                                 (merge (env->opts repl-env) opts))
                            (remove (comp #{["goog"]} :provides)))
                          (map #(cljsc/source-on-disk opts %)
-                           (cljsc/add-js-sources [input] opts))))))]
+                              (cljsc/add-js-sources [input] opts))))))]
      (when (:repl-verbose opts)
        (println (str "load-namespace " ns " , compiled:") (map :provides sources)))
-     (if (:output-dir opts)
-       ;; REPLs that read from :output-dir just need to add deps,
-       ;; environment will handle actual loading - David
-       (let [sb (StringBuffer.)]
-         (doseq [source sources]
-           (with-open [rdr (io/reader (:url source))]
-             (.append sb
-               (cljsc/add-dep-string opts source))))
-         (when (:repl-verbose opts)
-           (println (.toString sb)))
-         (-evaluate repl-env "<cljs repl>" 1 (.toString sb)))
-       ;; REPLs that stream must manually load each dep - David
-       (doseq [{:keys [url provides]} sources]
-         (-load repl-env provides url))))))
+     (load-sources repl-env sources opts)
+     sources)))
 
 (defn- load-dependencies
-  ([repl-env requires] (load-dependencies repl-env requires nil))
+  "Compile and load the given `requires` and return the compiled sources."
+  ([repl-env requires]
+   (load-dependencies repl-env requires nil))
   ([repl-env requires opts]
-   (doseq [ns (distinct requires)]
-     (load-namespace repl-env ns opts))))
+   (doall (mapcat #(load-namespace repl-env % opts) (distinct requires)))))
 
 (defn ^File js-src->cljs-src
   "Map a JavaScript output file back to the original ClojureScript source
@@ -544,11 +557,12 @@
                      (ana/no-warn (ana/analyze env form nil opts))
                      (catch #?(:clj Exception :cljs js/Error) e
                          (reset! env/*compiler* backup-comp)
-                       (throw e)))]
-           (load-dependencies repl-env
-             (into (vals (:requires ast))
-               (distinct (vals (:uses ast))))
-             opts)))
+                         (throw e)))
+               sources (load-dependencies repl-env
+                         (into (vals (:requires ast))
+                               (distinct (vals (:uses ast))))
+                         opts)]
+           (load-cljs-loader repl-env sources opts)))
        (when *cljs-verbose*
          (err-out (println wrap-js)))
        (let [ret (-evaluate repl-env filename (:line (meta form)) wrap-js)]
@@ -596,7 +610,8 @@
               (util/ns->relpath ns (util/ext (:source-url compiled))))
             (slurp src)))
         ;; need to load dependencies first
-        (load-dependencies repl-env (:requires compiled) opts)
+        (let [sources (load-dependencies repl-env (:requires compiled) opts)]
+          (load-cljs-loader repl-env (conj sources compiled) opts))
         (-evaluate repl-env f 1 (cljsc/add-dep-string opts compiled))
         (-evaluate repl-env f 1
           (cljsc/src-file->goog-require src