diff --git a/crates/ts_goport/src/execute/execute_tsc.rs b/crates/ts_goport/src/execute/execute_tsc.rs index 2ca2fbeab..6c8c688eb 100644 --- a/crates/ts_goport/src/execute/execute_tsc.rs +++ b/crates/ts_goport/src/execute/execute_tsc.rs @@ -590,10 +590,23 @@ pub(crate) fn get_trace_from_sys( // PORT: the new program is installed for the process (see the module // comment). Go `NewProgram` cannot fail; the Rust install fails only when // the current directory cannot be read, and that ends the run. -pub(crate) fn install_program(host: Rc, config: Rc) { +// PORT: in the multi-run flow (`program::multirun`), the program is a +// program version of the process instead (`install_and_enter`), so a later +// run can load another version; the returned scope makes it the current +// program of the thread and the caller holds it across the compile (see +// the callers). +#[must_use = "in the multi-run flow the returned scope keeps the program current; hold it for the compile"] +pub(crate) fn install_program( + host: Rc, + config: Rc, +) -> Option { + if crate::program::multirun::is_enabled() { + return Some(crate::program::multirun::install_and_enter(host, config)); + } if let Err(message) = crate::program::install_new_program(program_options(host, config)) { panic!("cannot load program: {message}"); } + None } /// Go `compiler.NewProgram(compiler.ProgramOptions{Config, Host})` in a @@ -673,6 +686,14 @@ fn perform_incremental_compilation( Some(get_trace_from_sys(sys, config.locale(), testing.clone())), content_mapper_project, ); + // PORT: in the multi-run flow, the host of the run sits over the parse + // cache of the process (`program::multirun`), as `perform_compilation` + // does: the build info read, the version install and the incremental + // program all run over the wrapped host, so an incremental run reuses + // the parses of the files that an earlier run saw with the same + // content. `wrap_host` returns the host unchanged while the flow is + // off (native one-program runs). + let host = crate::program::multirun::wrap_host(host); let config = Rc::new(config); let replacement = hooks.program_like(); let build_info_read_start = sys.now(); @@ -685,7 +706,12 @@ fn perform_incremental_compilation( start_tracing_if_needed(sys, &config, testing.is_some()); let parse_start = sys.now(); - install_program(host.clone(), config.clone()); + // PORT: the multi-run scope (`multirun::install_and_enter`): it keeps + // the version current for the rest of the compile and drops with this + // frame, so no program is current when `ts_run` calls `end_run` — + // whose release needs that (`release_program` panics on a current + // program). + let _install_scope = install_program(host.clone(), config.clone()); compile_times.borrow_mut().parse_time = since(sys, parse_start); let changes_compute_start = sys.now(); // Go: sys.Now @@ -778,12 +804,21 @@ fn perform_compilation( Some(get_trace_from_sys(sys, config.locale(), testing.clone())), content_mapper_project, ); + // PORT: in the multi-run flow, the host of the run sits over the parse + // cache of the process (`program::multirun`), so a run reuses the + // parses of the files that an earlier run saw with the same mtime. + let host = crate::program::multirun::wrap_host(host); let config = Rc::new(config); start_tracing_if_needed(sys, &config, testing.is_some()); let parse_start = sys.now(); - install_program(host, config.clone()); + // PORT: the multi-run scope (`multirun::install_and_enter`): it keeps + // the version current for the rest of the compile and drops with this + // frame, so no program is current when `ts_run` calls `end_run` — + // whose release needs that (`release_program` panics on a current + // program). + let _install_scope = install_program(host, config.clone()); compile_times.borrow_mut().parse_time = since(sys, parse_start); if let Some(content_mapper_host) = &content_mapper_host { compile_times.borrow_mut().content_mapper_times = content_mapper_host.timings(); diff --git a/crates/ts_goport/src/program.rs b/crates/ts_goport/src/program.rs index fdf778c56..1fb2c002e 100644 --- a/crates/ts_goport/src/program.rs +++ b/crates/ts_goport/src/program.rs @@ -29,6 +29,8 @@ use std::sync::{Arc, Mutex, OnceLock, PoisonError}; mod go_frontend; pub(crate) use go_frontend::cached_lazy_js_doc; pub mod ls_program; +/// PORT: not in Go. Multi-run wasm support (see the module comment). +pub mod multirun; // --------------------------------------------------------------------------- // Types @@ -5364,6 +5366,13 @@ pub fn get_bind_diagnostics(source_file: Node) -> Vec { // PORT: the compile path has no context; Go tsc passes context.Background() // (execute/tsc/emit.go:76). The same holds for the two functions below. pub fn get_semantic_diagnostics(source_file: Node) -> Vec { + // The whole-program diagnostics of a multi-run process (`program::multirun`): + // the reuse set of the run skips the files whose lists the cache returns + // (empty while the cache is cold or after a rule cleared it: the call + // checks everything and fills the cache). + if source_file.is_nil() && multirun::is_enabled() { + return multirun::semantic_diagnostics_with_reuse(); + } collect_checker_diagnostics_with(source_file, |c, f| { get_semantic_diagnostics_with_checker(&context::background(), c, f) }) diff --git a/crates/ts_goport/src/program/multirun.rs b/crates/ts_goport/src/program/multirun.rs new file mode 100644 index 000000000..b3a5092f2 --- /dev/null +++ b/crates/ts_goport/src/program/multirun.rs @@ -0,0 +1,816 @@ +//! PORT: not in Go. Multi-run wasm support (`ts_wasm`): one process serves +//! many tsc requests (`ts_run`) through the program-version machinery, the +//! way watch mode and the language server do. The first request loads a +//! program version (`Entry::Version`, not `core::set_prog`), and every +//! later request loads a new version that reuses the parses of unchanged +//! files: the parse cache below keys a parse by path and the hash of the +//! content the host file system serves, as the language server parse cache +//! keys an entry by content (`project::parsecache` `ParseCacheKey`). A +//! bundled lib file skips the hash: its text is embedded in the module and +//! cannot change within the process. The host needs no mtime contract — a +//! host file system that reports one mtime for every file (the npm +//! package's `memoryFileSystem`) is served correctly. The version that the +//! last run loaded stays loaded until the run after the next, so each +//! version is built from it (`new_program_version` `previous`), which +//! shares the unchanged frontend data (`GoSharedState::new`). +//! +//! The host enables this once (`enable`); `execute::execute_tsc` +//! `install_program` and `perform_compilation` then take the version paths +//! here, and the compile runs under the `ProgramScope` that +//! `install_and_enter` returns (the sanctioned multi-program pattern, +//! `core::enter_program`): the scope makes the version current and the +//! thread back to no current program when it drops, which is before +//! `end_run`. `end_run` releases the version of two runs ago (its checker pool +//! stops) and drops the parse cache entries of paths that left the program +//! (as the Go watcher drops the cache entries of files that left), so a +//! per-run path does not accumulate parses over the runs; the parse is +//! freed with the versions that held it. A parse whose path this process +//! never published is a freeable file version too (`note_published_path` +//! before the parse, `FileVersion` after), so the first parse of each new +//! path is freed with its holders as well and only the bookkeeping of the +//! live paths stays. + +use std::cell::RefCell; +use std::rc::Rc; + +use crate::contentmapper::{Mapper, Project, SourceFiles}; +use crate::frontend::prelude::*; +use crate::gostd::GoError; +use xxhash_rust::xxh3::xxh3_128; + +static ENABLED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// True when the run that is ending installed a program version. A run can +/// end without one (an early return such as `--version`): its `end_run` +/// must leave `LAST_VERSION` and `RETIRED` alone, or it would release the +/// version that `LAST_VERSION` still names as the `previous` of the next +/// version and retire it before any version was built from it. +static INSTALLED_RUN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +thread_local! { + /// The version that the last run loaded: the `previous` of the next + /// version (`new_program_version`), set by `install_and_enter` and read + /// by it on the next install. It is not the current program of the + /// thread — the compile of a run reads the version through the + /// `ProgramScope` that `install_and_enter` returns, and between runs + /// (and while a version loads) no program is current. + static LAST_VERSION: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// The version before `LAST_VERSION`, kept loaded while the last run's + /// version was built from it. `end_run` releases it; by then the + /// program scope of its run has dropped, so it is not current + /// (`release_program` panics on a current program). + static RETIRED: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// The parses of the file versions that the loads of this process have + /// seen, by path, with the content key the host file system served. A + /// load that reads the same content takes the parse with no read of the + /// parser (one read to hash) and no parse. Files change between runs + /// when the host writes them; the changed content is a new key. + static PARSE_CACHE: RefCell> = RefCell::new(FxHashMap::default()); + /// The paths whose parses the program version of the last run keeps: + /// the files of its program and its config file. `end_run` drops the + /// other parse cache entries, so the parse of a path that left the + /// program is freed with the versions that held it instead of staying + /// for the life of the instance. + static LIVE_PATHS: RefCell> = RefCell::new(FxHashSet::default()); + /// The config file of the last installed version, by the host name the + /// request named. Part of the file world (`file_world_fingerprint`): + /// the config text is the file set of a `files` config, and the config + /// parse does not go through the parse cache (the host reads it before + /// the run installs a version). + static CONFIG_STAMP: RefCell> = const { RefCell::new(None) }; + /// The content hashes that the loads of the run that is running have + /// read (`get_source_file` computes them for its keys), by path. + /// `end_of_run_world_fingerprint` hashes them instead of reading the + /// files again. Every fingerprint takes what it finds, so a hash never + /// outlives the run whose read made it: the pre-run fingerprint + /// (`file_world_fingerprint`) drops the leftover (the host may have + /// changed since that read), and hashes every file fresh. + static RUN_HASHES: RefCell>> = const { RefCell::new(None) }; +} + +struct CachedParse { + /// True when the parse is of a bundled lib file: its text is embedded + /// in the module, so the entry skips the content hash and matches by + /// parse options alone. + bundled: bool, + /// The `xxh3`-128 of the text the host file system served for the + /// parse. Part of the key with the path. + content_hash: u128, + file: Rc, +} + +/// The parse-cache counters of one interval (`take_parse_cache_counts`). +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct ParseCounts { + /// The `get_source_file` calls the cache served (no read of the + /// parser, no parse). + pub hits: u64, + /// The `get_source_file` calls that missed the cache and parsed + /// through the host of the run. A read that failed (a path that does + /// not exist) counts in neither field. + pub parses: u64, +} + +thread_local! { + /// The counters since the last `take_parse_cache_counts`, for the + /// tests' direct cache assertions (the ts_wasm parse-reuse test). + static PARSE_COUNTS: std::cell::Cell = + const { std::cell::Cell::new(ParseCounts { hits: 0, parses: 0 }) }; +} + +/// PORT: not in Go. Test hook: takes the parse-cache counters of this +/// thread since the last take and resets them. +#[doc(hidden)] +#[must_use] +pub fn take_parse_cache_counts() -> ParseCounts { + PARSE_COUNTS.with(std::cell::Cell::take) +} + +/// Enters the version of the last run (`LAST_VERSION`). Between runs the +/// multi-run flow leaves no current program, so a caller that needs one +/// (the ts_wasm wasm test's `release_checker`, an embedder that frees a +/// checker of the last run) makes this scope; it drops on the thread that +/// made it, before the next run installs a version. `None` before the +/// first run. +#[doc(hidden)] +#[must_use = "the version is current only while the scope lives"] +pub fn enter_last_version() -> Option { + LAST_VERSION + .with(|last| last.get()) + .map(|version| crate::core::enter_program(Some(version))) +} + +/// Turns the multi-run flow on for the rest of the process, and turns on the +/// freeable file versions of a watch process (`ast::set_watch_process`), so +/// a re-parse of a changed file is freed with its last holder instead of +/// staying static (watcher.rs `note_kept_parse`, the 9 MiB per edit). +pub fn enable() { + if ENABLED.swap(true, std::sync::atomic::Ordering::Relaxed) { + return; + } + crate::ast::set_watch_process(); +} + +/// True when `enable` has run. +#[must_use] +pub fn is_enabled() -> bool { + ENABLED.load(std::sync::atomic::Ordering::Relaxed) +} + +/// `execute::execute_tsc::install_program` in the multi-run flow: loads the +/// program version of `host` and `config` (the tsc `ProgramOptions`) and +/// returns the scope that makes it the current program of the thread, in +/// place of `program::install_new_program` (`core::set_prog`), which a +/// process can call once. The caller holds the scope across the compile; +/// when it drops, the thread is back to no current program +/// (`core::enter_program` restores what it replaced), which is what +/// `end_run` needs: it releases the retired version, and +/// `release_program` panics on a current program. The scope drops when the +/// compile function returns, before `ts_run` calls `end_run`. +pub(crate) fn install_and_enter( + host: Rc, + config: Rc, +) -> crate::core::ProgramScope { + let config_path = config.config_file.as_ref().map(|c| c.path.clone()); + let config_file_name = config.config_file.as_ref().map(|c| c.file_name.clone()); + CONFIG_STAMP.with(|stamp| *stamp.borrow_mut() = config_file_name); + // The version is parsed and built with no current program (the + // `new_program_version` contract, `core::enter_program`: "None clears + // it, for example while the frontend parses a new version"): this + // scope covers the whole load. `new_frontend_program` and + // `new_program_version` keep their own scopes; this one also covers + // the bookkeeping between them, which reads `np`, not the current + // program — the version of the run before stays out of the way until + // the new one is made current below. + let no_program = crate::core::enter_program(None); + let np = crate::execute::execute_tsc::new_frontend_program(host, config); + crate::program::mark_freeable_parses(&np); + // The paths whose parses this version keeps: the files of its program + // and its config file. `end_run` drops the parse cache entries of the + // other paths, so a path that left the program frees its parse with + // the versions that held it. + LIVE_PATHS.with(|live| { + let mut live = live.borrow_mut(); + live.clear(); + live.extend(np.get_source_files().iter().map(|file| file.path().clone())); + if let Some(config_path) = config_path { + live.insert(config_path); + } + }); + // The `previous` of the new version: the version of the last run, + // which `LAST_VERSION` keeps loaded for exactly this. It is not read + // off the thread — the load runs with no current program, and between + // runs none is current either. + let previous = LAST_VERSION.with(|last| last.get()); + // The semantic-diagnostic reuse set of this run, from `np` and the cache + // of the runs before (see `SemanticCache`). The options pointer is the + // interned copy `build_program` leaks, so runs with the same options + // share it. + prepare_semantic_reuse(&np, super::intern_compiler_options(np.options())); + let version = crate::program::new_program_version(&np, previous); + drop(np); + drop(no_program); + LAST_VERSION.with(|last| last.set(Some(version))); + INSTALLED_RUN.store(true, std::sync::atomic::Ordering::Release); + // The program of the run: the scope makes it current and the caller + // holds it for the compile (see the docstring). + crate::core::enter_program(Some(version)) +} + +/// The end of a run: releases the version that the run before the last +/// loaded, and retires the last version there. Its checker pool stops; the +/// published stores and the parse cache stay, so the next run reuses the +/// parses of its unchanged files. The parse cache entries of paths that +/// left the program are dropped, so their parses free with the versions +/// that held them. Releases nothing before the second run. A run that +/// installed no version (an early return such as `--version`) rotates +/// nothing: `LAST_VERSION` and `RETIRED` keep their versions. No program +/// is current by then: the run's scope (`install_and_enter`) dropped when +/// the compile function returned, so the released version is not current +/// on this thread (`release_program` panics on a current program). +pub fn end_run() { + if !INSTALLED_RUN.swap(false, std::sync::atomic::Ordering::AcqRel) { + return; + } + let retired = RETIRED.with(|retired| retired.replace(None)); + if let Some(old) = retired { + crate::program::release_program(old); + } + RETIRED.with(|retired| retired.set(LAST_VERSION.with(|last| last.get()))); + LIVE_PATHS.with(|live| { + let live = live.borrow(); + PARSE_CACHE.with(|cache| { + cache.borrow_mut().retain(|path, _| live.contains(path)); + }); + }); +} + +/// The host of a run over the parse cache, in the multi-run flow. Every +/// method but `get_source_file` goes to the host of the run (a fresh +/// `CachedFs` compiler host, as a one-run process makes). +struct MultiRunHost { + inner: Rc, +} + +impl MultiRunHost { + /// The cache with a parse of `path` when its content key is the one the + /// host file system serves now and the parse was made with the parse + /// options of this run. A file whose content changed, one whose earlier + /// parse was of other content, or one whose earlier parse used other + /// compiler options (a later request can pass another tsc argument + /// list, and the options choose `external_module_indicator_options`) + /// is a miss. `None` (a failed read) is always a miss. + fn cached( + &self, + path: &Path, + key: Option<(bool, u128)>, + opts: &SourceFileParseOptions, + ) -> Option> { + let (bundled, content_hash) = key?; + PARSE_CACHE.with(|cache| { + let cache = cache.borrow(); + let cached = cache.get(path)?; + (cached.bundled == bundled + && cached.content_hash == content_hash + && cached + .file + .parse_options() + .external_module_indicator_options + == opts.external_module_indicator_options) + .then(|| cached.file.clone()) + }) + } +} + +impl CompilerHost for MultiRunHost { + fn fs(&self) -> Rc { + self.inner.fs() + } + + fn default_library_path(&self) -> String { + self.inner.default_library_path() + } + + fn get_current_directory(&self) -> String { + self.inner.get_current_directory() + } + + fn trace(&self, msg: &'static Message, args: Vec) { + self.inner.trace(msg, args); + } + + // The language server parse cache's test (`project::parsecache`): the + // content the host serves, hashed, then the parse of the cache when the + // hash matches, else the parse of the run's host, cached with the new + // hash. A bundled lib file skips the hash (its text is embedded in the + // module); a failed read is a miss with no insert, as a failed stat + // was in the watch host. + fn get_source_file(&self, opts: &SourceFileParseOptions) -> Option> { + let bundled = crate::frontend::bundled::bundled_text(&opts.file_name).is_some(); + let key = if bundled { + Some((true, 0)) + } else { + let (text, ok) = self.inner.fs().read_file(&opts.file_name); + ok.then(|| (false, xxh3_128(text.as_bytes()))) + }; + // The read's hash is the fingerprint's too (`RUN_HASHES`): the + // post-run check takes it instead of reading the file a second + // time. A failed read is no hash (the post-run check reads the + // path and hashes its failure). + match key { + Some((false, content_hash)) => RUN_HASHES.with(|hashes| { + if let Some(hashes) = hashes.borrow_mut().as_mut() { + hashes.insert(opts.path.clone(), content_hash); + } + }), + None => RUN_HASHES.with(|hashes| { + if let Some(hashes) = hashes.borrow_mut().as_mut() { + hashes.remove(&opts.path); + } + }), + Some((true, _)) => {} + } + if let Some(file) = self.cached(&opts.path, key, opts) { + PARSE_COUNTS.with(|counts| { + let current = counts.get(); + counts.set(ParseCounts { + hits: current.hits + 1, + ..current + }); + }); + return Some(file); + } + // A new parse of a path this process never published is freeable + // too: noting the path published turns `freeable_path` on for the + // parse, so its store owns its nodes, and the `FileVersion` below + // (as `execute::watcher` `note_kept_parse`) frees it with its last + // holder — the first parse of each new per-run path does not stay + // in the leaked arena for the life of the instance. Program files + // get their version from `mark_freeable_parses` at install; this + // covers a parse the program did not take. + crate::ast::note_published_path(&opts.path.0); + let file = self.inner.get_source_file(opts)?; + PARSE_COUNTS.with(|counts| { + let current = counts.get(); + counts.set(ParseCounts { + parses: current.parses + 1, + ..current + }); + }); + if file.version.get().is_none() + && !crate::ast::is_published(file.store) + && crate::ast::freeable_path(&file.path().0) + { + let _ = file.version.set(crate::ast::FileVersion::new(file.store)); + } + if let Some((bundled, content_hash)) = key { + PARSE_CACHE.with(|cache| { + cache.borrow_mut().insert( + opts.path.clone(), + CachedParse { + bundled, + content_hash, + file: file.clone(), + }, + ); + }); + } + Some(file) + } + + fn get_content_mapped_source_files( + &self, + parse_options: &SourceFileParseOptions, + mapper: &Rc, + ) -> Result { + self.inner + .get_content_mapped_source_files(parse_options, mapper) + } + + fn content_mapper_project(&self) -> Option> { + self.inner.content_mapper_project() + } + + fn get_resolved_project_reference( + &self, + file_name: &str, + path: &Path, + ) -> Option> { + self.inner.get_resolved_project_reference(file_name, path) + } + + // The watch host's test: a load with parses in the cache starts no + // workers (they would parse the cached files again for nothing). + fn prefetch_parses(&self) -> bool { + PARSE_CACHE.with(|cache| cache.borrow().is_empty()) && self.inner.prefetch_parses() + } + + // The watch hosts return true (see the trait): a worker parse of a path + // that this process published before is a freeable file version. + fn freeable_worker_parses(&self) -> bool { + true + } +} + +/// The host file system as the parse cache sees it, hashed: every path the +/// cache has parsed, with the hash of the content the file system serves +/// now, and the config file of the last run (`CONFIG_STAMP`). A bundled +/// lib file hashes its path only: its text is embedded in the module and +/// cannot change within the process. A replayed request (`ts_wasm` +/// `FINGERPRINT`) is the same check only while this value is the one +/// stored with its reply: a changed or deleted file changes a read, and a +/// changed config changes the read of the file set it names (a new program +/// file of a `files` config changes the config text). This is the parse +/// cache's own content test (`cached`), so a fingerprint hit means the run +/// would serve the same parses. The pre-run check: any hash a run before +/// left (`RUN_HASHES`) describes files the host may have changed since +/// that read, so it is dropped and every file is read. +pub fn file_world_fingerprint(fs: &dyn Fs) -> u64 { + RUN_HASHES.with(|hashes| *hashes.borrow_mut() = None); + world_fingerprint(fs, None) +} + +/// The post-run check: the content hashes that this run's parse-cache +/// lookups already read (`RUN_HASHES`) stand for their files, so no cached +/// file is read a second time. A path the run never consulted (or whose +/// read failed) is read here, as the pre-run check reads every path. +pub fn end_of_run_world_fingerprint(fs: &dyn Fs) -> u64 { + let run_hashes = RUN_HASHES.with(|hashes| hashes.borrow_mut().take()); + world_fingerprint(fs, run_hashes.as_ref()) +} + +/// The fingerprint over `fs`: the content hash of every path the cache +/// knows, and the config text. A reused hash (`run_hashes`) and a fresh +/// read of the same text produce the same `xxh3` value, so a fingerprint +/// matches the one stored with a reply either way. +fn world_fingerprint(fs: &dyn Fs, run_hashes: Option<&FxHashMap>) -> u64 { + use std::hash::{Hash, Hasher}; + // The reads run with no cache borrow held: a host read can re-enter + // (`HostFs` crosses the wasm boundary). + let entries: Vec<(Path, String, bool)> = PARSE_CACHE.with(|cache| { + let mut entries: Vec<(Path, String, bool)> = cache + .borrow() + .iter() + .map(|(path, cached)| { + ( + path.clone(), + cached.file.parse_options().file_name.clone(), + cached.bundled, + ) + }) + .collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + entries + }); + let config_file_name = CONFIG_STAMP.with(|stamp| stamp.borrow().clone()); + let mut hasher = std::hash::DefaultHasher::new(); + for (path, file_name, is_bundled) in &entries { + path.0.hash(&mut hasher); + if *is_bundled { + continue; + } + if let Some(content_hash) = run_hashes.and_then(|hashes| hashes.get(path)) { + true.hash(&mut hasher); + content_hash.hash(&mut hasher); + continue; + } + let (text, ok) = fs.read_file(file_name); + ok.hash(&mut hasher); + xxh3_128(text.as_bytes()).hash(&mut hasher); + } + if let Some(file_name) = config_file_name { + hasher.write(b"config"); + file_name.hash(&mut hasher); + let (text, ok) = fs.read_file(&file_name); + ok.hash(&mut hasher); + text.hash(&mut hasher); + } + hasher.finish() +} + +/// The compiler host of a run in the multi-run flow: the run's host over the +/// process parse cache. With the flow off, the host unchanged. +pub(crate) fn wrap_host(host: Rc) -> Rc { + if !is_enabled() { + return host; + } + Rc::new(MultiRunHost { inner: host }) +} + +// ---- semantic-diagnostic reuse (phase 9) ---- +// +// The check phase of a warm run is the whole floor (the decomposition that +// drove this: `C_install_program` 1-3 ms, `D_check_emit` 48-54 ms of a +// 55-70 ms run, ~47-50 ms of it `get_semantic_diagnostics` over all files — +// mostly the 57 lib .d.ts files). The parse cache already makes the parses +// of unchanged files shared `Rc`s, so a file whose parse is the same `Rc` +// as the run before was checked under the same text — and when nothing it +// reads changed, its semantic diagnostics are the same list. This cache +// hands those lists back instead of re-checking the files, so a warm run +// checks only what a change can reach. +// +// Safety rules (a reused list must be the list a full check would compute): +// +// 1. Same program shape: the same path set as the cached run, and the same +// interned compiler options (a pointer compare). Else everything is +// re-checked. +// 2. Changed files (a new parse store for their path) must be external +// modules with no `declare global`/ambient modules and no +// augmentations: their declarations cannot reach the global scope that +// every file shares. A changed script can (a `const console` in a guest +// script reports TS2451 on guest-globals.d.ts too — measured), so a +// changed script re-checks everything. +// 3. A file is reusable only when it imports no changed file, not even +// transitively (the resolved module and type-reference edges of the +// program) and carries no triple-slash references (those resolve +// outside `resolved_modules`). +// 4. `GOPORT_SEMANTIC_CACHE=0` turns the reuse off (a kill switch for a +// diagnosis). +// +// The cache holds no program, no tables and no checker: `Rc` diagnostics +// keyed by path, so `release_program` lifetime rules are untouched. + +struct CachedDiags { + store: usize, + diags: Rc>, +} + +thread_local! { + /// The per-path semantic diagnostics of the last run that computed + /// them, with the parse store they belong to. + static SEMANTIC_CACHE: RefCell = RefCell::new(SemanticCache { + options: None, + by_path: FxHashMap::default(), + contributions: FxHashMap::default(), + }); + /// Store id -> cached diagnostics that the check of the run that is + /// running now returns instead of re-checking. Cleared at every + /// `prepare_semantic_reuse`. + static SEMANTIC_REUSE: RefCell>>> = + RefCell::new(FxHashMap::default()); + /// Store id -> path of every file of the run that is running now, for + /// `note_semantic_diagnostics` (the check reads nodes, not parses). + static NOTE_PATHS: RefCell> = + RefCell::new(FxHashMap::default()); +} + +struct SemanticCache { + /// The interned options the cache was built under. + options: Option<&'static crate::options::CompilerOptions>, + by_path: FxHashMap, + /// The global-scope contribution hash of each cached global-scope file + /// (`global_contribution_hash`), from the run that cached its + /// diagnostics. Cleared with `by_path`. + contributions: FxHashMap, +} + +/// True unless `GOPORT_SEMANTIC_CACHE=0`. Read once. +fn semantic_cache_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var("GOPORT_SEMANTIC_CACHE").map_or(true, |v| v != "0")) +} + +/// A file whose change can reach the global scope every file shares: a +/// script, or a module that augments or declares ambient modules. +fn reaches_global_scope(file: &ParsedSourceFile) -> bool { + file.external_module_indicator.is_nil() + || !file.module_augmentations.is_empty() + || !file.ambient_module_names.is_empty() +} + +/// What a script file contributes to the global scope, hashed: every +/// top-level statement's kind and source text — except a function +/// declaration with an explicit return type annotation, which contributes +/// its signature only (the text up to its body): a body-only edit cannot +/// change the type other files see. A changed script whose hash is the one +/// the cache saw (the common agent edit: a new body for the same signed +/// function) cannot contaminate any other file, so the run re-checks just +/// the script. +fn global_contribution_hash(file: &ParsedSourceFile) -> u64 { + use std::hash::{Hash, Hasher}; + let full: &str = match &file.text { + crate::ast::FileText::Static(text) => text, + crate::ast::FileText::Shared(text) => text, + }; + let mut hasher = std::hash::DefaultHasher::new(); + for stmt in file.root.iter_children() { + stmt.kind().hash(&mut hasher); + let end = if stmt.kind() == crate::astdata::SyntaxKind::FunctionDeclaration + && !stmt.body().is_nil() + && !stmt.type_().is_nil() + { + stmt.body().pos() + } else { + stmt.end() + }; + let start = usize::try_from(stmt.pos()).unwrap_or(0); + let end = usize::try_from(end).unwrap_or(full.len()).min(full.len()); + hasher.write(full.get(start..end).unwrap_or_default().as_bytes()); + } + hasher.finish() +} + +/// The program files that (transitively) import a changed file: the reverse +/// of the program's resolved module and type-reference edges, walked from +/// the changed paths. +fn influenced_by( + np: &crate::frontend::compiler::NewProgram, + changed: &FxHashMap, +) -> FxHashSet { + let mut reverse: FxHashMap< + &crate::frontend::tspath::Path, + Vec<&crate::frontend::tspath::Path>, + > = FxHashMap::default(); + for (importer, cache) in np.processed_files.resolved_modules.iter() { + for resolution in cache.values() { + if resolution.is_resolved() { + reverse + .entry(&resolution.resolved_path) + .or_default() + .push(importer); + } + } + } + for (importer, cache) in np.processed_files.type_resolutions_in_file.iter() { + for resolution in cache.values() { + if resolution.is_resolved() { + reverse + .entry(&resolution.resolved_path) + .or_default() + .push(importer); + } + } + } + let mut influenced: FxHashSet<&crate::frontend::tspath::Path> = changed.keys().collect(); + let mut queue: Vec<&crate::frontend::tspath::Path> = influenced.iter().copied().collect(); + while let Some(path) = queue.pop() { + if let Some(importers) = reverse.get(path) { + for importer in importers { + if influenced.insert(importer) { + queue.push(importer); + } + } + } + } + influenced.into_iter().cloned().collect() +} + +/// Computes the reuse set of the run that is loading now, from `np` and the +/// cache of the runs before. Called on the loading thread before +/// `new_program_version`; `options` is the interned copy of the run's +/// compiler options. +fn prepare_semantic_reuse( + np: &crate::frontend::compiler::NewProgram, + options: &'static crate::options::CompilerOptions, +) { + SEMANTIC_REUSE.with(|reuse| reuse.borrow_mut().clear()); + let files = np.source_files(); + NOTE_PATHS.with(|note| { + let mut note = note.borrow_mut(); + note.clear(); + note.extend(files.iter().map(|f| (f.store, f.path().clone()))); + }); + if !semantic_cache_enabled() { + return; + } + SEMANTIC_CACHE.with(|cache_cell| { + SEMANTIC_REUSE.with(|reuse_cell| { + let mut cache = cache_cell.borrow_mut(); + let mut reuse = reuse_cell.borrow_mut(); + // Rule 1: the options changed since the cache was built. + if !cache + .options + .is_some_and(|cached| std::ptr::eq(cached, options)) + { + cache.options = Some(options); + cache.by_path.clear(); + } + // Rule 1: the path set changed. + let same_set = cache.by_path.len() == files.len() + && files.iter().all(|f| cache.by_path.contains_key(f.path())); + if !same_set { + cache.by_path.clear(); + return; + } + // The files whose parse changed (or are new to the cache, which + // the same-set test rules out). + let changed: FxHashMap = files + .iter() + .filter(|f| { + cache + .by_path + .get(f.path()) + .is_none_or(|c| c.store != f.store) + }) + .map(|f| (f.path().clone(), f.store)) + .collect(); + if changed.is_empty() { + // Every parse is the run before's: every cached list stands. + for file in files { + let cached = &cache.by_path[file.path()]; + reuse.insert(file.store, Rc::clone(&cached.diags)); + } + return; + } + // Rule 2: a changed file that can reach the global scope keeps + // the rest reusable only when its contribution is the one the + // cache saw (a body-only edit of an annotated function). A new + // top-level name or signature re-checks everything. + let mut global_change = false; + for file in files { + if !changed.contains_key(file.path()) || !reaches_global_scope(file) { + continue; + } + let hash = global_contribution_hash(file); + global_change |= cache.contributions.get(file.path()) != Some(&hash); + cache.contributions.insert(file.path().clone(), hash); + } + if global_change { + // The stored contribution stays: it is the new parse's, and + // the next change of this path compares against it. + cache.by_path.clear(); + return; + } + // Rule 3: keep the files outside the influence of the changed + // ones that carry no triple-slash references. + let influenced = influenced_by(np, &changed); + for file in files { + if changed.contains_key(file.path()) || influenced.contains(file.path()) { + continue; + } + if !file.referenced_files.is_empty() || !file.type_reference_directives.is_empty() { + continue; + } + let cached = &cache.by_path[file.path()]; + reuse.insert(file.store, Rc::clone(&cached.diags)); + } + }); + }); +} +/// The path of a published source-file node of the run that is checking +/// now (`NOTE_PATHS`, from the same load). +fn noted_path(store: usize) -> Option { + NOTE_PATHS.with(|note| note.borrow().get(&store).cloned()) +} + +/// `program::get_semantic_diagnostics(Node::NIL)` of a multi-run process: +/// re-checks only the files outside `SEMANTIC_REUSE`, splices the cached +/// lists of the rest in file order, notes what it computed, and sorts once +/// at the end — the same calls, order and result as the full fan-out, with +/// the reused files skipped. +pub(crate) fn semantic_diagnostics_with_reuse() -> Vec { + let files = super::source_files(); + let reuse = SEMANTIC_REUSE.with(|reuse| std::mem::take(&mut *reuse.borrow_mut())); + enum Slot { + Cached(Rc>), + Check(usize), + } + let mut slots: Vec = Vec::with_capacity(files.len()); + let mut to_check: Vec = Vec::with_capacity(files.len()); + for file in &files { + match reuse.get(&file.file_index()) { + Some(diags) => slots.push(Slot::Cached(Rc::clone(diags))), + None => { + slots.push(Slot::Check(to_check.len())); + to_check.push(*file); + } + } + } + let mut checked = if to_check.is_empty() { + Vec::new() + } else { + super::collect_checker_diagnostics_from_files(&to_check, |c, f| { + super::get_semantic_diagnostics_with_checker(&crate::gostd::context::background(), c, f) + }) + }; + let mut fresh: Vec<( + crate::frontend::tspath::Path, + usize, + Rc>, + )> = Vec::with_capacity(checked.len()); + let mut out = Vec::new(); + for (i, file) in files.iter().enumerate() { + match &slots[i] { + Slot::Cached(diags) => out.extend(diags.iter().cloned()), + Slot::Check(k) => { + let diags = std::mem::take(&mut checked[*k]); + let diags = Rc::new(diags); + if let Some(path) = noted_path(file.file_index()) { + fresh.push((path, file.file_index(), Rc::clone(&diags))); + } + out.extend(diags.iter().cloned()); + } + } + } + if semantic_cache_enabled() { + SEMANTIC_CACHE.with(|cache| { + let mut cache = cache.borrow_mut(); + for (path, store, diags) in fresh { + cache.by_path.insert(path, CachedDiags { store, diags }); + } + }); + } + super::filter_and_sort_diagnostics(out) +} diff --git a/crates/ts_wasm/src/host.rs b/crates/ts_wasm/src/host.rs index 13fec4ff1..df83f1c0b 100644 --- a/crates/ts_wasm/src/host.rs +++ b/crates/ts_wasm/src/host.rs @@ -307,12 +307,32 @@ impl Fs for HostFs { pub mod test_host { use super::Op; use std::collections::BTreeMap; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; static FILES: Mutex>> = Mutex::new(BTreeMap::new()); + /// Bumped at each `set_files`: the mtime that `Stat` reports (in + /// nanoseconds), so a file that a later `set_files` wrote is a parse + /// cache miss (`program::multirun`), even when its size is the same. + static GENERATION: AtomicU64 = AtomicU64::new(1); + + /// When set, `Stat` reports one mtime for every file, whatever + /// `set_files` wrote: a file system with no mtime honesty, as the npm + /// package's `memoryFileSystem` is (every file at mtime 0). The + /// multi-run parse cache keys its entries by content, so it serves + /// such a host; keyed by mtime it would replay a stale parse of a + /// changed file. + static CONSTANT_MTIME: AtomicBool = AtomicBool::new(false); + + /// Sets whether `Stat` reports a constant mtime (see `CONSTANT_MTIME`). + pub fn set_constant_mtime(constant: bool) { + CONSTANT_MTIME.store(constant, Ordering::Relaxed); + } + /// Replaces the files of the test host. pub fn set_files(files: impl IntoIterator) { + GENERATION.fetch_add(1, Ordering::Relaxed); let mut map = FILES.lock().unwrap_or_else(PoisonError::into_inner); *map = files .into_iter() @@ -344,10 +364,17 @@ pub mod test_host { let is_dir = map.keys().any(|key| key.starts_with(&prefix)); match op { Op::Read => map.get(path).cloned(), - Op::Stat => match map.get(path) { - Some(bytes) => Some(format!("f {} 0", bytes.len()).into_bytes()), - None => is_dir.then(|| b"d 0 0".to_vec()), - }, + Op::Stat => { + let mtime = if CONSTANT_MTIME.load(Ordering::Relaxed) { + 1 + } else { + GENERATION.load(Ordering::Relaxed) + }; + match map.get(path) { + Some(bytes) => Some(format!("f {} {}", bytes.len(), mtime).into_bytes()), + None => is_dir.then(|| format!("d 0 {mtime}").into_bytes()), + } + } Op::ReadDir => { if !is_dir { return None; diff --git a/crates/ts_wasm/src/lib.rs b/crates/ts_wasm/src/lib.rs index b1de60f73..bb092bd8b 100644 --- a/crates/ts_wasm/src/lib.rs +++ b/crates/ts_wasm/src/lib.rs @@ -4,8 +4,13 @@ //! //! The host writes a request to the input buffer (`ts_input`), calls //! `ts_run`, and reads the reply at `ts_output` (`ts_output_len` bytes). -//! One instance runs one request, because the port keeps one program per -//! process (`core::set_prog`). The host makes a new instance for each. +//! One instance serves many requests: each `ts_run` loads a new program +//! version (`program::multirun`), whose unchanged files keep their parses +//! of the runs before, so a warm run parses only what changed. The parse +//! cache is keyed by the content of each file (bundled lib files skip the +//! hash: their text is embedded in the module), so the host needs no +//! mtime contract. A panic still ends the instance (wasm cannot unwind): +//! the host makes a new one. //! //! The request is NUL-separated UTF-8 fields: the current directory, the //! flags (a decimal number of `FLAG_*` bits), then the tsc arguments. tsc @@ -17,6 +22,7 @@ //! `proc_exit`: 2 for a Go panic, 70 for any other. use std::cell::RefCell; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::rc::Rc; use std::sync::Arc; @@ -29,6 +35,24 @@ use ts_goport::frontend::vfs::{Fs, OsOverride, install_os_override}; use ts_goport::gostd::context; use ts_goport::scanner_util::go_string_from_utf8; +/// The current directory of the run that installed the OS override (see +/// `run`). Every run of the instance must ask for it. +static OVERRIDE_CWD: std::sync::OnceLock = std::sync::OnceLock::new(); + +// E1: the request hash, the host world hash and the reply of the last run +// that used the whole pipeline. An identical request (the same bytes: cwd, +// flags and args) replayed while every file the parse cache knows serves +// the content it had (`multirun::file_world_fingerprint` — the module's +// only view of the files is what its loads read and hashed, and the held +// flow's config text is a function of its file set) returns the stored +// status and reply with no version, no check and no marshal. A changed +// guest reads differently, so its rerun re-checks. Only a +// `FLAG_DIAGNOSTICS_JSON` run is stored: its reply carries the whole +// answer, while a text run would also have to replay stdout. +thread_local! { + static FINGERPRINT: RefCell> = const { RefCell::new(None) }; +} + pub mod host; /// Report the diagnostics as JSON in the reply, not as text on stdout. @@ -92,18 +116,94 @@ pub fn run(request: &str) -> (i32, String) { eprintln!("error: diagnostics as JSON are not supported with --build"); return (1, String::new()); } + // E1: an identical request over an unchanged host file system replays + // the last reply (see `FINGERPRINT`). The hash covers the whole request; + // the world hash is every path the parse cache knows, read and hashed + // now (bundled lib files hash their path only), so a changed, deleted + // or newly written guest never hits. + let mut fingerprint = DefaultHasher::new(); + request.hash(&mut fingerprint); + let request_hash = fingerprint.finish(); + let world_hash = if flags & FLAG_DIAGNOSTICS_JSON != 0 { + ts_goport::program::multirun::file_world_fingerprint(&host::HostFs::new()) + } else { + 0 + }; + if flags & FLAG_DIAGNOSTICS_JSON != 0 + && let Some((seen, world, status, reply)) = FINGERPRINT.with(|cell| { + cell.borrow() + .as_ref() + .map(|(h, w, s, r)| (*h, *w, *s, r.clone())) + }) + && seen == request_hash + && world == world_hash + { + return (status, reply); + } - let _ = host::CASE_SENSITIVE.set(flags & FLAG_CASE_INSENSITIVE == 0); - install_os_override(OsOverride { - fs: Arc::new(|| Rc::new(host::HostFs::new()) as Rc), - current_directory: cwd, - }); + // The case-insensitivity of the host file system is read once per + // instance (`host::CASE_SENSITIVE`): a later run with the other flag + // would run in the wrong mode with no error. Reject it explicitly, as + // a cwd change below. + let case_sensitive = flags & FLAG_CASE_INSENSITIVE == 0; + match host::CASE_SENSITIVE.get() { + Some(installed) if *installed != case_sensitive => { + eprintln!( + "error: this instance serves a {} file system, the request asks for {} (multi-run: one case mode per instance)", + if *installed { + "case-sensitive" + } else { + "case-insensitive" + }, + if case_sensitive { + "case-sensitive" + } else { + "case-insensitive" + }, + ); + return (1, String::new()); + } + Some(_) => {} + None => { + let _ = host::CASE_SENSITIVE.set(case_sensitive); + } + } + // The override replaces the OS for the process, so it installs once + // (`osvfs::install_os_override` panics on the second call): every run + // of the instance resolves paths against the directory of the first + // one. A request with another cwd is rejected explicitly instead of + // being served with the wrong directory. The file system closure makes + // a new `HostFs` per thread, so the file system of a run is the host's + // now. + match OVERRIDE_CWD.get() { + Some(installed) if *installed != cwd => { + eprintln!( + "error: cwd is {cwd}, but this instance serves {installed} (multi-run: one cwd per instance)" + ); + return (1, String::new()); + } + Some(_) => {} + None => { + install_os_override(OsOverride { + fs: Arc::new(|| Rc::new(host::HostFs::new()) as Rc), + current_directory: cwd.clone(), + }); + let _ = OVERRIDE_CWD.set(cwd); + } + } ts_goport::execute::tsc::stdio::init(); let sys: Rc = match new_os_system() { Ok(sys) => Rc::new(sys), Err(status) => return (status.code(), String::new()), }; + // Multi-run (see the `program::multirun` module): the run's program is + // a program version, built from the version of the run before, whose + // unchanged files keep their parses. The compile holds the program + // scope (`multirun::install_and_enter`), which drops with the compile + // frame: no program is current here. `end_run` releases the version of + // the run before the last, after the diagnostics of this run are made. + ts_goport::program::multirun::enable(); let diagnostics = (flags & FLAG_DIAGNOSTICS_JSON != 0).then(Rc::default); let hooks = WasmTsc { diagnostics: diagnostics.clone(), @@ -111,10 +211,25 @@ pub fn run(request: &str) -> (i32, String) { let status = command_line(&context::background(), sys.clone(), &args, &hooks) .status .code(); + ts_goport::program::multirun::end_run(); let _ = sys.writer().borrow_mut().flush(); let reply = diagnostics.map_or_else(String::new, |diagnostics| { format!("[{}]", diagnostics.take().join(",")) }); + // E1: remember the answer of this exact request over this exact host + // state (see `FINGERPRINT`). The world is hashed again: the run may + // have written files (an emit), and the replay must not serve a reply + // from before them. The post-run check reuses the content hashes the + // run's parse-cache lookups read (`multirun:: + // end_of_run_world_fingerprint`): no cached file is read a second + // time. + if flags & FLAG_DIAGNOSTICS_JSON != 0 { + let world = + ts_goport::program::multirun::end_of_run_world_fingerprint(&host::HostFs::new()); + FINGERPRINT.with(|cell| { + *cell.borrow_mut() = Some((request_hash, world, status, reply.clone())); + }); + } (status, reply) } @@ -211,17 +326,24 @@ mod exports { } } -// Native only: a process runs one request, and the wasm test binary makes -// its one request in `wasm_tests`. +// Native only: the tests run over `host::test_host`. A wasm build has no +// test host, because its file system is the JavaScript host's; its test +// binary makes its one request in `wasm_tests`. #[cfg(all(test, not(target_family = "wasm")))] mod tests { use super::*; + use std::sync::Mutex; + use std::sync::PoisonError; + + /// The test host of the process is one state (the files, the OS + /// override, the parse cache), so the tests take turns. + static HOST: Mutex<()> = Mutex::new(()); /// One run over the test host: the reply has the diagnostics as JSON, - /// and the emit writes to the host. A process runs one request, so this - /// is the crate's only test of `run`. + /// and the emit writes to the host. #[test] fn run_checks_and_emits_through_the_host() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); host::test_host::set_files([ ( "/p/tsconfig.json".to_string(), @@ -244,6 +366,466 @@ mod tests { Some("export const n = 'one';\n") ); } + + /// PORT: not in Go. The multi-run flow (`program::multirun`): many + /// runs in this one process (one `WebAssembly.Instance` in the shipped + /// module), across guests that change between runs. Every run reports + /// the diagnostics of the files it was given, including a file whose + /// text changed: the parse cache is keyed by content, so the changed + /// text is a cache miss and the parse is redone. + #[test] + fn many_runs_in_one_process_report_each_guest() { + const CLEAN_A: &str = "export const a: number = 1;\n"; + const B: &str = "import { a } from './a';\nexport const b: number = a + 1;\n"; + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts","b.ts"]}"# + .to_string(), + ) + }; + let a = |text: &str| ("/p/a.ts".to_string(), text.to_string()); + let b = |text: &str| ("/p/b.ts".to_string(), text.to_string()); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + + // Run 1: a clean guest (cold: no parse cache, no earlier version). + host::test_host::set_files([tsconfig(), a(CLEAN_A), b(B)]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + assert_eq!( + host::test_host::file("/p/out/a.js").as_deref(), + Some("export const a = 1;\n"), + "run 1 emit" + ); + + // Run 2: a.ts now has a type error, b.ts unchanged. The changed + // file must be re-parsed (stale-parse test); the unchanged one may + // reuse its parse. + host::test_host::set_files([tsconfig(), a("export const a: number = 'one';\n"), b(B)]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + assert!(reply.contains(r#""fileName":"/p/a.ts""#), "{reply}"); + assert_eq!( + host::test_host::file("/p/out/a.js").as_deref(), + Some("export const a = 'one';\n"), + "run 2 emit" + ); + + // Run 3: the error is fixed, and the reply is clean again. + host::test_host::set_files([tsconfig(), a("export const a: number = 2;\n"), b(B)]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + + // Run 4: exactly the files of run 3 again (the warm, unchanged + // case: every parse comes from the cache). + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + assert_eq!( + host::test_host::file("/p/out/b.js").as_deref(), + Some("import { a } from './a';\nexport const b = a + 1;\n"), + "run 4 emit" + ); + } + + /// The parse cache serves the parses of unchanged files across runs, + /// asserted on the cache's own counters (`multirun::take_parse_cache_counts`), + /// and the version chain stays sound across the `end_run` rotation: run + /// 1 parses every file (cold), the runs after — including a third run, + /// past the rotation that released the first version — parse nothing + /// and take every parse from the cache, and an edit re-parses only the + /// changed file while the rest stays cached. The request bytes + /// alternate (`-p .` / `-p /p`), so no run replays the fingerprint + /// reply of the run before (`run` compares only against the last + /// run's request) and every run really loads a version over the parse + /// cache. + #[test] + fn parse_cache_reuses_parses_across_runs() { + const CLEAN_A: &str = "export const a: number = 1;\n"; + const B: &str = "import { a } from './a';\nexport const b: number = a + 1;\n"; + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts","b.ts"]}"# + .to_string(), + ) + }; + let a = |text: &str| ("/p/a.ts".to_string(), text.to_string()); + let b = || ("/p/b.ts".to_string(), B.to_string()); + let request = |project: &str| format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0{project}"); + + // Run 1 (cold): every parse misses the cache. + host::test_host::set_files([tsconfig(), a(CLEAN_A), b()]); + let (status, reply) = run(&request(".")); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + let cold = ts_goport::program::multirun::take_parse_cache_counts(); + assert!(cold.parses > 0, "{cold:?}"); + assert_eq!(cold.hits, 0, "{cold:?}"); + + // Run 2 (warm, unchanged): no parse at all, every file served by + // the cache. + let (status, reply) = run(&request("/p")); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + let warm = ts_goport::program::multirun::take_parse_cache_counts(); + assert_eq!(warm.parses, 0, "{warm:?}"); + assert!(warm.hits > 0, "{warm:?}"); + + // Run 3: a third run, past the rotation that released the version + // of run 1 (`end_run`): the reuse chain survives it, still without + // one parse. + let (status, reply) = run(&request(".")); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + let warm3 = ts_goport::program::multirun::take_parse_cache_counts(); + assert_eq!(warm3.parses, 0, "{warm3:?}"); + assert!(warm3.hits > 0, "{warm3:?}"); + + // Run 4: a.ts edited (same length, so size-based caches would not + // notice either): only the changed file re-parses, the rest of the + // program stays cached, and the fresh check reports the error. + host::test_host::set_files([tsconfig(), a("export const a: number = 'one';\n"), b()]); + let (status, reply) = run(&request("/p")); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + assert!(reply.contains(r#""fileName":"/p/a.ts""#), "{reply}"); + let edit = ts_goport::program::multirun::take_parse_cache_counts(); + assert!(edit.parses >= 1, "{edit:?}"); + assert!(edit.hits > 0, "{edit:?}"); + } + + /// An early-return run (`--version`) installs no program version: its + /// `end_run` must rotate nothing, and a compile after it must still + /// work on the version chain (the released-as-`previous` bug). + #[test] + fn early_return_run_does_not_rotate_versions() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let files = || { + [ + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts"]}"# + .to_string(), + ), + ( + "/p/a.ts".to_string(), + "export const n: number = 1;\n".to_string(), + ), + ] + }; + host::test_host::set_files(files()); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + + // Run 1: installs version 1; `end_run` retires it. + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + + // Run 2: an early return installs nothing. + let (status, reply) = run(&["/p", "0", "--version"].join("\0")); + assert_eq!(status, 0, "{reply}"); + + // Run 3: must build on the live version, not a released one. + host::test_host::set_files(files()); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + } + + /// A run whose cwd differs from the one of the instance is rejected + /// explicitly (`run` documents one cwd per multi-run instance). + #[test] + fn cwd_change_is_rejected() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + host::test_host::set_files([ + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"types":[]},"files":["a.ts"]}"#.to_string(), + ), + ( + "/p/a.ts".to_string(), + "export const n: number = 1;\n".to_string(), + ), + ]); + let (status, reply) = run(&format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0.")); + assert_eq!(status, 0, "{reply}"); + let (status, reply) = run(&format!("/q\0{FLAG_DIAGNOSTICS_JSON}\0-p\0.")); + assert_eq!(status, 1, "{reply}"); + assert!(reply.is_empty(), "{reply}"); + } + + /// A run whose case-insensitivity flag differs from the one the + /// instance installed is rejected explicitly (`run` documents one case + /// mode per multi-run instance), and the instance keeps serving the + /// installed mode after the rejected run. + #[test] + fn case_flag_change_is_rejected() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + host::test_host::set_files([ + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"types":[]},"files":["A.ts"]}"#.to_string(), + ), + ( + "/p/A.ts".to_string(), + "export const n: number = 1;\n".to_string(), + ), + ]); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + // The other flag: rejected with no reply. + let (status, reply) = run(&format!( + "/p\0{}\0-p\0.", + FLAG_DIAGNOSTICS_JSON | FLAG_CASE_INSENSITIVE + )); + assert_eq!(status, 1, "{reply}"); + assert!(reply.is_empty(), "{reply}"); + // The instance still serves the installed mode. + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + } + + /// A file that leaves the program frees its parse (`end_run` evicts + /// the parse cache entries of paths that left): a later run that + /// brings the path back with other content reports THIS content — a + /// stale kept parse of the old text must not be served. + #[test] + fn a_path_that_returns_with_other_content_rereports() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = |files: &str| { + ( + "/p/tsconfig.json".to_string(), + format!(r#"{{"compilerOptions":{{"outDir":"out","types":[]}},"files":[{files}]}}"#), + ) + }; + let clean = "export const b: number = 1;\n"; + let broken = "export const b: number = 'one';\n"; + let a = ( + "/p/a.ts".to_string(), + "export const a: number = 1;\n".to_string(), + ); + let b = |text: &str| ("/p/b.ts".to_string(), text.to_string()); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + + // Run 1: a.ts and b.ts, clean. + host::test_host::set_files([tsconfig(r#""a.ts","b.ts""#), a.clone(), b(clean)]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + + // Run 2: b.ts left the program (its parse is evicted at end_run). + host::test_host::set_files([tsconfig(r#""a.ts""#), a.clone()]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + + // Run 3: b.ts is back with other content: the fresh parse must be + // checked, and the error must be reported. + host::test_host::set_files([tsconfig(r#""a.ts","b.ts""#), a.clone(), b(broken)]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""fileName":"/p/b.ts""#), "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + } + + // E1 + the semantic-diagnostic reuse: the reply of a run whose request + // bytes are the same as the run before must still be THIS run's — the + // host file system changed under the identical request, so neither the + // replay nor the per-file cache may serve the old answer. + #[test] + fn identical_request_over_a_changed_guest_rereports() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts"]}"#.to_string(), + ) + }; + let red = "const n: number = 'not a number';\nconsole.log(n);\n"; + host::test_host::set_files([tsconfig(), ("/p/a.ts".to_string(), red.to_string())]); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + // Same request bytes, new text: the world hash changes with the + // content, so the replay must miss and the fresh check must report + // clean. + host::test_host::set_files([ + tsconfig(), + ( + "/p/a.ts".to_string(), + "export const a: number = 1;\n".to_string(), + ), + ]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + // And back to red (the parse and the cached per-file lists are for + // the old parse): the fresh check must report the error again. + host::test_host::set_files([tsconfig(), ("/p/a.ts".to_string(), red.to_string())]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + } + + /// An incremental config takes `perform_incremental_compilation`, + /// whose host must sit over the parse cache like a plain run's host + /// (`multirun::wrap_host`): the build info read, the version install + /// and the incremental program all run over the wrapped host. The + /// runs below change the guest between runs in this one process, and + /// each reply must be the fresh check's. + #[test] + fn incremental_config_runs_report_each_guest() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"incremental":true,"outDir":"out","types":[]},"files":["a.ts"]}"# + .to_string(), + ) + }; + let a = |text: &str| ("/p/a.ts".to_string(), text.to_string()); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + + // Run 1: red, cold (no build info yet): the full check reports. + host::test_host::set_files([tsconfig(), a("export const n: number = 'one';\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + assert!( + host::test_host::file("/p/out/tsconfig.tsbuildinfo").is_some(), + "run 1 build info" + ); + + // Run 2: the guest is fixed (a parse-cache hit for the unchanged + // config and libs, a re-parse of the changed a.ts): the reply is + // the fresh check's. + host::test_host::set_files([tsconfig(), a("export const n: number = 1;\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + + // Run 3: red again over the warm build info: the error is + // reported again, and the emit is the new text's. + host::test_host::set_files([tsconfig(), a("export const n: number = 'two';\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + assert!(reply.contains("'two'"), "{reply}"); + assert_eq!( + host::test_host::file("/p/out/a.js").as_deref(), + Some("export const n = 'two';\n"), + "run 3 emit" + ); + } + + /// The reviewer's repro: a host that reports one mtime for every file + /// (the npm package's `memoryFileSystem`, `host::test_host` + /// `set_constant_mtime`). The parse cache is keyed by the content the + /// host serves, so a guest whose text changed under the constant mtime + /// is re-parsed and re-reported; a cache keyed by mtime would replay + /// the stale parse of the old text and report the old answer. + #[test] + fn constant_mtime_host_still_rechecks_changed_content() { + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + host::test_host::set_constant_mtime(true); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts"]}"#.to_string(), + ) + }; + let a = |text: &str| ("/p/a.ts".to_string(), text.to_string()); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + + // Run 1: a red guest. The mtime is the same for every file and + // stays so for the whole test. + host::test_host::set_files([tsconfig(), a("export const n: number = 'one';\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + + // Run 2: only the content changed (same length, so the same size + // too), and the mtime is still constant. The fresh parse must be + // checked: the reply is the new content's. + host::test_host::set_files([tsconfig(), a("export const n: number = 3.141;\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + assert_eq!( + host::test_host::file("/p/out/a.js").as_deref(), + Some("export const n = 3.141;\n"), + "run 2 emit" + ); + + // Run 3: back to red (the parse and the cached per-file lists are + // for the green text): the error is reported again. + host::test_host::set_files([tsconfig(), a("export const n: number = 'one';\n")]); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""code":2322"#), "{reply}"); + host::test_host::set_constant_mtime(false); + } + + // A script guest whose top-level shape is the same (the same annotated + // function, a new body) re-checks only itself; a script whose top-level + // shape changed re-checks everything. Both replies must be the fresh + // check's. + #[test] + fn script_body_change_rereports_and_shape_change_rechecks() { + const HEAD: &str = "function slugify(input: string): string {\n"; + const FOOT: &str = "}\ntest('a', () => { expect(slugify('A B')).toEqual('a-b'); });\n"; + const GLOBALS: &str = "declare function test(name: string, fn: () => void): void;\ndeclare function expect(actual: unknown): { toEqual(expected: unknown): void; };\n"; + let _host = HOST.lock().unwrap_or_else(PoisonError::into_inner); + let tsconfig = || { + ( + "/p/tsconfig.json".to_string(), + r#"{"compilerOptions":{"outDir":"out","types":[]},"files":["a.ts","globals.d.ts"]}"#.to_string(), + ) + }; + let broken = + format!("{HEAD} const slug: number = input.toLowerCase();\n return slug;\n{FOOT}"); + let fixed = + format!("{HEAD} return input.trim().toLowerCase().replace(/\\s+/g, \"-\");\n{FOOT}"); + host::test_host::set_files([ + tsconfig(), + ("/p/globals.d.ts".to_string(), GLOBALS.to_string()), + ("/p/a.ts".to_string(), broken), + ]); + let request = || format!("/p\0{FLAG_DIAGNOSTICS_JSON}\0-p\0."); + let (status, reply) = run(&request()); + assert_eq!(status, 2, "{reply}"); + assert!(reply.contains(r#""fileName":"/p/a.ts""#), "{reply}"); + // The body edit keeps the signature: the guest is re-checked and + // the reply is the fixed one. + host::test_host::set_files([ + tsconfig(), + ("/p/globals.d.ts".to_string(), GLOBALS.to_string()), + ("/p/a.ts".to_string(), fixed), + ]); + let (status, reply) = run(&request()); + assert_eq!(status, 0, "{reply}"); + assert_eq!(reply, "[]", "{reply}"); + // A new top-level declaration changes the global scope: the whole + // program re-checks, and the duplicate it makes is reported on both + // sides that the full check reports (here: the globals side keeps + // its own diagnostic). + host::test_host::set_files([ + tsconfig(), + ("/p/globals.d.ts".to_string(), GLOBALS.to_string()), + ( + "/p/a.ts".to_string(), + "const test = 5;\nconsole.log(test);\n".to_string(), + ), + ]); + let (status, _reply) = run(&request()); + assert_eq!(status, 2); + } } // wasm only: `release_checker` is not in a native build, where each checker @@ -300,6 +882,12 @@ mod wasm_tests { ]); // No flags: diagnostics as text, on a case-sensitive host. assert_eq!(run("/p\0\0-p\0."), (0, String::new())); + // The multi-run flow leaves no current program between runs (the + // compile's `ProgramScope` drops with the compile frame): enter the + // run's version for the releases, as an embedder that frees a + // checker of the last run does (`multirun::enter_last_version`). + let version = ts_goport::program::multirun::enter_last_version() + .expect("the run installed a version"); let held = LIVE.load(Ordering::Relaxed); ts_goport::program::release_checker(0); let released = LIVE.load(Ordering::Relaxed); @@ -309,5 +897,6 @@ mod wasm_tests { ts_goport::program::release_checker(0); ts_goport::program::release_checker(usize::MAX); assert_eq!(LIVE.load(Ordering::Relaxed), released); + drop(version); } } diff --git a/npm/wasm/core.js b/npm/wasm/core.js index 7b3150d9d..e515c8a73 100644 --- a/npm/wasm/core.js +++ b/npm/wasm/core.js @@ -97,8 +97,15 @@ function fsCall(fs, op, request) { return undefined; } -/** The state and imports of one run (`runTsc`, `runTscAsync`). */ -function prepareRun(module, options) { +/** + * The state and imports of one run (`runTsc`, `runTscAsync`). Exported for + * a held instance: make the instance yourself (`new WebAssembly.Instance( + * module, run.imports)`), call the `ts_run` that `run.start(instance)` + * returns once per request, and read each reply with `run.result`. One + * instance serves many requests (the multi-run flow), so a probe that + * holds it sees the warm runs. + */ +export function prepareRun(module, options) { const { args = [], cwd = "/",