diff --git a/freevideo_engine/agent/access.py b/freevideo_engine/agent/access.py index e7ed6807..2c5f9e02 100644 --- a/freevideo_engine/agent/access.py +++ b/freevideo_engine/agent/access.py @@ -20,9 +20,16 @@ def peer_address(request): def lan_enabled(): from ..comfy_bridge import installation_root - from ..launcher_settings import Store + from ..desktop_runtime import launcher_root + from ..launcher_settings import Store, read + # The launcher's switch saves into the launcher folder, the page's switch into the installation. Access needs + # every copy of one of the two to say yes, and the most recently saved copy anywhere to say yes too, so a switch + # turned off in one place stays off even when another place still holds an older yes. try: - return Store(installation_root()).currently_allows('lan_access') + stores = [Store(root()) for root in (installation_root, launcher_root)] + copies = [read(path) for store in stores for path in store.paths] + newest = max(copies, key=lambda saved: saved.get('saved_at_ns', 0)) + return newest.get('lan_access') is True and any(store.currently_allows('lan_access') for store in stores) except (OSError, ValueError): return False diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index ddcf4eda..5a9dbed4 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -158,12 +158,35 @@ def disk_review(plan, engine, comfy, *, separate, new_comfy): return extra, list(disks.values()), errors(disks.values()) +def _is_comfy_root(path): + return (path / 'main.py').is_file() and (path / 'folder_paths.py').is_file() + + +def nearby_comfy(selected): + """Name of a ComfyUI folder just above or one level below the chosen one, else ''. + + Only a name is returned: the launcher hides absolute paths in its messages. + """ + try: + for parent in (selected, *list(selected.parents)[:3]): + if _is_comfy_root(parent): + return parent.name + for child in sorted(p for p in selected.iterdir() if p.is_dir())[:200]: + if _is_comfy_root(child) or _is_comfy_root(child / 'ComfyUI'): + return child.name + except OSError: + pass + return '' + + def layout(path): try: selected = resolved(Path(path).expanduser()) root = selected / 'ComfyUI' if (selected / 'ComfyUI' / 'main.py').is_file() else selected - if not (root / 'main.py').is_file() or not (root / 'folder_paths.py').is_file(): - raise ValueError('Select the ComfyUI folder, or the portable folder containing ComfyUI.') + if not _is_comfy_root(root): + found = nearby_comfy(selected) + raise ValueError('Select the ComfyUI folder, or the portable folder containing ComfyUI.' + + (f' Found one at "{found}"; select that folder.' if found else '')) if not (root / 'comfy_api' / 'latest').is_dir(): raise ValueError('This ComfyUI is too old for native VIDEO / V3 nodes. Update ComfyUI first; its files have not been changed.') candidates = [] @@ -418,14 +441,26 @@ def lan_access_links(engine, port, *, password_set=None): return [address(host, port) + suffix for host in hosts] +def reserved_port(error): + """Whether no program may listen on this port at all. Windows with WSL or Hyper-V reserves + whole blocks of ports (often 100 or more in a row), and binding one fails with WSAEACCES.""" + return error.errno == errno.EACCES or getattr(error, 'winerror', None) == 10013 + + def next_free_address(url, count=12): - """The next local address after this one that nothing listens or answers on, as setup.sh chooses.""" + """The next local address after this one that nothing listens or answers on, as setup.sh chooses. + Reserved ports are skipped without counting, so a reserved block does not end the search.""" parsed = urlsplit(url) - for port in range((parsed.port or 80) + 1, min((parsed.port or 80) + 1 + count, 65536)): + port = parsed.port or 80 + while count > 0 and port < 65535: + port += 1 try: bind_check(parsed.hostname, port) - except OSError: + except OSError as error: + if not reserved_port(error): + count -= 1 continue + count -= 1 candidate = address(parsed.hostname, port) if server_info(loopback_url(candidate))['status'] == 'offline': return candidate @@ -1122,7 +1157,7 @@ def _move_address(self, selected): """Take the next free port for this ComfyUI and keep it in the selection the launcher saves.""" url = next_free_address(selected['url']) if url is None: - raise ValueError('Other programs use this port and the next ones. Set a different local ComfyUI address.') + raise ValueError('This port and the next ones are in use or reserved by the system. Set a different local ComfyUI address.') if self.port_move is None: self.port_move = urlsplit(selected['url']).port or 80 selected = self.selection = dict(selected, url=url) @@ -1197,7 +1232,7 @@ def _start_access_proxy(self): port = (urlsplit(selected['url']).port or 80) + 1 candidate = next_free_address(address('0.0.0.0', port - 1)) if port <= 65535 else None if candidate is None: - raise ValueError('The ports for LAN access are in use. Choose another ComfyUI port.') + raise ValueError('The ports for LAN access are in use or reserved by the system. Choose another ComfyUI port.') port = urlsplit(candidate).port self.selection = selected = dict(selected, lan_port=port) self.state = dict(self.state, selection=dict(selected)) diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index f2dc172f..d2a09cad 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -394,6 +394,22 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep return dict(title='这块硬盘没有连接' if zh else 'The disk is not connected', kind='disk-missing', detail='', action=('请接上“%s”后点击“重新检查”' % disk_name + ('，或者改装到其他硬盘。' if other_disk else '。') if zh else 'Connect "%s" and click Check again' % disk_name + (', or install on another disk.' if other_disk else '.'))) + folder = 'Select the ComfyUI folder, or the portable folder containing ComfyUI.' + # A saved installation that vanished raises the same sentence after its own message; the + # user chose no folder then, so it keeps its own path. + if folder in summary and not re.search(r'Locate or repair this installation|重新定位或修复这份安装', summary): + # The retained error carries a traceback whose source lines quote the message + # (and its Found part) too; only the exception line itself names the folder. + named = list(re.finditer(re.escape(folder) + r'(?: Found one at "([^"\r\n]+)"; select that folder\.)?[ \t\r]*$', + summary, re.M)) + found = named[-1].group(1) if named else None + if found: + action = ('请改选“%s”文件夹。' if zh else 'Select the "%s" folder instead.') % found + else: + action = ('请选择包含 main.py 的 ComfyUI 文件夹，或包含它的便携包文件夹。' if zh else + 'Select the ComfyUI folder (it contains main.py), or the portable folder that contains it.') + return dict(title='这不是 ComfyUI 文件夹' if zh else "This isn't a ComfyUI folder", detail='', + action=action, kind='comfy-folder') for pattern, kind, title, action in PLACE_RULES: if re.search(pattern, summary): return dict(title=title[zh], detail='', action=action[zh], kind=kind) @@ -631,6 +647,25 @@ def _report(run): return report if isinstance(report, dict) else {} +def _checkpoint_change(report): + """The file-identity fields a streamed checkpoint check found changed, or ''. + + Only the fixed names kept by the diagnostics: no paths or timestamp values. + """ + from .diagnostic_resources import checkpoint_change + encoding, attempts = report.get('encoding_failure'), report.get('resource_attempts') + found = [report, encoding.get('failure') if isinstance(encoding, dict) else None] + found += [row.get('failure') for row in attempts if isinstance(row, dict)] if isinstance(attempts, list) else [] + for holder in found: + rows = holder.get('exception') if isinstance(holder, dict) else None + for row in rows if isinstance(rows, list) else [rows]: + if isinstance(row, dict) and row.get('category') == 'checkpoint_changed': + change = checkpoint_change(row.get('checkpoint_change')) + if change.get('source') and change['fields']: + return 'Checkpoint changed: %s (%s)' % (change['source'], ', '.join(change['fields'])) + return '' + + # Stands in for the repair sentence until the report text is redacted: the # redactor reads "Install / repair" as a path. REPAIR_MARK = 'FREEVIDEO_REPAIR_SENTENCE' @@ -669,6 +704,9 @@ def generation_failure(run, exit_code=None, *, repair=None): explained = _runtime_explanation(report, REPAIR_MARK) if explained: lines.append(explained) + changed = _checkpoint_change(report) + if changed: + lines.append(changed) error = report.get('error_message') or report.get('error') if isinstance(error, str): lines.append(error[:8192]) diff --git a/freevideo_engine/image_gen/failure.py b/freevideo_engine/image_gen/failure.py index a29582f7..c7cde206 100644 --- a/freevideo_engine/image_gen/failure.py +++ b/freevideo_engine/image_gen/failure.py @@ -13,14 +13,15 @@ from ..failure_details import Redactor -ERRORS = frozenset(('no_memory', 'busy_video', 'busy', 'pack_missing', 'model_unreadable', +ERRORS = frozenset(('no_memory', 'small_memory', 'busy_video', 'busy', 'pack_missing', 'model_unreadable', 'device_unavailable', 'worker_exit', 'timeout', 'cancelled', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'generation_failed')) -NO_REPORT = frozenset(('busy', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'cancelled')) +NO_REPORT = frozenset(('small_memory', 'busy', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'cancelled')) STAGES = ('start', 'load', 'text', 'encode', 'denoise', 'decode') KINDS = ('generate', 'edit', 'cast', 'compose', 'download', 'install') MESSAGE_ZH = { 'no_memory': '显存不够，最小尺寸也没能生成。关掉其他占用显卡的程序后再试。', + 'small_memory': '这台 Mac 的内存不足 16 GB，无法生成图片。', 'busy_video': '视频正在生成，完成后再试。', 'busy': '已有一张图在生成，请等它完成。', 'pack_missing': '请先在启动器里下载“图像生成模型”。', @@ -47,7 +48,10 @@ r'safetensors?|(?:deserializ|invalid|incomplete|too large).*header|' r'(?:checkpoint|weights?).*(?:cannot|could not|unable|failed|corrupt|unreadable|mismatch)|' r'pytorchstreamreader|unexpected (?:end of file|eof)', re.I) +_MPS_MEMORY = re.compile(r'MPS backend out of memory', re.I) _LIMIT = 16 * 1024 +# On a Mac the GPU's memory is the machine's: the same failure, in its words. +UNIFIED_MESSAGE_ZH = {'no_memory': '内存不够，最小尺寸也没能生成。关掉其他程序后再试。'} class FailureError(ValueError): @@ -70,7 +74,7 @@ def classify_exception(error, *, stage='start', cancelled=False): error = error.__cause__ or (None if error.__suppress_context__ else error.__context__) for item in chain: names = {cls.__name__ for cls in type(item).__mro__} - if isinstance(item, MemoryError) or 'OutOfMemoryError' in names: + if isinstance(item, MemoryError) or 'OutOfMemoryError' in names or _MPS_MEMORY.search(str(item)): return 'no_memory' for item in chain: cuda_init = any(frame.f_globals.get('__name__') == 'torch.cuda' diff --git a/freevideo_engine/image_gen/image_policy.py b/freevideo_engine/image_gen/image_policy.py index 32914af0..75e4ddbd 100644 --- a/freevideo_engine/image_gen/image_policy.py +++ b/freevideo_engine/image_gen/image_policy.py @@ -82,6 +82,19 @@ def size(layer): MARGIN_FRACTION = 0.06 MIN_HOST_WORKING = 4 * GiB # beyond the weights: images, tokenizer, the process itself +# Apple GPUs draw on the host's memory, where the weights are mapped file pages the +# system can drop. Planning against half of it kept a 24 GB M5 with a browser open +# out of heavy swapping; planning against what was free at the time let the second +# request keep 16 encoder layers resident and swap 4 GB. A Mac below 16 GB cannot +# plan even with every block streamed (4 GiB), and closing programs does not change it. +UNIFIED_SHARE = 0.5 +UNIFIED_MIN_TOTAL = 15 * GiB # 16 GB Macs report exactly 16 GiB + + +def unified_budget(total_bytes): + """What a request on an Apple GPU plans against, from the machine's total memory.""" + return int(total_bytes * UNIFIED_SHARE) + @dataclass(frozen=True) class ImagePlan: diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.py index b73d0214..e4e11cac 100644 --- a/freevideo_engine/image_gen/image_runner.py +++ b/freevideo_engine/image_gen/image_runner.py @@ -97,11 +97,12 @@ def leave(self, stage): runner.dit_stack.set_resident(_kept(runner.last_plan)) self._trim() - @staticmethod - def _trim(): + def _trim(self): # Stage peaks are planned on allocated bytes; give the cached blocks back so the # next stage's workspace is not fragmented by the previous one's. - if torch.cuda.is_available(): + if self.runner.device.type == 'mps': + torch.mps.empty_cache() + elif torch.cuda.is_available(): torch.cuda.empty_cache() @@ -117,6 +118,7 @@ class ImageRunner: """ full_card = False + mapped = False def __init__(self, pack_dir, *, device='cuda', gpu_budget_bytes=None, log=None, progress=None): from .qwen_pipeline import QwenImageGenerator @@ -136,9 +138,12 @@ def path(directory, name): loading = _LoadProgress(progress or (lambda stage, fraction: None)) started = time.monotonic() # A card that holds every weight plus a request's working memory keeps no host copy. - self.full_card = self._fits_whole_model() + # An Apple GPU shares the host's memory: its layers stay views of the mapped files (mapped.py). + self.mapped = self.device.type == 'mps' + self.full_card = not self.mapped and self._fits_whole_model() self.transformer = load_dit(self.pack_dir / _DIT_FILE, device=self.device if self.full_card else 'cpu', - progress=lambda done, total: loading.report('dit', done, total)) + progress=lambda done, total: loading.report('dit', done, total), + **({'mapped': True} if self.mapped else {})) self.load_seconds['dit'] = time.monotonic() - started loading.complete('dit') started = time.monotonic() @@ -150,7 +155,8 @@ def path(directory, name): config_dir = self.pack_dir / 'text_encoder' processor_dir = self.pack_dir / 'processor' text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu', - progress=lambda done, total: loading.report('text', done, total)) + progress=lambda done, total: loading.report('text', done, total), + **({'mapped': True} if self.mapped else {})) from transformers import Qwen3VLProcessor processor = Qwen3VLProcessor.from_pretrained(str(processor_dir), local_files_only=True) self.load_seconds['text'] = time.monotonic() - started @@ -183,7 +189,15 @@ def pin_progress(index): return lambda done, total: loading.report( 'pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total) - if self.full_card: + if self.mapped: + from .mapped import MappedStack + self.dit_stack = MappedStack(stacks[0], name='dit', device=self.device) + self.te_stack = MappedStack(stacks[1], name='text_encoder', device=self.device) + from . import mlx_attention, qwen_dit + if mlx_attention.available() and not getattr(qwen_dit.attention, 'mlx', False): + qwen_dit.attention = mlx_attention.routed(qwen_dit.attention) + qwen_dit.attention.mlx = True + elif self.full_card: # Move the encoder layers over one at a time so the host copy is freed as it goes. for layer in stacks[1]: layer.to(self.device) @@ -246,6 +260,9 @@ def _to_streamed(self): def budget(self): if self.gpu_budget_bytes is not None: return int(self.gpu_budget_bytes) + if self.device.type == 'mps': + from ..system import system_memory + return image_policy.unified_budget(system_memory()['total_bytes']) free, _ = torch.cuda.mem_get_info(self.device) return int(free + torch.cuda.memory_reserved(self.device)) @@ -267,7 +284,7 @@ def plan(self, width, height, reference_images=None): memory = system_memory() free = memory.get('physical_available_bytes') or memory['available_bytes'] host_available = free + self.dit_stack.host_bytes + self.te_stack.host_bytes - if self.full_card: + if self.full_card or self.mapped: host_available = None except Exception: host_available = None diff --git a/freevideo_engine/image_gen/int8_linear.py b/freevideo_engine/image_gen/int8_linear.py new file mode 100644 index 00000000..92d92bad --- /dev/null +++ b/freevideo_engine/image_gen/int8_linear.py @@ -0,0 +1,114 @@ +"""The int8 projections of the image model, per platform. + +``ops()`` is the engine's CUDA ``int8_ops`` everywhere but macOS, where Triton +does not exist. There the same prepared ConvRot rows run on the Apple GPU: on +an M5 or newer through the TensorOps int8 kernels of ``backends.mps_int8`` +(one int8 scale per activation row, like CUDA); on older Apple GPUs the +activation is rotated and multiplied by the int8 rows widened to BF16, with +the row scales applied to the output. +""" +import sys + +import torch +from torch.nn import functional as F + +GROUP = 256 +_H16 = {} + + +def ops(): + """The module providing ``Int8Linear`` and ``install_cached_linears`` on this platform.""" + if sys.platform == 'darwin': + return sys.modules[__name__] + from .. import int8_ops + return int8_ops + + +def hadamard16(device): + """H4 (x) H4 / 4 with H4 = [[1,1,1,-1],[1,1,-1,1],[1,-1,1,1],[-1,1,1,1]] (FP32), as in int8_ops.""" + key = str(device) + if key not in _H16: + h4 = torch.tensor([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=torch.float32) + _H16[key] = (torch.kron(h4, h4) / 4).to(device).contiguous() + return _H16[key] + + +def rotate(rows): + """[M, K] rows times the group-256 ConvRot rotation H256 = H16 (x) H16, in FP32.""" + h16 = hadamard16(rows.device) + groups = rows.float().reshape(rows.shape[0], -1, 16, 16) + return (h16 @ groups @ h16).reshape(rows.shape) + + +def _tensor_ops(linear): + from ..backends import mps_int8 + return (mps_int8.available() and linear.in_features <= mps_int8.CONVROT_DEPTH + and linear.out_features % mps_int8.TILE == 0) + + +class Int8Linear(torch.nn.Module): + """A prepared ConvRot int8 Linear on an Apple GPU; the buffers are those of int8_ops.Int8Linear. + + The most recent prepared input is shared by every Int8Linear, so Q/K/V + quantize (or rotate) the same activation once; the entry keeps its source alive. + """ + _last = {} + + @property + def bias(self): + return self.original.bias + + def prepared(self, rows, quantize): + key = (rows.data_ptr(), rows._version, tuple(rows.shape), tuple(rows.stride()), str(rows.device), quantize) + last = Int8Linear._last + if last.get('key') != key: + last.clear() + if quantize: + from ..backends import mps_int8 + value = mps_int8.quantize_rows(rows, convrot=True) + else: + value = rotate(rows).to(rows.dtype) + last.update(key=key, source=rows, value=value) + return last['value'] + + def forward(self, value): + rows = value.reshape(-1, value.shape[-1]) + if rows.stride(-1) != 1: + rows = rows.contiguous() + scales = self.weight_scale.reshape(-1) + quantize = value.dtype in (torch.bfloat16, torch.float16) and _tensor_ops(self) + if quantize: + from ..backends import mps_int8 + output = mps_int8.matmul(self.prepared(rows, True), self.weight_int8, scales, dtype=value.dtype) + else: + # int8 values are exact in BF16; the row scales go on the output, in FP32. + output = F.linear(self.prepared(rows, False), self.weight_int8.to(value.dtype)) + output = output.float().mul_(scales).to(value.dtype) + if self.bias is not None: + output = output + self.bias + return output.reshape(*value.shape[:-1], self.out_features) + + +def install_cached_linears(model, linears, rotation): + """Replace prepared int8 Linears with meta int8 placeholders for layer loading (as int8_ops).""" + if rotation is None or rotation.get('kind') != 'convrot' or rotation.get('group') != GROUP: + raise ValueError('Int8 projections need ConvRot group-256 weights') + for name, spec in linears.items(): + if spec.get('storage') != 'int8' or spec.get('convrot_group') != GROUP: + raise ValueError('Int8 cache entry is not a ConvRot int8 matrix: ' + name) + original = model.get_submodule(name) + if not isinstance(original, torch.nn.Linear): + raise ValueError('Int8 cache target is not a Linear: ' + name) + if list(original.weight.shape) != spec['weight_shape'] or original.in_features % GROUP: + raise ValueError('Int8 cache shape differs from the official model: ' + name) + replacement = Int8Linear.__new__(Int8Linear) + torch.nn.Module.__init__(replacement) + replacement.register_buffer('weight_int8', torch.empty(spec['weight_shape'], dtype=torch.int8, device='meta')) + replacement.register_buffer('weight_scale', torch.empty(spec['scale_shape'], dtype=torch.float32, device='meta')) + replacement.original = torch.nn.Module() + replacement.original.register_parameter('bias', original.bias) + replacement.in_features = original.in_features + replacement.out_features = original.out_features + replacement.input_dtype = getattr(torch, spec.get('input_dtype', 'bfloat16')) + prefix, _, child = name.rpartition('.') + setattr(model.get_submodule(prefix) if prefix else model, child, replacement) diff --git a/freevideo_engine/image_gen/mapped.py b/freevideo_engine/image_gen/mapped.py new file mode 100644 index 00000000..1fd84545 --- /dev/null +++ b/freevideo_engine/image_gen/mapped.py @@ -0,0 +1,130 @@ +"""File-backed image weights for Apple GPUs. + +On a Mac the card's memory is the host's. Weights copied into ordinary memory +and onto the GPU heap stay wired while the GPU uses them, and a 24 GB machine +with a browser open swapped the other programs out at gigabytes per second to +hold the 15 GB model. Here the stacked layers' weights are read-only views of +the checkpoint's own mapped pages, which the system can drop and read again +instead of swapping, and ``MappedStack`` lends each layer to the GPU for its +pass. Only ``set_resident`` layers keep their GPU copy between passes. +""" +import json +import mmap +import struct +import warnings + +import torch + +_DTYPES = {'BF16': torch.bfloat16, 'F16': torch.float16, 'F32': torch.float32, 'I8': torch.int8, + 'U8': torch.uint8, 'I32': torch.int32, 'I64': torch.int64} + + +class MappedCheckpoint: + """A safetensors reader whose ``get_tensor`` returns views of the mapped file. + + Everything else (keys, slices, metadata) is the wrapped reader's, so the + loaders validate exactly what they validate without it. + """ + + def __init__(self, reader, path): + self.reader = reader + with open(path, 'rb') as stream: + size, = struct.unpack('= self.resident: + self._unbind(index) + + def set_resident(self, count): + """Keep the first ``count`` layers' GPU copies between passes (filled on their next use).""" + count = max(0, min(int(count), len(self.layers))) + for index in sorted(self.bound): + if index >= count: + self._unbind(index) + self.resident = count + return count + + def stats(self): + return dict(name=self.name, layers=len(self.layers), layer_bytes=self.layer_bytes, pinned_layers=0, + pinned_bytes=0, resident_layers=self.resident, cached_bytes=len(self.bound) * self.layer_bytes, + copied_bytes=self.copied_bytes, mapped=True) + + def close(self): + for hook in self.hooks: + hook.remove() + for index in list(self.bound): + self._unbind(index) diff --git a/freevideo_engine/image_gen/mlx_attention.py b/freevideo_engine/image_gen/mlx_attention.py new file mode 100644 index 00000000..ad4573d1 --- /dev/null +++ b/freevideo_engine/image_gen/mlx_attention.py @@ -0,0 +1,55 @@ +"""Image attention through MLX's fused Metal kernel on Apple GPUs. + +At the 1024 x 1024 shape (4125 keys, 32 heads of 128) Torch's MPS attention +ran at about 3.5 TFLOPS on an M5; this kernel is the one the Mac video engine +already uses. Q/K/V and the output share their Metal buffers through DLPack. +""" +import math + +import torch + +from ..backends.mps_mlx_attention import to_mlx_many, to_torch + + +def available(): + try: + import mlx.core # noqa: F401 + except Exception: + return False + return True + + +def routed(fallback, *, min_queries=256): + """``attention`` for image-sized calls; short text segments, where the hand-over costs more, keep ``fallback``. + + A call MLX cannot run goes to ``fallback`` instead of failing the picture, and so does + every later call of the process. + """ + failed = [] + + def run(query, key, value, attn_mask=None): + if failed or query.shape[1] < min_queries: + return fallback(query, key, value, attn_mask=attn_mask) + try: + return attention(query, key, value, attn_mask=attn_mask) + except Exception as error: + failed.append(f'{type(error).__name__}: {error}'[:300]) + return fallback(query, key, value, attn_mask=attn_mask) + run.failed = failed + return run + + +def attention(query, key, value, attn_mask=None): + """Same contract as ``qwen_dit.attention``: (batch, sequence, heads, dim), boolean mask or None.""" + import mlx.core as mx + tensors = (query, key, value) if attn_mask is None else (query, key, value, attn_mask) + arrays = to_mlx_many(tensors) + q, k, v = (mx.swapaxes(array, 1, 2) for array in arrays[:3]) + mask = arrays[3] if attn_mask is not None else None + out = mx.fast.scaled_dot_product_attention(q, k, v, scale=1 / math.sqrt(query.shape[-1]), mask=mask) + out = mx.contiguous(mx.swapaxes(out, 1, 2)) + result = to_torch(out, query.device) + del arrays, q, k, v, mask, out + # One unified pool: do not let MLX keep the freed buffers from Torch. + mx.clear_cache() + return result diff --git a/freevideo_engine/image_gen/qwen_pipeline.py b/freevideo_engine/image_gen/qwen_pipeline.py index 15ebbc98..b0eeb89e 100644 --- a/freevideo_engine/image_gen/qwen_pipeline.py +++ b/freevideo_engine/image_gen/qwen_pipeline.py @@ -400,6 +400,7 @@ def close(self) -> None: self.text_encoder = self.transformer = self.vae = None self._closed = True # Int8Linear keeps the last activation alive for Q/K/V sharing. - module = sys.modules.get("freevideo_engine.int8_ops") - if module is not None: - module.Int8Linear._last.clear() + for name in ("freevideo_engine.int8_ops", "freevideo_engine.image_gen.int8_linear"): + module = sys.modules.get(name) + if module is not None: + module.Int8Linear._last.clear() diff --git a/freevideo_engine/image_gen/qwen_te_weights.py b/freevideo_engine/image_gen/qwen_te_weights.py index 86d62994..7765a9fd 100644 --- a/freevideo_engine/image_gen/qwen_te_weights.py +++ b/freevideo_engine/image_gen/qwen_te_weights.py @@ -117,7 +117,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None): _dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads), requires_grad=False ) else: - from ..int8_ops import Int8Linear + from .int8_linear import ops + Int8Linear = ops().Int8Linear weight = st.get_tensor(name + ".weight") if weight.dtype != torch.int8 or weight.shape != shape or shape[1] % 256: @@ -158,7 +159,7 @@ def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None): def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtype=torch.bfloat16, - chunk_rows=8192, progress=None): + chunk_rows=8192, progress=None, mapped=False): """Return a Qwen3VLForConditionalGeneration with int8 language projections. ``config`` accepts a transformers Qwen3VLConfig or its dictionary. Otherwise @@ -196,6 +197,9 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtyp with safe_open(str(path), framework="pt", device="cpu") as st: reads = (TensorProgress(st, progress, (key for key in st.keys() if not key.startswith(("model.visual.", "lm_head.")))) if progress is not None else None) + if mapped: # CPU only: tensors stored as they are on disk stay views of the mapped file + from .mapped import MappedCheckpoint + st = MappedCheckpoint(st, path) _install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows, reads=reads) model.eval().requires_grad_(False).to(device=device) model_ref = weakref.ref(model) diff --git a/freevideo_engine/image_gen/qwen_weights.py b/freevideo_engine/image_gen/qwen_weights.py index 662d12c9..fe64360a 100644 --- a/freevideo_engine/image_gen/qwen_weights.py +++ b/freevideo_engine/image_gen/qwen_weights.py @@ -92,15 +92,19 @@ def _quant_targets(model, name): return targets, modules -def load_dit(path, device="cpu", *, config=None, progress=None): +def load_dit(path, device="cpu", *, config=None, progress=None, mapped=False): """Load a Comfy Turbo int8_convrot DiT; CPU supports storage, CUDA execution. The optional config override is for small models/tests. Production files without JSON config metadata use the official 32-block Qwen-Image 2.1 config. An unknown marker, tensor layout, missing key, or extra key is an error. ``progress(done_bytes, total_bytes)`` reports completed source tensor batches. + ``mapped`` (CPU only) leaves tensors stored as they are on disk as views of the mapped file. """ with _open_safetensors(path, framework="pt", device="cpu") as checkpoint: + if mapped: + from .mapped import MappedCheckpoint + checkpoint = MappedCheckpoint(checkpoint, path) reads = TensorProgress(checkpoint, progress) if progress is not None else None with torch.device("meta"): model = QwenImageDiT(**_config(checkpoint.metadata() or {}, QwenImageDiT, config)) @@ -159,7 +163,8 @@ def load_dit(path, device="cpu", *, config=None, progress=None): for key in ordinary: _check_tensor(checkpoint, keys[key], expected[key].shape, {"BF16", "F32"}) # Reuse the engine's wrapper construction and exact runtime member names. - from ..int8_ops import install_cached_linears + from .int8_linear import ops + install_cached_linears = ops().install_cached_linears specs = {} for targets in quantized.values(): for target in targets: diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index c351c588..6a0dcb6f 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -17,7 +17,8 @@ from . import image_pack from ..download_settings import observed_speed -from .failure import Context, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, classify_exception +from .failure import (Context, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, UNIFIED_MESSAGE_ZH, + classify_exception) BODY_LIMIT = 64 * 1024 CANCEL_GRACE = 5. @@ -156,12 +157,28 @@ def info(self): segment=dict(ready=image_pack.ready(machine['model_root'], group='segment'), max_points=16, device='cpu', install=self.segment_installer.state() if self.segment_installer else None), - bytes_per_second=None if ready else observed_speed(engine_root)) + bytes_per_second=None if ready else observed_speed(engine_root), + **({'unified_memory': True} if machine.get('device_backend') == 'mps' else {})) + + @staticmethod + def unified_total(machine): + """Total memory of a Mac whose Apple GPU shares it; None for a separate card.""" + if machine.get('device_backend') != 'mps': + return None + from ..system import system_memory + try: + return int(system_memory()['total_bytes']) + except (OSError, KeyError, TypeError, ValueError): + return None def size_tiers(self, machine): """The sizes this card offers: the standard one always, the high one only with enough card memory.""" total = None - if machine.get('device_backend', 'cuda') == 'cuda': + unified = self.unified_total(machine) + if unified is not None: + from .image_policy import unified_budget + total = unified_budget(unified) + elif machine.get('device_backend', 'cuda') == 'cuda': from ..background_room import Gpu from ..gpu_budget import open_bounded @@ -495,6 +512,10 @@ def prepare(self, action, value, context, *, waiting=False): try: if not image_pack.ready(machine['model_root']): raise ValueError('pack_missing') + from .image_policy import UNIFIED_MIN_TOTAL + unified = self.unified_total(machine) + if unified is not None and unified < UNIFIED_MIN_TOTAL: + raise ValueError('small_memory') requested = {name: value.get(name) for name in ('width', 'height')} if isinstance(value, dict) else {} if action == 'compose': from . import compose @@ -598,6 +619,10 @@ def wait_queued(self): def memory_budget(self, machine): """Bound the driver query, without importing Torch or starting a worker.""" + unified = self.unified_total(machine) + if unified is not None: + from .image_policy import unified_budget + return unified_budget(unified) if machine.get('device_backend', 'cuda') != 'cuda': return None from ..background_room import Gpu, family @@ -679,6 +704,8 @@ def failed(self, code, context, *, row=None, error=None, video_busy=None): result = dict(phase='failed', error=code, report_id=None, stage=stage) if code in MESSAGE_ZH: result['message_zh'] = MESSAGE_ZH[code] + if context.machine.get('device_backend') == 'mps' and code in UNIFIED_MESSAGE_ZH: + result['message_zh'] = UNIFIED_MESSAGE_ZH[code] if code not in NO_REPORT: if video_busy is None: try: diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 200b7d12..5b26d445 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -1707,12 +1707,12 @@ ApplicationWindow { FText { objectName: "updateActionNotice"; visible: !!s.action_notice; text: s.action_notice || ""; color: theme.muted; Layout.fillWidth: true } FText { text: s.update.candidate ? t("Update available", "有可用更新") : s.update.engine ? t("Engine update", "引擎更新") : t("Updates", "更新"); font.pixelSize: theme.section + 2; font.weight: Font.DemiBold } FText { objectName: "updateDialogText"; text: s.update.phase === "waiting" ? t("A video is still generating. FreeVideo restarts and updates as soon as it finishes.", "还有视频正在生成，完成后会自动重启并更新。") : s.update.status === "ready" && s.update.candidate ? t("Downloaded. Restart to finish; models and settings are kept.", "下载完成，重启即可完成更新，模型和设置都会保留。") : s.update.candidate && s.update.manual ? t("FreeVideo ", "FreeVideo ") + releaseVersion(s.update.candidate) + t(" is available (current ", " 已发布（当前 ") + releaseVersion(currentRelease) + t("). Download it and replace this version; models and settings are kept.", "）。下载新版后替换当前版本，模型和设置会保留。") : s.update.candidate ? t("FreeVideo ", "FreeVideo ") + releaseVersion(s.update.candidate) + t(" is available (current ", " 已发布（当前 ") + releaseVersion(currentRelease) + t("). Updating keeps your models and settings. FreeVideo restarts after the download; running videos finish first.", "）。更新会保留模型和设置，下载完成后自动重启；正在生成的视频会先完成。") : s.update.engine ? t("This launcher already includes engine ", "启动器已带有新版引擎 ") + releaseVersion(currentRelease) + t(" (installed: ", "（已安装 ") + engineVersion + (s.status === "open" ? t("). Updating takes about a minute, restarts ComfyUI once and keeps your models and settings.", "）。更新约需 1 分钟，会重启一次 ComfyUI，模型和设置都会保留。") : t("). Updating takes about a minute, then FreeVideo starts; models and settings are kept.", "）。更新约需 1 分钟，完成后自动启动，模型和设置都会保留。")) : s.update.status === "current" ? t("You're up to date.", "已是最新版本。") : s.update.status === "development" ? t("Running from source. Update with Git.", "当前从源码运行，请通过 Git 更新。") : s.update.status === "error" ? t("Could not check for updates.", "暂时无法检查更新。") : t("Checking the latest release…", "正在检查最新版本…"); color: theme.muted; Layout.fillWidth: true } - FMeter { Layout.fillWidth: true; visible: s.update.status === "downloading"; active: true; fraction: s.update.progress && s.update.progress.total ? s.update.progress.done/s.update.progress.total : -1 } - FReleaseNotes { objectName: "updateReleaseNotes"; visible: !!availableRelease; Layout.fillWidth: true; release: availableRelease; earlier: availableEarlier; zh: s.zh; heading: t("What's new", "更新内容") + " · " + releaseVersion(availableRelease) } - FButton { text: t("Version & release notes", "版本与更新说明"); flat: true; onClicked: releaseNotesOpen = true } FText { visible: !!s.update.error; text: s.update.error || ""; color: theme.danger; Layout.fillWidth: true; font.pixelSize: theme.micro } FButton { objectName: "updateManualDownloadButton"; text: t("Download from the release page ↗", "去发布页手动下载 ↗"); flat: true; Layout.fillWidth: true; visible: s.update.status === "error" && !!s.update.release_page; onClicked: backend.link(s.update.release_page) } FButton { objectName: "updateDialogReport"; text: t("Report a problem", "提交问题"); flat: true; Layout.fillWidth: true; visible: s.update.status === "error" && !!s.update.error; onClicked: backend.reportProblem(s.update.error, t("Update", "更新")) } + FMeter { Layout.fillWidth: true; visible: s.update.status === "downloading"; active: true; fraction: s.update.progress && s.update.progress.total ? s.update.progress.done/s.update.progress.total : -1 } + FReleaseNotes { objectName: "updateReleaseNotes"; visible: !!availableRelease; Layout.fillWidth: true; release: availableRelease; earlier: availableEarlier; zh: s.zh; heading: t("What's new", "更新内容") + " · " + releaseVersion(availableRelease) } + FButton { text: t("Version & release notes", "版本与更新说明"); flat: true; onClicked: releaseNotesOpen = true } } } RowLayout { diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index b47e7fb2..d5436a91 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -180,7 +180,8 @@ def source_name(value, zh=False): LAN_ERRORS = { 'Enter localhost or an IP address of this computer, for example http://127.0.0.1:8188.': '请填写 localhost 或本机的 IP 地址，例如 http://127.0.0.1:8188。', 'This address does not belong to this computer. Enter one of its IP addresses or 0.0.0.0.': '这个地址不属于本机。请填写本机的 IP 地址或 0.0.0.0。', - 'The ports for LAN access are in use. Choose another ComfyUI port.': '局域网访问要用的端口已被占用，请换一个 ComfyUI 端口。', + 'This port and the next ones are in use or reserved by the system. Set a different local ComfyUI address.': '这个端口和后面的端口都已被占用或被系统保留，请换一个本机 ComfyUI 地址。', + 'The ports for LAN access are in use or reserved by the system. Choose another ComfyUI port.': '局域网访问要用的端口已被占用或被系统保留，请换一个 ComfyUI 端口。', 'A generation is running. Change the address after it finishes.': '正在生成，完成后可修改地址。', 'The ComfyUI already running does not use this address. After its jobs finish, restart it with this address and connect again.': '已在运行的 ComfyUI 没有使用这个地址。请在任务完成后用这个地址重新启动 ComfyUI，再重新连接。', 'The ComfyUI already running is open to the network directly. After its jobs finish, restart it on 127.0.0.1, then turn on LAN access.': '已在运行的 ComfyUI 正在直接对局域网开放。请在任务完成后改用 127.0.0.1 重新启动它，再开启局域网访问。', diff --git a/freevideo_engine/mac_app_update.py b/freevideo_engine/mac_app_update.py index baf8f790..28d7b3b4 100644 --- a/freevideo_engine/mac_app_update.py +++ b/freevideo_engine/mac_app_update.py @@ -14,7 +14,8 @@ MARK = '.FreeVideo-update-' -BACKUP = '.FreeVideo-backup-' +BACKUP = '.FreeVideo-backup-' # the name older versions used, still cleaned up +BACKUPS = '.FreeVideo-backup.noindex' # Spotlight and Launchpad skip folders ending in .noindex MARGIN_BYTES = 256 * 2**20 HDIUTIL, DITTO, CODESIGN, XATTR = '/usr/bin/hdiutil', '/usr/bin/ditto', '/usr/bin/codesign', '/usr/bin/xattr' @@ -32,15 +33,32 @@ restore() { rm -rf "$app" if [ -d "$backup" ]; then /bin/mv "$backup" "$app"; fi + rmdir "$(dirname "$backup")" "$(dirname "$(dirname "$backup")")" 2>/dev/null rm -rf "$staged" "$opener" "$app" } [ "$inject" = "before_backup" ] && { note "injected failure"; rm -rf "$staged"; "$opener" "$app"; exit 1; } -if ! /bin/mv "$app" "$backup"; then note "could not set the old app aside"; rm -rf "$staged"; "$opener" "$app"; exit 1; fi +if ! { /bin/mkdir -p "$(dirname "$backup")" && /bin/mv "$app" "$backup"; }; then note "could not set the old app aside"; rm -rf "$staged"; "$opener" "$app"; exit 1; fi if [ "$inject" = "after_backup" ] || ! /bin/mv "$staged" "$app"; then note "could not put the new app in place"; restore; exit 1; fi if [ "$inject" = "after_swap" ] || ! "$opener" "$app"; then note "the new app did not open"; /bin/mv "$app" "$staged" && restore; exit 1 fi +# A new app that opens and then dies is as bad as one that does not open: it has to stay up for +# STABLE ticks. This is judged from outside, so it also holds for an app that cannot run any code. +tick="${FREEVIDEO_SWAP_TICK:-1}"; stable_need="${FREEVIDEO_SWAP_STABLE:-15}"; appear_max="${FREEVIDEO_SWAP_APPEAR:-60}" +# pgrep reads the pattern as a regular expression, so a folder named "Apps (old)" or "Apps+1" needs escaping. +pattern=$(printf '%s\n' "$app/Contents/MacOS/" | /usr/bin/sed 's/[][\.*^$+?(){}|]/\\&/g') +alive() { "${FREEVIDEO_PGREP:-/usr/bin/pgrep}" -f -- "$pattern" >/dev/null 2>&1; } +stable=0; seen=0; tries=0 +while [ "$stable" -lt "$stable_need" ]; do + if alive; then seen=1; stable=$((stable + 1)); elif [ "$seen" = 1 ]; then break; fi + tries=$((tries + 1)) + if [ "$tries" -gt $((appear_max + stable_need)) ]; then break; fi + sleep "$tick" +done +if [ "$stable" -lt "$stable_need" ]; then + note "the new app did not stay open"; /bin/mv "$app" "$staged" && restore; exit 1 +fi # The old app stays until the new launcher has run well; it removes it (settle). ''' @@ -85,12 +103,14 @@ def _size(path): def clear_stale(app): for entry in app.parent.glob(MARK + '*.app'): shutil.rmtree(entry, ignore_errors=True) + settle(app) def settle(app): """Remove the old versions kept beside a running app; called once it has run well.""" for entry in app.parent.glob(BACKUP + '*.app'): shutil.rmtree(entry, ignore_errors=True) + shutil.rmtree(app.parent / BACKUPS, ignore_errors=True) def prepare(dmg, app, directory, *, run=subprocess.run): @@ -149,7 +169,7 @@ def activate(staged, app, directory, *, popen=subprocess.Popen, inject=''): script = directory / 'swap-app.sh' script.write_text(SWAP_SCRIPT, encoding='utf-8') script.chmod(0o755) - backup = app.parent / (BACKUP + '%d.app' % int(time.time())) + backup = app.parent / BACKUPS / str(int(time.time())) / app.name failure = directory.parent / 'mac-update-failed.json' failure.unlink(missing_ok=True) with (directory.parent / 'mac-update.log').open('ab') as log: diff --git a/freevideo_engine/precompile.py b/freevideo_engine/precompile.py index 41da3131..99d6cc67 100644 --- a/freevideo_engine/precompile.py +++ b/freevideo_engine/precompile.py @@ -37,6 +37,10 @@ # default prompt encodes to 460 tokens. DEFAULT_SHAPE = dict(task='t2va', width=1344, height=768, frames=243, two_pass=True, base_steps=8, refine_steps=3, text_tokens=(460,)) +# Medium starts from a larger first pass, which is another canvas to compile. An +# installation with no request on record has not shown which level it uses. +MEDIUM_SHAPE = dict(DEFAULT_SHAPE, upscale_factor=1.5) +DEFAULT_SHAPES = [DEFAULT_SHAPE, MEDIUM_SHAPE] SHAPES = 2 LENGTHS = 2 TIMEOUT_SECONDS = 600 @@ -187,9 +191,9 @@ def run(machine, *, environment=None, timeout=TIMEOUT_SECONDS, background=False, hardware = detect() history = ResourceHistory(history_path()) identity = local_identity(hardware, Path(machine['cache'])) - shapes = recent_shapes(history) or [DEFAULT_SHAPE] + shapes = recent_shapes(history) or DEFAULT_SHAPES except Exception as error: - hardware, history, identity, shapes = None, None, None, [DEFAULT_SHAPE] + hardware, history, identity, shapes = None, None, None, DEFAULT_SHAPES receipt['history_error'] = repr(error) receipt.update(identity=identity, requested=shapes) try: diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index e2011fa9..069a80a2 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,31 +1,58 @@ { "schema": 1, - "product_version": "0.4.16", + "product_version": "0.4.17", "en": { - "summary": "Any part of a long video can be generated again as a new video, phones on the same network can follow chats and generations behind an optional password, and the Mac app now updates itself.", + "summary": "Image generation and editing come to Apple silicon Macs, a Mac update that fails to open is rolled back, and phones on the same network connect more reliably.", "changes": [ - "Long videos: the part list shows each part's length and what it follows, and marks failed parts. Any finished part can be generated again as a new video that keeps the parts before it, and the original video stays as it was. In the joined video, click a part's length to play two seconds either side of its seam. The list shows only each part's own events, and editing a part keeps the rest of its description unchanged.", - "Phone access: phones on the same network can open FreeVideo from the phone icon and QR code in the top bar, chat with AI Autopilot and follow generations. Phone access can be switched on or off and protected with a password that phones sign in with.", - "Mac: an update is downloaded in the background, and Restart & update replaces the app in place and opens the new version. Models and settings are kept, and if any step fails the previous version is restored.", - "Reference videos: a reference video can now be used without its own sound.", - "Editor: parts cut off when the engine stopped now show \"Interrupted\". A problem reported from a failure now includes the step that failed.", - "Linux: on graphics cards that can hold the whole video model, video generation no longer keeps a copy of the weights in system memory, which saves about 20 GB.", - "Reporting problems: memory readings in feedback record free physical memory and commit headroom separately." + "Mac: Macs with Apple silicon and at least 16 GB of memory can now generate and edit images.", + "Mac: in later updates, if the new version stops right after opening, the previous version is put back. The previous version is kept in a hidden folder so it no longer shows up in search or Launchpad. When an update fails, the reason and both buttons now appear right below the description.", + "Phone access: the phone page connects when phone access is switched on in the launcher, and also works on older phone browsers.", + "Queue: a single failed read of the queue no longer shows the connection lost card. The card appears after three failures in a row and goes away when the connection returns, and a running loop is kept through a short outage.", + "Launcher: when the chosen folder is not a ComfyUI folder, the launcher names a ComfyUI folder found nearby. Windows: the engine now starts when its usual port falls inside a range that Windows has reserved.", + "Installation: an installation with no generation on record now also compiles the standard size in the background.", + "Failures: when the model file changed during generation, the failure text names what changed." ] }, "zh": { - "summary": "长视频的任何一段都可以重新生成为一条新视频；同一网络下的手机可以查看对话和生成进度，可设密码；Mac 版可以自动更新。", + "summary": "Apple 芯片的 Mac 可以生成和编辑图片；Mac 更新后打不开时会还原旧版；同一网络下的手机连接更稳定。", "changes": [ - "长视频：分段列表显示每段的时长和接在哪一段之后，并标出失败的段。任何已完成的段都可以从这里重新生成一条新视频，前面的段保持不变，原视频也不受影响。整条视频生成后，点某段的时长可播放它接缝前后各 2 秒。列表只显示每段自己的动作，编辑某一段时其余描述保持不变。", - "手机访问：同一网络下的手机可以通过顶栏的手机图标和二维码打开 FreeVideo，和 AI 全自动对话、查看生成进度。手机访问可以开关，也可以设置密码，手机登录后才能使用。", - "Mac：新版在后台下载，点“重启并更新”后原位替换应用并打开新版。模型和设置都会保留，任何一步失败都会还原旧版。", - "参考视频：参考视频可以只用画面、不使用它自己的声音。", - "剪辑视图：引擎中断而没做完的片段显示“上次中断”。从失败提示提交的问题会写明是哪一步出错。", - "Linux：显存放得下整个视频模型的显卡，生成视频时不再在系统内存里留一份权重副本，约省 20 GB 内存。", - "提交问题：反馈里的内存读数分开记录物理可用内存和提交余量。" + "Mac：Apple 芯片、内存 16 GB 及以上的 Mac 可以生成和编辑图片。", + "Mac：以后的更新中，新版刚打开就停止时会自动换回旧版。旧版保存在隐藏文件夹里，不再出现在搜索和启动台中。更新失败时，失败原因和两个按钮紧跟在说明下方显示。", + "手机访问：在启动器中打开手机访问后，手机页面即可连接，较旧的手机浏览器也能使用。", + "队列：读取队列只失败一次时不再弹出“连接已断开”卡片；连续失败三次才出现，连接恢复后自动消失；短时间断线时，正在运行的循环会保留。", + "启动器：所选文件夹不是 ComfyUI 文件夹时，会指出附近找到的 ComfyUI 文件夹。Windows：引擎常用端口落在 Windows 保留的端口段里时，也能正常启动。", + "安装：没有任何生成记录的安装，后台也会预编译标准尺寸。", + "失败提示：生成过程中模型文件发生变化时，失败文字会写明变化的内容。" ] }, "history": [ + { + "product_version": "0.4.16", + "en": { + "summary": "Any part of a long video can be generated again as a new video, phones on the same network can follow chats and generations behind an optional password, and the Mac app now updates itself.", + "changes": [ + "Long videos: the part list shows each part's length and what it follows, and marks failed parts. Any finished part can be generated again as a new video that keeps the parts before it, and the original video stays as it was. In the joined video, click a part's length to play two seconds either side of its seam. The list shows only each part's own events, and editing a part keeps the rest of its description unchanged.", + "Phone access: phones on the same network can open FreeVideo from the phone icon and QR code in the top bar, chat with AI Autopilot and follow generations. Phone access can be switched on or off and protected with a password that phones sign in with.", + "Mac: an update is downloaded in the background, and Restart & update replaces the app in place and opens the new version. Models and settings are kept, and if any step fails the previous version is restored.", + "Reference videos: a reference video can now be used without its own sound.", + "Editor: parts cut off when the engine stopped now show \"Interrupted\". A problem reported from a failure now includes the step that failed.", + "Linux: on graphics cards that can hold the whole video model, video generation no longer keeps a copy of the weights in system memory, which saves about 20 GB.", + "Reporting problems: memory readings in feedback record free physical memory and commit headroom separately." + ] + }, + "zh": { + "summary": "长视频的任何一段都可以重新生成为一条新视频；同一网络下的手机可以查看对话和生成进度，可设密码；Mac 版可以自动更新。", + "changes": [ + "长视频：分段列表显示每段的时长和接在哪一段之后，并标出失败的段。任何已完成的段都可以从这里重新生成一条新视频，前面的段保持不变，原视频也不受影响。整条视频生成后，点某段的时长可播放它接缝前后各 2 秒。列表只显示每段自己的动作，编辑某一段时其余描述保持不变。", + "手机访问：同一网络下的手机可以通过顶栏的手机图标和二维码打开 FreeVideo，和 AI 全自动对话、查看生成进度。手机访问可以开关，也可以设置密码，手机登录后才能使用。", + "Mac：新版在后台下载，点“重启并更新”后原位替换应用并打开新版。模型和设置都会保留，任何一步失败都会还原旧版。", + "参考视频：参考视频可以只用画面、不使用它自己的声音。", + "剪辑视图：引擎中断而没做完的片段显示“上次中断”。从失败提示提交的问题会写明是哪一步出错。", + "Linux：显存放得下整个视频模型的显卡，生成视频时不再在系统内存里留一份权重副本，约省 20 GB 内存。", + "提交问题：反馈里的内存读数分开记录物理可用内存和提交余量。" + ] + } + }, { "product_version": "0.4.15", "en": { @@ -246,43 +273,6 @@ "启动器：浏览器没能自动打开时，提示点“复制地址”到浏览器里打开，中文界面也显示中文。画布边距标签更清楚、对齐。节点：宽度、高度、时长和种子有中文标签。" ] } - }, - { - "product_version": "0.4.8", - "en": { - "summary": "Long videos keep a steadier voice from part to part, and continuing a video keeps your reference pictures and characters. LoRA strength can be typed up to 100, large downloads move to a faster source on their own, and the second-pass button uses the earlier wording again, with smaller fixes to uploads, your creations, the waiting queue, diagnostic reports and the interface.", - "changes": [ - "Second pass: the result button, preview badge, library and delete messages use the 0.4.6 wording again (Run second pass, First-pass preview).", - "Long videos: the voice keeps the same loudness and fullness from part to part, and with dialogue a continued part no longer opens with a second or two of unclear speech. Continuing a video from the creative panel keeps your reference pictures and characters, so the person stays the same; up to 3 pictures are used.", - "Characters: two character names typed back to back after the @ key are both recognised, the list no longer reopens after a finished name, and Save in the character library says what is still missing. A character mentioned with @ in the editing view's generation bar now carries that character's description and pictures, as it does in the creation panel.", - "LoRA: you can type a strength beyond -4 to 4, up to 100, and a note marks values outside the usual range; the slider and default strength are unchanged. A trigger word that only appears inside a longer word (sks in asks) is now added.", - "Image generation: an out-of-memory failure says where memory ran out and at which size, and the progress line shows the smaller size while retrying; if the background process exits right after starting, it is retried once. Clicking Update while the image model is downloading pauses the download (what is already downloaded is kept and can be resumed) and starts the update at once. The option to install the image model as well checks free space on the drive that holds the models and is not offered when there is not enough room.", - "Downloads: large model downloads check the other sources while a transfer is slow and move to a clearly faster one, resuming the same file; a transfer that stops moving for 30 seconds changes source as well. The missing-model hint names each download and its size, and is checked again after a generation ends.", - "Media uploads: the upload buttons list only the file types FreeVideo can use, and unsupported files in a multiple selection are skipped. Files over 2 GiB are turned away in the page, a failed or interrupted upload no longer leaves a partial file, and upload errors appear in your language.", - "Your creations: an interrupted share or export no longer blocks deleting the creation, and its leftover working copy is removed at the next start. Adding the workflow to earlier videos no longer moves them to the top or changes their date.", - "Waiting queue: each task shows the start of its prompt, and Move to front highlights the moved task and confirms it.", - "Diagnostic report: the prompt-rewrite report no longer replaces the video's, the same file is saved and sent, and a large report is saved as a zip.", - "Wording: Agent chat, Settings, image generation and editor messages are shorter and say what to do next; the character library uses \"character\" throughout and its view labels are clearer.", - "Interface details: advanced node settings show Chinese names in the Chinese interface; generation cards in chat show a single progress figure; the Downloads card in Settings is tidier; Export in the editor keeps the primary colour and the Text and Subtitles tools use line icons; narrow windows lay out the preview, generate buttons, library title and notices better and keep Feedback in the header; long prompts no longer run under the @ button; library times follow the interface language; a result video that fails to load no longer offers actions that cannot work." - ] - }, - "zh": { - "summary": "长视频各段人声更一致，接着生成时保留参考图和角色。LoRA 强度可以手动输入到 100，大文件下载会自动换到更快的下载源，二次采样按钮恢复此前的说法；上传、作品库、等待队列、诊断报告和界面细节也有多处改进。", - "changes": [ - "二次采样：结果卡按钮、预览角标、作品库和删除提示恢复 0.4.6 的说法（继续二采、一采预览）。", - "长视频：人声的音量和厚度在各段之间保持一致；有对话时，接续的段开头不再先冒出一两秒听不清的话。在创作面板接着生成视频时，会保留参考图和角色，人物前后保持一致，最多使用 3 张图片。", - "角色：紧挨着写的两个 @名字都能识别，写完名字后列表不再重新弹出，角色库保存按钮灰着时会说明还缺什么；剪辑视图的生成条里写 @角色，也会和创作面板一样带上该角色的描述和图片。", - "LoRA：强度可以手动输入超出 -4 到 4 的数，最大到 100，超出常用范围时有提示；滑块和默认强度不变。触发词只出现在更长的单词里（如 asks 里的 sks）时，现在也会自动加上。", - "图像生成：显存不够时写明在哪一步、哪个尺寸，自动缩小重试时显示新尺寸；后台进程刚启动就退出时自动重试一次。图像模型在后台下载时点“更新”，下载先停下（已下的部分保留，之后可以继续），更新马上开始；“同时安装图像模型”按模型所在的盘检查空间，空间不够时不再显示为可选。", - "下载：大文件下载变慢时会测试其他下载源，明显更快时自动换源并接着下载；停滞 30 秒也会换源。缺模型提示写明是哪几项、各多大，生成结束后重新检查。", - "素材上传：上传按钮只列出能用的文件格式，一次选多个时跳过不支持的；超过 2 GiB 的文件在页面上直接拦下；上传失败或中断不再留下残留文件，出错提示用当前界面语言。", - "作品库：分享或导出中断后留下的临时文件不再挡住删除，下次启动时清理；给早期视频添加工作流后，它们不再跳到最前，日期保持原样。", - "等待队列：每条任务显示提示词开头；“排到最前”后被移动的任务会高亮并提示。", - "诊断报告：提示词增强不再顶替视频的报告，保存和提交是同一份，较大的报告存成 zip。", - "文字：Agent 对话、设置、图像生成和编辑器里的提示更简短，并写明下一步该点哪个按钮；角色库统一称“角色”，视角标签更清楚。", - "界面细节：中文界面的生成节点高级参数显示中文名；对话里的生成卡只显示一个进度；设置“下载”卡更整齐；剪辑视图导出按钮保持主色，文字、字幕工具改用线性图标；窄窗口下预览、生成按钮、作品库标题和提示的排布更合理，顶栏仍保留“反馈”入口；长提示词不再被“@”按钮压住；作品卡片时间跟随界面语言；结果视频加载失败时不再显示无法使用的操作。" - ] - } } ] } diff --git a/web/cast_generate.js b/web/cast_generate.js index 0d861dc1..396c201d 100644 --- a/web/cast_generate.js +++ b/web/cast_generate.js @@ -178,7 +178,7 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt }); if (state.error) { const key = `${state.error}:${state.report_id || ''}:${state.stage || ''}`; - if (failureKey !== key) { renderImageFailure(note, state, tr, {fetchApi: fetcher}); failureKey = key; } + if (failureKey !== key) { renderImageFailure(note, {...state, info}, tr, {fetchApi: fetcher}); failureKey = key; } } else { failureKey = null; note.textContent = isMissing ? imagePackUnavailableMessage(info, tr) diff --git a/web/image_generation.js b/web/image_generation.js index 2481c6c2..bc720902 100644 --- a/web/image_generation.js +++ b/web/image_generation.js @@ -18,8 +18,15 @@ export const IMAGE_GEN_POLL_MS = 500; export const IMAGE_GEN_INFO_POLL_MS = 10000; export const IMAGE_PACK_POLL_MS = 2000; export const BLANK_CANVAS_RATIOS = Object.freeze(['16:9', '9:16', '1:1', '4:3', '3:4']); -export function imageGenerationSupported(platform = globalThis.navigator?.userAgentData?.platform || globalThis.navigator?.platform || '') { - return !/Mac|iPhone|iPad/i.test(platform); +// The one switch the editor, media panel and cast library consult. Every engine platform +// generates pictures now; a Mac too small for it gets the service's small_memory reason. +export function imageGenerationSupported() { + return true; +} +// On a Mac the GPU's memory is the machine's: the same notes without the graphics card. +function memoryWords([en, zh], unified) { + return unified ? [en.replace('GPU memory', 'memory').replace(' using the GPU', ''), + zh.replace('显存', '内存').replace('占用显卡的程序', '程序')] : [en, zh]; } export function blankCanvasRatio(target) { return BLANK_CANVAS_RATIOS.find(ratio => { @@ -162,23 +169,26 @@ export function imageGenError(code, info, t) { invalid_media: ['The reference image cannot be read. Choose another image.', '参考图读不出来，请换一张图。'], reload: ['Image generation was interrupted. Try again.', '图片生成中途被打断，请重试。'], generation_failed: ['Image generation ran into an error. Try again.', '图像生成出错了，请再试一次。'], + small_memory: ['This Mac has less than 16 GB of memory and cannot generate pictures.', + '这台 Mac 的内存不足 16 GB，无法生成图片。'], }; + const say = (en, zh) => t(...memoryWords([en, zh], info?.unified)); if (code === 'pack_missing') return imagePackUnavailableMessage(info, t); if (code === 'no_memory') { // Where it ran out decides what helps: the reference pictures are read at a fixed size, // so a smaller canvas does not change that stage. const stage = info?.stage, size = info?.fitted; if (stage === 'encode') - return t('Not enough GPU memory to load the reference images. Use fewer reference images, or close other apps using the GPU, then try again.', + return say('Not enough GPU memory to load the reference images. Use fewer reference images, or close other apps using the GPU, then try again.', '显存不够，参考图没能载入。少用几张参考图，或关掉其他占用显卡的程序后再试。'); if (stage === 'load' || stage === 'text') - return t('Not enough GPU memory to load the model. Close other apps using the GPU, then try again.', + return say('Not enough GPU memory to load the model. Close other apps using the GPU, then try again.', '显存不够，模型没能载入。关掉其他占用显卡的程序后再试。'); if (size?.width && size?.height) - return t(`Not enough GPU memory, even at ${size.width}\u00a0×\u00a0${size.height}. Close other apps using the GPU, then try again.`, + return say(`Not enough GPU memory, even at ${size.width}\u00a0×\u00a0${size.height}. Close other apps using the GPU, then try again.`, `显存不够，${size.width}\u00a0×\u00a0${size.height} 也没能生成。关掉其他占用显卡的程序后再试。`); } - return t(...(Object.hasOwn(reasons, code) ? reasons[code] : reasons.generation_failed)); + return say(...(Object.hasOwn(reasons, code) ? reasons[code] : reasons.generation_failed)); } // Keep image failure copy here for editorial review. The submit card and its @@ -188,8 +198,8 @@ export const QUEUE_LABELS = Object.freeze({ image: ['Added to the queue. It starts when the previous one finishes.', '已加入队列，前面的完成后自动开始。'], video: ['Queued. It starts when the video finishes.', '已排队，视频生成完后自动开始。'], }); -export const fittedNote = (size, t) => t(`Not enough GPU memory. Made at ${size.width}\u00a0×\u00a0${size.height} instead.`, - `显存不足，已改为 ${size.width}\u00a0×\u00a0${size.height} 生成。`); +export const fittedNote = (size, t, unified = false) => t(...memoryWords([`Not enough GPU memory. Made at ${size.width}\u00a0×\u00a0${size.height} instead.`, + `显存不足，已改为 ${size.width}\u00a0×\u00a0${size.height} 生成。`], unified)); export const REPORT_LABELS = Object.freeze({ download: ['Diagnostic report', '诊断报告'], description: ['Diagnostic report: hardware, settings and timings', '诊断报告：硬件、设置和耗时'], @@ -199,7 +209,7 @@ export const REPORT_LABELS = Object.freeze({ // Shared by the editor and the other image entry points. Only the opaque report // ID goes in the request; raw service messages are never shown in the page. -export function renderImageFailure(host, {error, report_id, stage, fitted} = {}, t, +export function renderImageFailure(host, {error, report_id, stage, fitted, info} = {}, t, {productVersion = () => null, fetchApi = globalThis.fetch.bind(globalThis)} = {}) { host.replaceChildren(); if (!error || error === 'cancelled') return null; @@ -210,7 +220,7 @@ export function renderImageFailure(host, {error, report_id, stage, fitted} = {}, } const card = doc.createElement('div'); card.className = 'fv-result-links'; const reason = doc.createElement('div'); reason.className = 'fv-image-failure-reason'; - reason.setAttribute('role', 'status'); reason.textContent = imageGenError(error, {stage, fitted}, t); + reason.setAttribute('role', 'status'); reason.textContent = imageGenError(error, {stage, fitted, unified: info?.unified_memory}, t); card.append(reason); host.append(card); if (!report_id) return card; const fetchReport = async () => { @@ -723,10 +733,10 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection : missing ? imagePackUnavailableMessage(state.info, t) : promptRequired ? t('Enter a prompt first.', '请先填写提示词。') : state.running && state.waiting ? t(...(QUEUE_LABELS[state.waiting] || QUEUE_LABELS.image)) - : state.running && state.fitted ? t(`Not enough GPU memory. Retrying at ${state.fitted.width}\u00a0×\u00a0${state.fitted.height}`, - `显存不足，改用 ${state.fitted.width}\u00a0×\u00a0${state.fitted.height} 重试`) + ' · ' + percent + : state.running && state.fitted ? t(...memoryWords([`Not enough GPU memory. Retrying at ${state.fitted.width}\u00a0×\u00a0${state.fitted.height}`, + `显存不足，改用 ${state.fitted.width}\u00a0×\u00a0${state.fitted.height} 重试`], state.info?.unified_memory)) + ' · ' + percent : state.running ? (state.stage === 'load' ? t('Loading the model', '加载模型') + ' ' : '') + percent - : state.fitted ? fittedNote(state.fitted, t) : ''; + : state.fitted ? fittedNote(state.fitted, t, state.info?.unified_memory) : ''; } note.hidden = !note.textContent; bar.dataset.running = String(state.running); diff --git a/web/phone.js b/web/phone.js index 48438fd5..864a2121 100644 --- a/web/phone.js +++ b/web/phone.js @@ -67,10 +67,22 @@ root.append(top, offline, main, warning, compose, viewer, toast, tabs, pw, sheet function notice(text = '') { warning.textContent = text; warning.hidden = !text; } function connection(ok) { offline.hidden = ok || locked; } function requirePassword() { if (locked) return; locked = true; document.querySelectorAll('dialog.phone-confirm').forEach(dialog => dialog.close()); viewer.querySelector('video')?.pause(); renderShell(); password.focus(); } +// AbortSignal.any and .timeout are missing before iOS Safari 17.4 / 16 and Firefox 124 / 100, so the signals are joined by hand. +function requestSignal(signals, ms) { + const control = new AbortController(), off = []; + const stop = reason => { off.splice(0).forEach(remove => remove()); if (!control.signal.aborted) control.abort(reason); }; + for (const source of signals) { + if (source.aborted) { stop(source.reason); break; } + const on = () => stop(source.reason); + source.addEventListener('abort', on, {once: true}); off.push(() => source.removeEventListener('abort', on)); + } + setTimeout(() => stop(new DOMException('The operation timed out.', 'TimeoutError')), ms); + return control.signal; +} const api = {apiURL: path => path, async fetchApi(path, options = {}) { - const signals = [closed.signal, options.signal, AbortSignal.timeout(10000)].filter(Boolean); + const signals = [closed.signal, options.signal].filter(Boolean); // The proxy hands each signed-in device its write proof at /m/csrf; the desktop page asks the agent route. - const response = await fetch(path === '/freevideo/agent/session' ? '/m/csrf' : path, {...options, signal: AbortSignal.any(signals), credentials: 'same-origin'}); + const response = await fetch(path === '/freevideo/agent/session' ? '/m/csrf' : path, {...options, signal: requestSignal(signals, 10000), credentials: 'same-origin'}); // Without a valid device cookie the proxy answers every request with its password page, never JSON. if ([401, 429].includes(response.status) && (response.headers.get('Content-Type') || '').startsWith('text/html')) { requirePassword(); const wait = Number(response.headers.get('X-FreeVideo-Retry-After')); if (wait > 0) lockFor(wait); diff --git a/web/studio.js b/web/studio.js index a41b8a60..0e686187 100644 --- a/web/studio.js +++ b/web/studio.js @@ -1492,7 +1492,7 @@ export function openStudio(node, {extendFrom = null} = {}) { chain_cancel_failed: t('Could not cancel. Refresh the page and try again.', '没有取消成功，请刷新页面后再试。'), chain_resume_failed: t('Could not continue. Refresh the page and try again.', '没有继续生成，请刷新页面后再试。'), }[error?.message] || (error?.said ? t(error.said[0], error.said[1]) : null) || (isNetworkTypeError(error) ? connectionLostText(t) : error?.message) || String(error)); - let lastQueueError = null; + let lastQueueError = null, connectionNotice = false; function showQueueError(error) { if (disposed) return; status.dataset.error = 'true'; @@ -1506,11 +1506,21 @@ export function openStudio(node, {extendFrom = null} = {}) { : queueError(error); // Recoverable chain actions need one visible sentence, not a second error card. if (error?.action) { status.scrollIntoView({block: 'nearest'}); return; } - const identity = error?.schema === 'freevideo.error-report' ? error : error?.message || error; + // A loop stopped by a long outage is a new state of the same notice: show its final details once. + const identity = error?.schema === 'freevideo.error-report' ? error : error?.loopStopped ? `${error.message} (loop stopped)` : error?.message || error; if (lastQueueError === identity) return; lastQueueError = identity; const report = failure.show(error, error?.schema === 'freevideo.error-report', {stage: 'queue', node, graph: app.graph}); node.freevideoFailureReport = report; node.freevideoFailure = errorText(report); + connectionNotice = isNetworkTypeError(error) && !report.browser_extension; + } + // The queue read works again: take the lost-connection notice down. A failure card for anything else stays. + function clearConnectionNotice() { + if (!connectionNotice) return; + connectionNotice = false; + if (node.freevideoFailureReport?.stage !== 'queue') return; + lastQueueError = null; failure.clear(); node.freevideoFailure = ''; node.freevideoFailureReport = null; + status.dataset.error = 'false'; status.textContent = ''; } function updateRunButton() { const submitting = capturing || queueState?.submitting; @@ -1620,7 +1630,7 @@ export function openStudio(node, {extendFrom = null} = {}) { } if (movedId && state.pending.some(row => row.id === movedId) && !queueList.querySelector('.fv-queue-moved')) showMoved(); if (!busy) cancelling = false; - if (state.error) showQueueError(state.error); + if (state.error) showQueueError(state.error); else clearConnectionNotice(); updateRunButton(); } const syncQueue = () => queueController.refresh(); diff --git a/web/studio_queue.js b/web/studio_queue.js index e34ddc86..75802379 100644 --- a/web/studio_queue.js +++ b/web/studio_queue.js @@ -1,8 +1,13 @@ // ComfyUI owns execution. This controller only submits frozen workflows and // replenishes a loop after its previous request has actually completed. -import {fetchQueueRead, detectBrowserExtension} from './browser_extension.js'; +import {fetchQueueRead, detectBrowserExtension, isNetworkTypeError} from './browser_extension.js'; const clone = value => structuredClone(value); +// One dropped read is usually a connection reset or an engine restart. Only this many +// polls failing in a row are shown; the next successful read clears the notice. +// While the outage lasts the polls back off, and a loop is stopped once it passes the limit. +const OUTAGE_POLLS = 3, OUTAGE_LOOP_LIMIT_MS = 180000, OUTAGE_MAX_DELAY_MS = 8000; + // ComfyUI leaves a muted (2) or bypassed (4) node out of the request. The creation panel is where // the node is used, so a request from it turns the node back on; true when it was off. export function enableNode(node) { @@ -123,7 +128,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 formatError = error => error} = {}) { nodeId = String(nodeId); let running = null, pending = [], submitting = false, loop = null, timer = null; - let refreshing = null, refreshAgain = false, lastError = null, disposed = false; + let refreshing = null, refreshAgain = false, lastError = null, disposed = false, outage = null; const listeners = new Set(); const owned = new Set(), delivered = new Set(); function remember(id) { owned.add(id); if (owned.size > 128) { const old = owned.values().next().value; owned.delete(old); delivered.delete(old); } } @@ -136,7 +141,8 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 const publish = () => { for (const listener of listeners) listener(state()); }; function schedule() { clearTimeout(timer); timer = null; - if (!disposed && pollMs && (listeners.size || loop?.active)) timer = setTimeout(() => refresh().catch(() => {}), pollMs); + if (!disposed && pollMs && (listeners.size || loop?.active)) timer = setTimeout(() => refresh().catch(() => {}), outage && outage.failures > OUTAGE_POLLS + ? Math.min(pollMs * 2 ** (outage.failures - OUTAGE_POLLS), OUTAGE_MAX_DELAY_MS) : pollMs); } function fail(error) { if (loop) loop.active = false; @@ -218,6 +224,8 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 running = (queue.queue_running || []).filter(matches).map(entry)[0] || null; // Comfy returns its heap array; the queue number gives the real running order. pending = (queue.queue_pending || []).filter(matches).sort((a, b) => a[0] - b[0]).map(entry); + outage = null; + if (isNetworkTypeError(lastError) && !lastError.loopStopped) lastError = null; for (const row of [running, ...pending]) if (row) remember(row.id); await advanceLoop(); publish(); } @@ -226,7 +234,21 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (refreshing) { refreshAgain = true; return refreshing; } refreshing = (async () => { do { refreshAgain = false; await readQueue(); } while (refreshAgain && !disposed); - })().catch(error => { fail(error); throw error; }).finally(() => { refreshing = null; schedule(); }); + })().catch(error => { + if (!isNetworkTypeError(error) || detectBrowserExtension(error)) { fail(error); throw error; } + // A lost connection does not stop a loop at first: it only continues after a successful read. + outage = {failures: (outage?.failures || 0) + 1, since: outage?.since ?? Date.now()}; + if (outage.failures < OUTAGE_POLLS) return; + // The notice for a stopped loop stays as it is while the outage goes on. + if (lastError?.loopStopped) throw error; + const lasted = Date.now() - outage.since, giveUp = lasted >= OUTAGE_LOOP_LIMIT_MS && loop?.active; + const note = `Queue reads failed ${outage.failures} times in a row over ${Math.round(lasted / 1000)} s; ` + + `online=${globalThis.navigator?.onLine}, page hidden=${globalThis.document?.hidden}.${giveUp ? ' The loop was stopped.' : ''}`; + if (typeof error === 'object' && !String(error.details || '').includes('Queue reads failed')) error.details = [error.details, note].filter(Boolean).join('\n'); + // A stopped loop keeps its notice after the connection returns, so the user sees why it stopped. + if (giveUp) { if (typeof error === 'object') error.loopStopped = true; fail(error); throw error; } + lastError = error; publish(); throw error; + }).finally(() => { refreshing = null; schedule(); }); return refreshing; } async function start(snapshot, {count = 1, repeat = false, seedIndex = -1, onAccepted = null} = {}) {