diff --git a/.github/evals/cueq_challenges.py b/.github/evals/cueq_challenges.py new file mode 100644 index 00000000..79137c3d --- /dev/null +++ b/.github/evals/cueq_challenges.py @@ -0,0 +1,485 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Execute challenge submissions. This file is a verifier, never a skill resource. + +Standalone: python cueq_challenges.py --backend jax --case layout --solution FILE +Harbor: read entry.json and grade /workspace/solution.py in the prepared CPU venv. +Custom scores supplement ACES; they do not overwrite its metrics or pass rule. +""" + +from __future__ import annotations + +import argparse +import ast +import importlib.util +import json +import os +import subprocess +import sys +import traceback +from pathlib import Path + +PARAMETERS = ("radial_in", "radial_out", "projection") + + +def load_solution(path): + # Match normal execution of a saved Python module, including sibling helpers + # and decorators that resolve annotations through sys.modules. + sys.path.insert(0, str(path.resolve().parent)) + spec = importlib.util.spec_from_file_location("challenge_solution", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def array(value): + import numpy as np + + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + return np.asarray(value) + + +def close(actual, expected, *, exact=False): + import numpy as np + + actual, expected = array(actual), array(expected) + assert actual.shape == expected.shape, (actual.shape, expected.shape) + assert actual.dtype == expected.dtype, (actual.dtype, expected.dtype) + assert np.isfinite(actual).all(), "nonfinite result" + if exact: + np.testing.assert_array_equal(actual, expected) + else: + np.testing.assert_allclose(actual, expected, rtol=2e-8, atol=2e-9) + + +def numpy_blocks(irreps, flat): + """Independent oracle: slice original blocks and concatenate equal irreps.""" + import numpy as np + + parts, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)).swapaxes(-1, -2) + parts.setdefault(ir, []).append(block) + offset += mul * ir.dim + return {ir: np.concatenate(values, axis=-2) for ir, values in parts.items()} + + +def change_layout(flat, irreps, *, source): + import numpy as np + + parts, offset = [], 0 + for mul, ir in irreps: + suffix = (mul, ir.dim) if source == "mul_ir" else (ir.dim, mul) + block = flat[..., offset : offset + mul * ir.dim] + parts.append( + block.reshape(flat.shape[:-1] + suffix) + .swapaxes(-1, -2) + .reshape(flat.shape[:-1] + (mul * ir.dim,)) + ) + offset += mul * ir.dim + return np.concatenate(parts, axis=-1) + + +def check_layout(solution, backend): + import numpy as np + + import cuequivariance as cue + + if backend == "jax": + import jax + import jax.numpy as xp + + jax.config.update("jax_enable_x64", True) + else: + import torch + + for text in ("1o + 2x0e + 2x1o + 1e + 0o", "2x1e + 0o + 1o + 1e + 2x0e + 1o"): + irreps = cue.Irreps("O3", text) + for batch in ((), (4,), (2, 3)): + raw = np.arange(int(np.prod(batch)) * irreps.dim, dtype=np.float64) + raw = (raw.reshape(batch + (irreps.dim,)) + 0.25) / 8 + if backend == "jax": + x = xp.asarray(raw) + else: + # Exercise a noncontiguous tensor with the same values. + x = torch.tensor(np.stack((raw, -raw), -1))[..., 0] + x.requires_grad_() + assert not x.is_contiguous() + expected = numpy_blocks(irreps, raw) + blocks = solution.to_blocks(irreps, x) + assert set(blocks) == set(expected), ( + "keys must include both degree and parity" + ) + for ir, value in expected.items(): + close(blocks[ir], value, exact=True) + restored = solution.from_blocks(irreps, blocks) + close(restored, raw, exact=True) + + # Use a nonuniform linear loss so inverse permutations cannot hide. + cotangent = np.linspace(0.2, 1.3, raw.size).reshape(raw.shape) + if backend == "jax": + + def convert(z): + return solution.to_blocks(irreps, z) + + def round_trip(z): + return solution.from_blocks(irreps, convert(z)) + + close(jax.jit(round_trip)(x), raw, exact=True) + for ir, value in jax.jit(convert)(x).items(): + close(value, expected[ir], exact=True) + gradient = jax.grad( + lambda z: xp.sum(round_trip(z) * xp.asarray(cotangent)) + )(x) + else: + gradient = torch.autograd.grad( + (restored * torch.tensor(cotangent)).sum(), x + )[0] + close(gradient, cotangent) + + rep = cue.IrrepsAndLayout(irreps, cue.ir_mul) + for angles, parity in (([0.21, -0.32, 0.17], [0]), ([0, 0, 0], [1])): + matrix = rep.exp_map(angles, parity) + transformed = raw @ matrix.T + z = ( + xp.asarray(transformed) + if backend == "jax" + else torch.tensor(transformed) + ) + transformed_blocks = solution.to_blocks(irreps, z) + for ir in expected: + close( + transformed_blocks[ir], + expected[ir] @ ir.exp_map(angles, parity).T, + ) + + +def graph_descriptors(): + import cuequivariance as cue + + irreps_in = cue.Irreps("SO3", "2x0 + 2x1") + irreps_sh = cue.Irreps("SO3", "0 + 1 + 2") + irreps_out = cue.Irreps("SO3", "0 + 1") + product = cue.descriptors.channelwise_tensor_product( + irreps_in, irreps_sh, [cue.SO3(0), cue.SO3(1)] + ) + projection = cue.descriptors.linear(product.outputs[0].irreps, irreps_out) + harmonics = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + return irreps_in, irreps_out, product, projection, harmonics + + +def graph_reference(params, x, vectors, senders, receivers, num_dst): + """Core NumPy descriptors plus an explicit edge loop; no frontend forward.""" + import numpy as np + + irreps_in, irreps_out, product, projection, harmonics = graph_descriptors() + lengths = np.linalg.norm(vectors, axis=-1, keepdims=True) + weights = np.tanh(lengths @ params["radial_in"]) @ params["radial_out"] + sh = harmonics(vectors / lengths)[0] + flat = change_layout(x, irreps_in, source="mul_ir") + result = np.zeros((num_dst, irreps_out.dim), dtype=np.float64) + for edge, (src, dst) in enumerate(zip(senders, receivers)): + message = product(weights[edge], flat[src], sh[edge])[0] + projected = projection(params["projection"], message)[0] + result[dst] += change_layout(projected, irreps_out, source="ir_mul") + return result + + +def graph_inputs(seed): + import numpy as np + + rng = np.random.default_rng(seed) + irreps_in, _, product, projection, _ = graph_descriptors() + params = { + "radial_in": rng.normal(scale=0.4, size=(1, 4)), + "radial_out": rng.normal(scale=0.4, size=(4, product.inputs[0].dim)), + "projection": rng.normal(scale=0.4, size=(projection.inputs[0].dim,)), + } + x = rng.normal(size=(5, irreps_in.dim)) + vectors = rng.normal(size=(7, 3)) + np.array([0.4, -0.2, 0.7]) + senders = np.array([0, 1, 3, 0, 1, 4, 2], dtype=np.int64) + receivers = np.array([2, 0, 2, 1, 2, 0, 1], dtype=np.int64) + return params, x, vectors, senders, receivers + + +def check_graph(solution, backend): + import numpy as np + + import cuequivariance as cue + + if backend == "jax": + import jax + import jax.numpy as jnp + + jax.config.update("jax_enable_x64", True) + else: + import torch + + irreps_in, irreps_out, _, _, _ = graph_descriptors() + for seed in (41, 83): + params, x, vectors, senders, receivers = graph_inputs(seed) + expected = graph_reference(params, x, vectors, senders, receivers, 4) + if backend == "jax": + p = {k: jnp.asarray(v) for k, v in params.items()} + a, b, s, r = map(jnp.asarray, (x, vectors, senders, receivers)) + + def apply(pp, xx, vv, ss, rr): + return solution.apply(pp, xx, vv, ss, rr, 4) + + actual = apply(p, a, b, s, r) + close(jax.jit(apply)(p, a, b, s, r), expected) + + def loss(pp, xx, vv): + return jnp.sum(apply(pp, xx, vv, s, r) ** 2) + + derivatives = jax.grad(loss, argnums=(0, 1, 2))(p, a, b) + close( + jax.jit(jax.grad(loss))(p, a, b)["radial_in"], + derivatives[0]["radial_in"], + ) + grad_params, grad_x, grad_vectors = derivatives + else: + model = solution.GraphLayer().double() + assert isinstance(model, torch.nn.Module) + registered = dict(model.named_parameters()) + assert set(registered) == set(PARAMETERS), ( + "register exactly the three specified parameters" + ) + with torch.no_grad(): + for name, value in params.items(): + assert registered[name].shape == value.shape, name + registered[name].copy_(torch.tensor(value)) + a = torch.tensor(x, requires_grad=True) + b = torch.tensor(vectors, requires_grad=True) + s, r = torch.tensor(senders), torch.tensor(receivers) + + def apply(pp, xx, vv, ss, rr): + return model(xx, vv, ss, rr, 4) + + p = registered + actual = apply(p, a, b, s, r) + grads = torch.autograd.grad( + actual.square().sum(), [*(registered[k] for k in PARAMETERS), a, b] + ) + grad_params = dict(zip(PARAMETERS, grads[:3])) + grad_x, grad_vectors = grads[-2:] + close(actual, expected) + close(apply(p, a, b, s, r), expected) + close(actual[3], np.zeros(4), exact=True) + + # Relabel source nodes and reorder edges independently. + order = np.array([3, 0, 4, 1, 2]) + inverse = np.argsort(order) + edge_order = np.array([6, 1, 4, 0, 3, 5, 2]) + if backend == "jax": + new_s, new_r = map( + jnp.asarray, (inverse[senders][edge_order], receivers[edge_order]) + ) + else: + new_s, new_r = map( + torch.tensor, (inverse[senders][edge_order], receivers[edge_order]) + ) + close(apply(p, a[order], b[edge_order], new_s, new_r), expected) + destination_order = np.array([3, 1, 0, 2]) + destination_inverse = np.argsort(destination_order) + cast = jnp.asarray if backend == "jax" else torch.tensor + close( + apply(p, a, b, s, cast(destination_inverse[receivers])), + expected[destination_order], + ) + if backend == "jax": + resized = solution.apply(p, a[:4], b[:4], s[:4], r[:4], 6) + else: + resized = model(a[:4], b[:4], s[:4], r[:4], 6) + close( + resized, + graph_reference(params, x[:4], vectors[:4], senders[:4], receivers[:4], 6), + ) + angles = [0.31, -0.23, 0.41] + din = cue.IrrepsAndLayout(irreps_in, cue.mul_ir).exp_map(angles, []) + dout = cue.IrrepsAndLayout(irreps_out, cue.mul_ir).exp_map(angles, []) + dv = cue.SO3(1).exp_map(angles, []) + close(apply(p, a @ cast(din.T), b @ cast(dv.T), s, r), expected @ dout.T) + + # Check one informative derivative in every parameter/input against NumPy. + gradients = {**grad_params, "features": grad_x, "vectors": grad_vectors} + values = {**params, "features": x, "vectors": vectors} + for name, gradient in gradients.items(): + gradient = array(gradient) + assert np.isfinite(gradient).all() and np.max(np.abs(gradient)) > 1e-9, name + index = np.unravel_index(np.argmax(np.abs(gradient)), gradient.shape) + losses = [] + for sign in (-1, 1): + shifted = {k: v.copy() for k, v in values.items()} + shifted[name][index] += sign * 1e-5 + y = graph_reference( + {k: shifted[k] for k in PARAMETERS}, + shifted["features"], + shifted["vectors"], + senders, + receivers, + 4, + ) + losses.append(np.sum(y**2)) + np.testing.assert_allclose( + gradient[index], (losses[1] - losses[0]) / 2e-5, rtol=2e-5, atol=2e-7 + ) + # A real parameter update must retain the reference semantics. + updated = {k: params[k] - 1e-3 * array(grad_params[k]) for k in PARAMETERS} + if backend == "jax": + after = apply({k: jnp.asarray(v) for k, v in updated.items()}, a, b, s, r) + else: + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer.zero_grad() + for name in PARAMETERS: + p[name].grad = grad_params[name].detach().clone() + optimizer.step() + after = apply(p, a, b, s, r) + assert {k: id(v) for k, v in model.named_parameters()} == { + k: id(v) for k, v in registered.items() + } + close(after, graph_reference(updated, x, vectors, senders, receivers, 4)) + assert not np.allclose(array(after), array(actual)), ( + "update did not affect output" + ) + + +def check_routing(solution, backend, path): + import numpy as np + + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + assert not any(a.name.startswith("cuequivariance") for a in node.names) + if isinstance(node, ast.ImportFrom): + assert not (node.module or "").startswith("cuequivariance") + for count in (1, 3, 7): + predictions = np.linspace(-0.7, 1.1, count).reshape(count, 1) + targets = np.linspace(1.2, -0.1, count) + expected = np.mean((predictions[:, 0] - targets) ** 2) + expected_grad = 2 * (predictions - targets[:, None]) / count + if backend == "jax": + import jax + import jax.numpy as jnp + + jax.config.update("jax_enable_x64", True) + p, t = jnp.asarray(predictions), jnp.asarray(targets) + value = jax.jit(solution.mse_loss)(p, t) + gradient = jax.grad(solution.mse_loss)(p, t) + else: + import torch + + p = torch.tensor(predictions, requires_grad=True) + value = solution.mse_loss(p, torch.tensor(targets)) + gradient = torch.autograd.grad(value, p)[0] + close(value, expected) + close(gradient, expected_grad) + + +def worker(backend, case, solution): + try: + module = load_solution(solution) + if case == "layout": + check_layout(module, backend) + elif case == "graph": + check_graph(module, backend) + else: + check_routing(module, backend, solution) + except Exception as error: + return { + "passed": False, + "error": f"{type(error).__name__}: {error}", + "traceback": traceback.format_exc(), + } + return {"passed": True} + + +def evaluate(backend, case, solution, *, timeout=180, python=None): + command = [ + python or sys.executable, + str(Path(__file__).resolve()), + "--worker", + "--backend", + backend, + "--case", + case, + "--solution", + str(solution.resolve()), + ] + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, "JAX_PLATFORMS": "cpu", "PYTHONDONTWRITEBYTECODE": "1"}, + ) + marker = "CUEQ_RESULT=" + if result.returncode or marker not in result.stdout: + return { + "passed": False, + "error": f"verifier exit {result.returncode}", + "output": (result.stdout + result.stderr)[-6000:], + } + return json.loads(result.stdout.rsplit(marker, 1)[1].splitlines()[0]) + except subprocess.TimeoutExpired: + return {"passed": False, "error": f"submission exceeded {timeout} seconds"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("jax", "torch")) + parser.add_argument("--case", choices=("layout", "graph", "routing")) + parser.add_argument("--solution", type=Path) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--worker", action="store_true") + args = parser.parse_args() + harbor = args.backend is None + if harbor: + entry_path = Path(os.environ.get("HARBOR_ENTRY_JSON", "/tests/entry.json")) + entry = json.loads(entry_path.read_text()) + backend, _, case, version = entry["id"].split("-") + assert version == "v1" and backend in ("jax", "torch") + assert case in ("layout", "graph", "routing") + args.backend, args.case = backend, case + args.solution = Path("/workspace/solution.py") + elif args.case is None or args.solution is None: + parser.error("--backend requires --case and --solution") + if args.worker: + print( + "CUEQ_RESULT=" + json.dumps(worker(args.backend, args.case, args.solution)) + ) + return + interpreter = ( + Path("/workspace/.venv/bin/python") if harbor else Path(sys.executable) + ) + result = evaluate( + args.backend, + args.case, + args.solution, + timeout=args.timeout, + python=str(interpreter), + ) + print(json.dumps(result, indent=2)) + if harbor: + reward = Path( + os.environ.get("HARBOR_REWARD_JSON", "/logs/verifier/reward.json") + ) + reward.parent.mkdir(parents=True, exist_ok=True) + reward.write_text( + json.dumps( + { + "custom_metrics": {"numerical_success": float(result["passed"])}, + "details": {"numerical_success": result}, + }, + indent=2, + ) + ) + elif not result["passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/evals/reference_jax_challenges.py b/.github/evals/reference_jax_challenges.py new file mode 100644 index 00000000..80c1bfcd --- /dev/null +++ b/.github/evals/reference_jax_challenges.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Maintainer reference only. Never copy this file into an agent environment.""" + +import cuequivariance_jax.ir_dict as ir_dict +import jax +import jax.numpy as jnp + +import cuequivariance as cue +import cuequivariance_jax as cuex + +jax.config.update("jax_enable_x64", True) + + +def to_blocks(irreps, flat): + data = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() + return ir_dict.flat_to_dict(data.irreps, data.array, layout="ir_mul") + + +def from_blocks(irreps, blocks): + offsets, parts = {}, [] + for mul, ir in irreps: + start = offsets.get(ir, 0) + block = blocks[ir][..., start : start + mul, :] + parts.append(block.swapaxes(-1, -2).reshape(block.shape[:-2] + (mul * ir.dim,))) + offsets[ir] = start + mul + return jnp.concatenate(parts, axis=-1) + + +INPUT = cue.Irreps("SO3", "2x0 + 2x1") +OUTPUT = cue.Irreps("SO3", "0 + 1") +PRODUCT = cue.descriptors.channelwise_tensor_product( + INPUT, cue.Irreps("SO3", "0 + 1 + 2"), [cue.SO3(0), cue.SO3(1)] +) +PROJECTION = cue.descriptors.linear(PRODUCT.outputs[0].irreps, OUTPUT) +HARMONICS = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + + +def transpose(flat, irreps, *, source): + parts, offset = [], 0 + for mul, ir in irreps: + suffix = (mul, ir.dim) if source == "mul_ir" else (ir.dim, mul) + block = flat[..., offset : offset + mul * ir.dim] + parts.append( + block.reshape(flat.shape[:-1] + suffix) + .swapaxes(-1, -2) + .reshape(flat.shape[:-1] + (mul * ir.dim,)) + ) + offset += mul * ir.dim + return jnp.concatenate(parts, axis=-1) + + +def run(descriptor, operands, rows): + return cuex.segmented_polynomial( + descriptor.polynomial, + operands, + [jax.ShapeDtypeStruct((rows, descriptor.outputs[0].dim), jnp.float64)], + method="naive", + )[0] + + +def apply(params, features, vectors, senders, receivers, num_dst): + lengths = jnp.linalg.norm(vectors, axis=-1, keepdims=True) + weights = jnp.tanh(lengths @ params["radial_in"]) @ params["radial_out"] + sh = run(HARMONICS, [vectors / lengths], vectors.shape[0]) + inputs = transpose(features, INPUT, source="mul_ir") + messages = run(PRODUCT, [weights, inputs[senders], sh], vectors.shape[0]) + summed = ( + jnp.zeros((num_dst, messages.shape[-1]), messages.dtype) + .at[receivers] + .add(messages) + ) + output = run(PROJECTION, [params["projection"], summed], num_dst) + return transpose(output, OUTPUT, source="ir_mul") diff --git a/.github/evals/reference_torch_challenges.py b/.github/evals/reference_torch_challenges.py new file mode 100644 index 00000000..68444c43 --- /dev/null +++ b/.github/evals/reference_torch_challenges.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Maintainer reference only. Never copy this file into an agent environment.""" + +import torch + +import cuequivariance as cue +import cuequivariance_torch as cuet + + +def to_blocks(irreps, flat): + parts, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)).transpose(-1, -2) + parts.setdefault(ir, []).append(block) + offset += mul * ir.dim + return {ir: torch.cat(values, dim=-2) for ir, values in parts.items()} + + +def from_blocks(irreps, blocks): + offsets, parts = {}, [] + for mul, ir in irreps: + start = offsets.get(ir, 0) + block = blocks[ir][..., start : start + mul, :] + parts.append( + block.transpose(-1, -2).reshape(block.shape[:-2] + (mul * ir.dim,)) + ) + offsets[ir] = start + mul + return torch.cat(parts, dim=-1) + + +class GraphLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.product = cuet.ChannelWiseTensorProduct( + cue.Irreps("SO3", "2x0 + 2x1"), + cue.Irreps("SO3", "0 + 1 + 2"), + [cue.SO3(0), cue.SO3(1)], + shared_weights=False, + internal_weights=False, + layout=cue.mul_ir, + method="naive", + dtype=torch.float64, + ) + self.readout = cuet.Linear( + self.product.irreps_out, + cue.Irreps("SO3", "0 + 1"), + internal_weights=False, + layout=cue.mul_ir, + method="naive", + dtype=torch.float64, + ) + self.sh = cuet.SphericalHarmonics([0, 1, 2], normalize=True, method="naive") + self.radial_in = torch.nn.Parameter(torch.zeros(1, 4, dtype=torch.float64)) + self.radial_out = torch.nn.Parameter( + torch.zeros(4, self.product.weight_numel, dtype=torch.float64) + ) + self.projection = torch.nn.Parameter( + torch.zeros(self.readout.weight_numel, dtype=torch.float64) + ) + + def forward(self, features, vectors, senders, receivers, num_dst): + lengths = torch.linalg.vector_norm(vectors, dim=-1, keepdim=True) + weights = torch.tanh(lengths @ self.radial_in) @ self.radial_out + messages = self.product( + features, + self.sh(vectors), + weight=weights, + indices_1=senders, + indices_out=receivers, + size_out=num_dst, + ) + return self.readout(messages, self.projection[None, :]) diff --git a/.github/scripts/check_skill_packaging.py b/.github/scripts/check_skill_packaging.py new file mode 100644 index 00000000..d84fb97f --- /dev/null +++ b/.github/scripts/check_skill_packaging.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Check skill resources and CLI through the same build path as publish.yml.""" + +import argparse +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PACKAGES = ("cuequivariance", "cuequivariance_jax", "cuequivariance_torch") + + +def check_cli(package, import_root, expected, cwd): + env = {**os.environ, "PYTHONPATH": str(import_root)} + # Verify import provenance too: an editable install must not hide a missing + # resource or module in the wheel being checked. + subprocess.run( + [ + sys.executable, + "-c", + "import importlib, pathlib, sys; " + "m = importlib.import_module(sys.argv[1]); " + "assert pathlib.Path(m.__file__).is_relative_to(sys.argv[2]), m.__file__", + package, + str(import_root), + ], + cwd=cwd, + env=env, + check=True, + ) + actual = subprocess.check_output( + [sys.executable, "-m", package, "skill"], cwd=cwd, env=env + ) + assert actual == expected + b"\n", ( + f"{package}: CLI output differs from canonical skill" + ) + + +def build(project, kind, destination): + subprocess.run( + [ + sys.executable, + "-m", + "hatchling", + "build", + "-t", + kind, + "-d", + str(destination), + ], + cwd=project, + check=True, + ) + + +def check_package(package, temp): + canonical = ROOT / "skills" / package.replace("_", "-") / "SKILL.md" + expected = canonical.read_bytes() + assert not canonical.is_symlink(), f"{canonical}: nv-carps needs a real file" + assert (ROOT / package / package / "SKILL.md").read_bytes() == expected + check_cli(package, ROOT / package, expected, temp) + + # Publishing already dereferences LICENSE, README, VERSION and SKILL links. + # Copy with dereferencing to exercise that behavior without modifying the + # developer's checkout. Direct sdists from the linked tree are not releases. + project = temp / "source" / package + shutil.copytree( + ROOT / package, + project, + symlinks=False, + ignore=shutil.ignore_patterns( + "__pycache__", ".pytest_cache", "dist", "*.egg-info" + ), + ) + wheel_dir, sdist_dir = temp / "wheel", temp / "sdist" + build(project, "wheel", wheel_dir) + build(project, "sdist", sdist_dir) + archive = next(sdist_dir.glob("*.tar.gz")) + with tarfile.open(archive) as source: + members = [ + m for m in source.getmembers() if m.name.endswith(f"/{package}/SKILL.md") + ] + assert len(members) == 1 and members[0].isfile(), ( + "sdist must contain a regular SKILL.md" + ) + assert source.extractfile(members[0]).read() == expected + source.extractall(temp / "unpacked", filter="data") + rebuilt = temp / "rebuilt" + build(next((temp / "unpacked").iterdir()), "wheel", rebuilt) + for label, directory in (("wheel", wheel_dir), ("rebuilt", rebuilt)): + wheel = next(directory.glob("*.whl")) + import_root = temp / f"import-{label}" + with zipfile.ZipFile(wheel) as contents: + assert contents.read(f"{package}/SKILL.md") == expected + contents.extractall(import_root) + check_cli(package, import_root, expected, temp) + print(f"{package}: source CLI, wheel, sdist, and rebuilt wheel passed") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package", choices=PACKAGES) + args = parser.parse_args() + for package in (args.package,) if args.package else PACKAGES: + with tempfile.TemporaryDirectory(prefix=f"{package}-skill-packaging-") as temp: + check_package(package, Path(temp)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/prepare_skill_challenges.py b/.github/scripts/prepare_skill_challenges.py new file mode 100644 index 00000000..b583fe77 --- /dev/null +++ b/.github/scripts/prepare_skill_challenges.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Stage challenge-v1 pilots without changing canonical skills or regression evals. + +Run with Python + PyYAML. --export-harbor additionally needs SkillEvaluator 1.5.6 +or a compatible builder that separates verifier material from runtime skills. +All output is created in a fresh directory; this command never launches agents. +""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import runpy +import shlex +import shutil +import tempfile +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +GRADER = ROOT / ".github/evals/cueq_challenges.py" +# Match the models resolved in the user's 2026-10-20 comparison. +MODELS = { + "claude-code": "aws/anthropic/bedrock-claude-opus-4-8", + "codex": "openai/openai/gpt-5.5", +} + + +def digest(path): + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify_environment(task, backend, with_skill): + """Fail the export if evaluator or reference material reaches the agent.""" + environment = task / "environment" + for path in environment.rglob("*"): + relative = path.relative_to(environment) + assert "evals" not in relative.parts, relative + assert path.name != "grader.py", relative + assert not path.name.startswith("reference_"), relative + assert path.name not in ("entry.json", "challenges.json"), relative + skill = environment / "skills" / f"cuequivariance-{backend}" / "SKILL.md" + assert skill.exists() == with_skill + if with_skill: + assert digest(skill) == digest(ROOT / "skills" / skill.parent.name / "SKILL.md") + assert (task / "tests/grader.py").read_bytes() == GRADER.read_bytes() + entry = json.loads((task / "tests/entry.json").read_text()) + assert entry["has_skill"] == with_skill + + +def prepare(destination, backends, *, export_harbor=False): + destination = Path(destination).resolve() + destination.mkdir(parents=True, exist_ok=False) + if export_harbor: + from skillevaluator.tier3.harbor.adapter import ( + generate_harbor_tasks, + stage_native_harbor_tasks, + ) + + if ( + "evaluator_skill_path" + not in inspect.signature(generate_harbor_tasks).parameters + ): + raise RuntimeError( + "Use the SkillEvaluator 1.5.6 builder with private evaluator-source support" + ) + manifest = { + "benchmark": "cuequivariance-challenges-v1", + "attempts_per_case_per_arm": 3, + "stop_on_pass": False, + "agents": ["claude-code", "codex"], + "agent_models": MODELS, + "grading": "aces_plus_custom", + "custom_metric": "numerical_success", + "custom_metric_changes_aces_gate": False, + "skills": {}, + } + for backend in backends: + name = f"cuequivariance-{backend}" + source, staged = ROOT / "skills" / name, destination / "skills" / name + staged.mkdir(parents=True) + shutil.copy2(source / "SKILL.md", staged / "SKILL.md") + # Only runtime resources and the challenge evaluator are staged. + for directory in ("scripts", "references", "assets", "agents"): + if (source / directory).is_dir(): + shutil.copytree(source / directory, staged / directory) + evals = staged / "evals" + evals.mkdir() + shutil.copy2(source / "evals/challenges.json", evals / "evals.json") + shutil.copy2(GRADER, evals / "grader.py") + config = yaml.safe_load((source / "evals/config.yml").read_text()) + config["grading"]["mode"] = "aces_plus_custom" + (evals / "config.yml").write_text(yaml.safe_dump(config, sort_keys=False)) + native = config["harbor"].get("task_source") == "native_harbor" + if native: + shutil.copy2(source / "evals/setup_cpu.sh", evals / "setup_cpu.sh") + # Render the challenge IDs/prompts, never copy regression tasks. + builder = runpy.run_path( + str(ROOT / ".github/scripts/prepare_skill_eval_tasks.py") + ) + builder["write_tasks"](staged) + command = [ + "skillevaluator", + "tier3", + "evaluate", + str(staged), + "--env-mode", + "k8s-sandbox", + "--agents", + "claude-code,codex", + "--agent-model", + f"claude-code={MODELS['claude-code']}", + "--agent-model", + f"codex={MODELS['codex']}", + "--n-attempts", + "3", + "--no-stop-on-pass", + "--n-concurrent", + "1", + "--max-agents", + "1", + "--harbor-keep-jobs", + "--results-dir", + str(destination / "results" / name), + ] + record = { + "path": str(staged), + "skill_digest": digest(staged / "SKILL.md"), + "dataset_digest": digest(evals / "evals.json"), + "config_digest": digest(evals / "config.yml"), + "grader_digest": digest(evals / "grader.py"), + "case_ids": [ + e["id"] for e in json.loads((evals / "evals.json").read_text())["evals"] + ], + "command": command, + } + if native: + record["setup_digest"] = digest(evals / "setup_cpu.sh") + record["native_task_digests"] = { + str(p.relative_to(evals / "harbor")): digest(p) + for p in sorted((evals / "harbor").rglob("*")) + if p.is_file() + } + if export_harbor: + tasks_by_arm = {} + emit_tasks = stage_native_harbor_tasks if native else generate_harbor_tasks + for arm, with_skill in (("with_skill", True), ("baseline", False)): + task_root = destination / "harbor" / backend / arm + tasks = emit_tasks( + staged, + task_root, + evaluator_skill_path=staged, + with_skill=with_skill, + workspace_mode="isolated", + grading_mode="aces_plus_custom", + env_mode="astra-sandbox", + runtime_env=config["harbor"]["runtime_env"], + pre_agent_setup=config["harbor"].get("pre_agent_setup"), + agent_workdir="/workspace", + copy_repo=False, + repo_context_exclude_paths=[ROOT / ".github/evals"], + ) + for task in tasks: + verify_environment(task, backend, with_skill) + tasks_by_arm[arm] = tasks + # Inputs and the verifier must be identical in the two conditions. + for with_task, without_task in zip( + tasks_by_arm["with_skill"], tasks_by_arm["baseline"], strict=True + ): + assert (with_task / "instruction.md").read_bytes() == ( + without_task / "instruction.md" + ).read_bytes() + assert (with_task / "tests/grader.py").read_bytes() == ( + without_task / "tests/grader.py" + ).read_bytes() + record["verified_harbor_tasks"] = sum(map(len, tasks_by_arm.values())) + manifest["skills"][name] = record + (destination / "pilot.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("jax", "torch", "both"), default="both") + parser.add_argument( + "--output", type=Path, help="New directory; existing paths are refused." + ) + parser.add_argument("--export-harbor", action="store_true") + args = parser.parse_args() + if args.output is None: + root = ROOT / ".venv" + root.mkdir(exist_ok=True) + # The parent is unique; prepare owns a new, initially absent child. + args.output = ( + Path(tempfile.mkdtemp(prefix="cueq-challenges-v1-", dir=root)) / "pilot" + ) + backends = ("jax", "torch") if args.backend == "both" else (args.backend,) + manifest = prepare(args.output, backends, export_harbor=args.export_harbor) + print(f"Pilot manifest: {args.output.resolve() / 'pilot.json'}") + for record in manifest["skills"].values(): + print(shlex.join(record["command"])) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/prepare_skill_eval_tasks.py b/.github/scripts/prepare_skill_eval_tasks.py new file mode 100644 index 00000000..dc45a517 --- /dev/null +++ b/.github/scripts/prepare_skill_eval_tasks.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Render repo-owned Harbor tasks from evals.json and setup_cpu.sh. + +SkillEvaluator 1.5.6 preserves native task healthcheck budgets. Its generated +pre_agent_setup checks instead impose a fixed 120-second installation timeout. +Native tasks also retain setup diagnostics and expose failures in Harbor's job +log, which SkillEvaluator includes in its Tier 3 report. ACES supplies the grader. +""" + +import argparse +import json +import re +import shlex +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SETUP_TIMEOUT_SECONDS = 600 + +# Harbor 0.20 omits healthcheck stdout/stderr from HealthcheckError, but records +# failed collection hooks' output in job.log, even when preflight fails before +# an agent starts. Replay the real setup status here; this hook never grades. +# Keep the excerpt short enough for SkillEvaluator's preflight log tail. +SETUP_DIAGNOSTICS_COMMAND = "bash -c " + shlex.quote( + """set -euo pipefail +cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" +cueq_status=125 +if [ -r "$cueq_root/cueq-setup.status" ]; then + read -r cueq_status < "$cueq_root/cueq-setup.status" || cueq_status=125 +fi +if [[ "$cueq_status" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then + cueq_status=$((10#$cueq_status)) +else + cueq_status=125 +fi +if [ "$cueq_status" -eq 0 ]; then + exit 0 +fi +printf 'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n' "$cueq_status" >&2 +if [ -f "$cueq_root/cueq-setup.log" ] && [ -r "$cueq_root/cueq-setup.log" ]; then + tail -n 12 "$cueq_root/cueq-setup.log" >&2 || printf 'Could not read the setup log.\\n' >&2 +else + printf 'No setup log was produced; setup may have failed before logging started.\\n' >&2 +fi +exit "$cueq_status" +""" +) + + +def task_files(skill_path): + """Return reproducible native task files, without answers or grader code.""" + skill_path = Path(skill_path) + evals = skill_path / "evals" + entries = json.loads((evals / "evals.json").read_text())["evals"] + setup = (evals / "setup_cpu.sh").read_text() + # Keep stdout/stderr in a task artifact, including when installation fails. + # Reset any previous status before starting. Save the setup exit code even + # through tee; a logging failure must also fail an otherwise successful run. + command = "bash -c " + shlex.quote( + "set -euo pipefail\n" + 'cueq_root="${CUEQ_EVAL_ROOT:-/workspace}"\n' + 'printf "125\\n" > "$cueq_root/cueq-setup.status"\n' + "set +e\n" + f"bash -lc {shlex.quote(setup)} 2>&1 " + '| tee "$cueq_root/cueq-setup.log"\n' + 'cueq_codes=("${PIPESTATUS[@]}")\n' + "set -e\n" + 'cueq_status="${cueq_codes[0]}"\n' + 'if [ "$cueq_status" -eq 0 ]; then cueq_status="${cueq_codes[1]}"; fi\n' + 'printf "%s\\n" "$cueq_status" > "$cueq_root/cueq-setup.status"\n' + 'exit "$cueq_status"\n' + ) + files = {} + seen = set() + for entry in entries: + case_id = entry["id"] + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]*", case_id): + raise ValueError(f"Invalid case ID: {case_id!r}") + if case_id in seen: + raise ValueError(f"Duplicate case ID: {case_id}") + seen.add(case_id) + case = Path(case_id) + files[case / "instruction.md"] = entry["prompt"] + "\n" + files[case / "environment/Dockerfile"] = ( + "# The evaluator rebases this image for its selected agent runtime.\n" + "FROM python:3.11-slim\n" + "WORKDIR /workspace\n" + ) + files[ + case / "task.toml" + ] = f"""# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-{case_id}" +description = "Skill evaluation task for {entry.get("expected_skill") or "none"}" + +[metadata] +entry_id = "{case_id}" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = {json.dumps(SETUP_DIAGNOSTICS_COMMAND)} +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = {json.dumps(command)} +interval_sec = 5.0 +timeout_sec = {float(SETUP_TIMEOUT_SECONDS)} +retries = 1 +""" + files[Path("dataset.toml")] = ( + "# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit.\n" + '[dataset]\nname = "nvidia/skillevaluator"\n' + 'description = "SkillEvaluator skill evaluation dataset"\n\n' + + "".join( + f'[[tasks]]\nname = "nvidia/{case_id}"\n\n' for case_id in sorted(seen) + ) + ) + return files + + +def write_tasks(skill_path, *, check=False): + """Refresh known task files; refuse stale cases rather than deleting them.""" + skill_path = Path(skill_path) + target = skill_path / "evals/harbor" + expected = task_files(skill_path) + expected_cases = {p.parts[0] for p in expected if p.name == "task.toml"} + existing_cases = {p.parent.name for p in target.glob("*/task.toml")} + if stale := existing_cases - expected_cases: + raise ValueError(f"Remove stale native cases explicitly: {sorted(stale)}") + differing = [] + for relative, content in expected.items(): + path = target / relative + if path.is_file() and path.read_text() == content: + continue + differing.append(str(relative)) + if not check: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + if check and differing: + raise ValueError(f"Regenerate native eval tasks: {', '.join(differing)}") + return target + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "skill", + type=Path, + nargs="?", + default=ROOT / "skills/cuequivariance-torch", + ) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + print(write_tasks(args.skill, check=args.check)) + + +if __name__ == "__main__": + main() diff --git a/.github/tests/challenge_checks.py b/.github/tests/challenge_checks.py new file mode 100644 index 00000000..4d931a10 --- /dev/null +++ b/.github/tests/challenge_checks.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared checks of the challenge verifier, references, and deliberate defects.""" + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +EVALS = Path(__file__).resolve().parents[1] / "evals" +spec = importlib.util.spec_from_file_location( + "cueq_challenge_verifier", EVALS / "cueq_challenges.py" +) +grader = importlib.util.module_from_spec(spec) +spec.loader.exec_module(grader) + + +def reference(backend): + return grader.load_solution(EVALS / f"reference_{backend}_challenges.py") + + +def run_mutation(backend, defect): + original = reference(backend) + if defect in ("overwrite_duplicate", "reshape_only", "merge_parity"): + + def broken(irreps, flat): + if defect == "merge_parity": + return { + ir.l: value + for ir, value in original.to_blocks(irreps, flat).items() + } + if defect == "reshape_only": + return { + ir: value.swapaxes(-1, -2).reshape(value.shape) + for ir, value in original.to_blocks(irreps, flat).items() + } + blocks, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)) + block = block.swapaxes(-1, -2) + blocks[ir] = block + offset += mul * ir.dim + return blocks + + mutant = SimpleNamespace(to_blocks=broken, from_blocks=original.from_blocks) + with pytest.raises((AssertionError, KeyError, ValueError)): + grader.check_layout(mutant, backend) + return + if backend == "jax": + import jax + import jax.numpy as jnp + + def broken(params, x, v, s, r, n): + if defect == "shared_radial": + v = jnp.broadcast_to(v[0], v.shape) + if defect == "detached_vectors": + v = jax.lax.stop_gradient(v) + output = original.apply(params, x, v, s, r, n) + if defect == "mean_reduction": + # Average repeated destinations instead of summing them. + counts = jnp.bincount(r, length=n).clip(1)[:, None] + output = output / counts + if defect == "wrong_layout": + output = original.apply(params, x[:, ::-1], v, s, r, n) + return output + + mutant = SimpleNamespace(apply=broken) + else: + import torch + + class Broken(original.GraphLayer): + def forward(self, x, v, s, r, n): + if defect == "shared_radial": + v = v[0].expand_as(v) + if defect == "detached_vectors": + v = v.detach() + output = super().forward(x, v, s, r, n) + if defect == "mean_reduction": + output = ( + output / torch.bincount(r, minlength=n).clamp_min(1)[:, None] + ) + if defect == "wrong_layout": + output = super().forward(x.flip(-1), v, s, r, n) + return output + + mutant = SimpleNamespace(GraphLayer=Broken) + with pytest.raises((AssertionError, RuntimeError)): + grader.check_graph(mutant, backend) + + +def check_routing(backend, tmp_path): + path = tmp_path / "solution.py" + path.write_text( + "def mse_loss(predictions, targets):\n" + " return ((predictions[:, 0] - targets) ** 2).mean()\n" + ) + grader.check_routing(grader.load_solution(path), backend, path) + # Shape-valid broadcasting is precisely the bug this negative case asks to fix. + path.write_text( + "def mse_loss(predictions, targets):\n" + " return ((predictions - targets) ** 2).mean()\n" + ) + with pytest.raises(AssertionError): + grader.check_routing(grader.load_solution(path), backend, path) diff --git a/.github/tests/test_core_skill_evals.py b/.github/tests/test_core_skill_evals.py new file mode 100644 index 00000000..66670b25 --- /dev/null +++ b/.github/tests/test_core_skill_evals.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical contracts behind the core skill evals; not agent benchmark scores.""" + +import dataclasses +import re + +import numpy as np +import pytest + +import cuequivariance as cue + + +@dataclasses.dataclass(frozen=True) +class Z3(cue.Irrep): + k: int + + def __post_init__(self): + if type(self.k) is not int or self.k not in (0, 1, 2): + raise ValueError("Z3 labels are integers 0, 1, 2") + + @classmethod + def regexp_pattern(cls): + return re.compile(r"([012])") + + @classmethod + def from_string(cls, string): + if string not in ("0", "1", "2"): + raise ValueError(string) + return cls(int(string)) + + def __repr__(self): + return str(self.k) + + def __lt__(self, other): + return self.k < other.k + + def __mul__(self, other): + return [Z3((self.k + other.k) % 3)] + + @classmethod + def iterator(cls): + return iter([cls(0), cls(1), cls(2)]) + + @classmethod + def clebsch_gordan(cls, a, b, c): + return np.ones((1, 1, 1, 1)) if c in a * b else np.zeros((0, 1, 1, 1)) + + @property + def dim(self): + return 1 + + def discrete_generators(self): + return np.array([[[np.exp(2j * np.pi * self.k / 3)]]]) + + def continuous_generators(self): + return np.zeros((0, 1, 1)) + + def algebra(self): + return np.zeros((0, 0, 0)) + + +def test_z3_group_and_cg_covariance(): + irreps = list(Z3.iterator()) + assert irreps[0] == Z3(0) + assert len(set(irreps)) == 3 + for invalid in (-1, 3, 1.5, True): + with pytest.raises(ValueError): + Z3(invalid) + for invalid in ("-1", "3", "01", "bad"): + with pytest.raises(ValueError): + Z3.from_string(invalid) + for a in irreps: + assert Z3.from_string(str(a)) == a + np.testing.assert_allclose(a.discrete_generators() ** 3, 1, atol=1e-14) + assert a.continuous_generators().shape == (0, 1, 1) + assert a.algebra().shape == (0, 0, 0) + for b in irreps: + assert a * b == [Z3((a.k + b.k) % 3)] + for c in irreps: + cg = cue.clebsch_gordan(a, b, c) + allowed = (a.k + b.k) % 3 == c.k + assert cg.shape == (int(allowed), 1, 1, 1) + if allowed: + np.testing.assert_allclose( + a.discrete_generators() * b.discrete_generators() * cg, + cg * c.discrete_generators(), + atol=1e-14, + ) + + +def test_full_tensor_product_dimension_and_equivariance(): + e = cue.descriptors.full_tensor_product( + cue.Irreps("SO3", "2x0 + 1 + 2"), cue.Irreps("SO3", "0 + 1") + ) + assert len(e.inputs) == 2 + assert e.outputs[0].irreps.regroup() == cue.Irreps("SO3", "3x0+5x1+3x2+3") + assert e.outputs[0].dim == 40 + rng = np.random.default_rng(12) + x = [rng.normal(size=(4, rep.dim)) for rep in e.inputs] + rotation = np.array([0.31, -0.19, 0.43]) + matrices = [rep.exp_map(rotation, []) for rep in (*e.inputs, *e.outputs)] + np.testing.assert_allclose( + e(*(v @ d.T for v, d in zip(x, matrices)))[0], + e(*x)[0] @ matrices[-1].T, + atol=1e-12, + ) + + +def test_ir_dict_operand_contract_and_dense_reference(): + a = cue.Irreps("SO3", "2x0+2x1+2x2") + b = cue.Irreps("SO3", "0+1+2+3") + keep = [cue.SO3(degree) for degree in range(3)] + split = cue.descriptors.channelwise_tensor_product_ir_dict(a, b, keep) + # The ir_dict descriptor simplifies outputs automatically. Request the + # same representation ordering in the dense reference as well. + dense = cue.descriptors.channelwise_tensor_product( + a, b, keep, simplify_irreps3=True + ) + rng = np.random.default_rng(14) + blocks = [ + [rng.normal(size=(3, mul, ir.dim)) for mul, ir in group] + for group in split.input_irreps + ] + operands = [[v.swapaxes(-1, -2).reshape(3, -1) for v in group] for group in blocks] + actual = split.polynomial(*(v for group in operands for v in group)) + expected = dense(*(np.concatenate(group, axis=-1) for group in operands))[0] + assert split.output_irreps[0] == dense.outputs[0].irreps + np.testing.assert_allclose(np.concatenate(actual, axis=-1), expected, atol=1e-12) + + +def test_layout_conversion_and_linear_equivariance(): + irreps = cue.Irreps("SO3", "2x0+3x1") + x = np.arange(2 * irreps.dim, dtype=float).reshape(2, -1) / 7 + + def convert(data, source_layout): + pieces, offset = [], 0 + for mul, ir in irreps: + shape = (mul, ir.dim) if source_layout == cue.mul_ir else (ir.dim, mul) + block = data[..., offset : offset + mul * ir.dim] + pieces.append( + block.reshape(*data.shape[:-1], *shape) + .swapaxes(-1, -2) + .reshape(*data.shape[:-1], -1) + ) + offset += mul * ir.dim + return np.concatenate(pieces, axis=-1) + + flat = convert(x, cue.mul_ir) + np.testing.assert_array_equal(convert(flat, cue.ir_mul), x) + assert not np.array_equal(flat, x) + e = cue.descriptors.linear(irreps, irreps) + w = np.random.default_rng(9).normal(size=e.inputs[0].dim) + rotation = np.array([0.2, 0.3, -0.4]) + d = e.inputs[1].exp_map(rotation, []) + np.testing.assert_allclose(e(w, flat @ d.T)[0], e(w, flat)[0] @ d.T, atol=1e-12) diff --git a/.github/tests/test_jax_skill_challenges.py b/.github/tests/test_jax_skill_challenges.py new file mode 100644 index 00000000..4e3d6b14 --- /dev/null +++ b/.github/tests/test_jax_skill_challenges.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical acceptance and defect rejection for challenge-v1 JAX submissions.""" + +import pytest +from challenge_checks import check_routing, grader, reference, run_mutation + + +def test_layout_reference(): + grader.check_layout(reference("jax"), "jax") + + +def test_graph_reference(): + grader.check_graph(reference("jax"), "jax") + + +@pytest.mark.parametrize( + "defect", + [ + "overwrite_duplicate", + "reshape_only", + "merge_parity", + "shared_radial", + "detached_vectors", + "mean_reduction", + "wrong_layout", + ], +) +def test_rejects_wrong_solution(defect): + run_mutation("jax", defect) + + +def test_nearby_negative_and_broadcast_defect(tmp_path): + check_routing("jax", tmp_path) diff --git a/.github/tests/test_jax_skill_evals.py b/.github/tests/test_jax_skill_evals.py new file mode 100644 index 00000000..b53c8f1c --- /dev/null +++ b/.github/tests/test_jax_skill_evals.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU API and numerical contracts for the JAX skill evaluation prompts.""" + +import cuequivariance_jax.ir_dict as ir_dict +import cuequivariance_jax.nnx as cue_nnx +import jax +import jax.numpy as jnp +import numpy as np +from flax import nnx + +import cuequivariance as cue +import cuequivariance_jax as cuex + +jax.config.update("jax_enable_x64", True) + + +def test_broadcast_jit_and_finite_difference(): + e = cue.descriptors.channelwise_tensor_product( + cue.Irreps("SO3", "2x0+2x1"), + cue.Irreps("SO3", "0+1"), + [cue.SO3(0), cue.SO3(1)], + ) + rng = np.random.default_rng(21) + w = rng.normal(size=e.inputs[0].dim) + x = rng.normal(size=(2, 1, e.inputs[1].dim)) + y = rng.normal(size=(1, 3, e.inputs[2].dim)) + + def run(weights): + return cuex.segmented_polynomial( + e.polynomial, + [weights, jnp.asarray(x), jnp.asarray(y)], + [jax.ShapeDtypeStruct((2, 3, e.outputs[0].dim), jnp.float64)], + method="naive", + )[0] + + expected = e(w, x, y)[0] + np.testing.assert_allclose(run(jnp.asarray(w)), expected, atol=1e-12) + np.testing.assert_allclose(jax.jit(run)(jnp.asarray(w)), expected, atol=1e-12) + grad = jax.grad(lambda v: jnp.sum(run(v) ** 2))(jnp.asarray(w)) + eps = 1e-5 + wp, wm = w.copy(), w.copy() + wp[0] += eps + wm[0] -= eps + finite_diff = (np.sum(e(wp, x, y)[0] ** 2) - np.sum(e(wm, x, y)[0] ** 2)) / ( + 2 * eps + ) + assert abs(grad[0]) > 1e-5 + np.testing.assert_allclose(grad[0], finite_diff, rtol=1e-7, atol=1e-7) + + +def test_dict_layout_conversion_is_not_a_flat_reshape(): + irreps = cue.Irreps("SO3", "2x0+3x1") + flat = jnp.arange(2 * irreps.dim).reshape(2, -1) + blocks = ir_dict.flat_to_dict(irreps, flat, layout="ir_mul") + ir_dict.assert_mul_ir_dict(irreps, blocks) + np.testing.assert_array_equal( + blocks[cue.SO3(1)], np.asarray(flat[:, 2:]).reshape(2, 3, 3).swapaxes(-1, -2) + ) + mul_ir = ir_dict.dict_to_flat(irreps, blocks) + assert not np.array_equal(mul_ir, flat) + restored = jnp.concatenate( + [blocks[ir].swapaxes(-1, -2).reshape(2, -1) for _, ir in irreps], axis=-1 + ) + np.testing.assert_array_equal(restored, flat) + + +def test_nnx_linear_normalization_missing_irrep_and_equivariance(): + a, b = cue.Irreps("SO3", "2x0+3x1"), cue.Irreps("SO3", "4x0+2x1+2") + layer = cue_nnx.IrrepsLinear(a, b, dtype=jnp.float64, rngs=nnx.Rngs(22)) + x = { + ir: jnp.asarray(np.random.default_rng(23 + ir.l).normal(size=(2, mul, ir.dim))) + for mul, ir in a + } + y = layer(x) + ir_dict.assert_mul_ir_dict(b, y) + np.testing.assert_array_equal(y[cue.SO3(2)], 0) + for mul, ir in a: + expected = np.einsum( + "uv,bui->bvi", np.asarray(layer.w[str(ir)][...]), x[ir] + ) / np.sqrt(mul) + np.testing.assert_allclose(y[ir], expected, atol=1e-12) + rotation = np.array([0.2, -0.3, 0.4]) + rotated = layer({ir: v @ ir.exp_map(rotation, []).T for ir, v in x.items()}) + for ir in y: + np.testing.assert_allclose( + rotated[ir], y[ir] @ ir.exp_map(rotation, []).T, atol=1e-12 + ) + + +def test_ir_dict_naive_matches_numpy(): + e = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("SO3", "2x0+2x1+2x2"), + cue.Irreps("SO3", "0+1+2+3"), + [cue.SO3(degree) for degree in range(3)], + ) + rng = np.random.default_rng(24) + inputs = [ + rng.normal(size=(2, mul, ir.dim)).swapaxes(-1, -2).reshape(2, -1) + for group in e.input_irreps + for mul, ir in group + ] + result = cuex.segmented_polynomial( + e.polynomial, + list(map(jnp.asarray, inputs)), + [ + jax.ShapeDtypeStruct((2, op.size), jnp.float64) + for op in e.polynomial.outputs + ], + method="naive", + ) + expected = e.polynomial(*inputs) + for actual, reference in zip(result, expected): + np.testing.assert_allclose(actual, reference, atol=1e-12) + + +def test_nnx_spherical_harmonics_cpu(): + layer = cue_nnx.SphericalHarmonics(max_degree=2) + output = layer(jnp.array([[1.0, 2.0, 3.0], [-0.4, 0.8, 0.2]])) + assert {ir.l: value.shape for ir, value in output.items()} == { + 0: (2, 1, 1), + 1: (2, 1, 3), + 2: (2, 1, 5), + } + assert all(np.isfinite(value).all() for value in output.values()) diff --git a/.github/tests/test_jax_skill_examples.py b/.github/tests/test_jax_skill_examples.py new file mode 100644 index 00000000..9336bfe8 --- /dev/null +++ b/.github/tests/test_jax_skill_examples.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Execute the published CPU recipes and exercise their layout helpers.""" + +import importlib +import re +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +import cuequivariance as cue + +SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-jax/SKILL.md" +EXAMPLES = [ + compile(code, f"{SKILL}:example-{index}", "exec") + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", SKILL.read_text(), re.S) + ) +] + + +def run_example(index): + namespace = {} + exec(EXAMPLES[1], namespace) # Common imports and precision configuration. + exec(EXAMPLES[index], namespace) + return namespace + + +@pytest.mark.parametrize("index", range(2, len(EXAMPLES))) +def test_published_cpu_example(index): + # Run each example independently: accidental dependencies on earlier local + # variables must fail here, just as they would for someone copying a recipe. + run_example(index) + + +@pytest.mark.parametrize( + "failed_module,error", + [ + ("jax", ImportError("JAX cannot be imported")), + ("cuequivariance", OSError("shared library cannot be loaded")), + ("cuequivariance_jax", ModuleNotFoundError("frontend is absent")), + ], +) +def test_environment_probe_preserves_partial_results(monkeypatch, failed_module, error): + modules = { + name: importlib.import_module(name) + for name in ("jax", "cuequivariance", "cuequivariance_jax") + } + + def import_module(name): + if name == failed_module: + raise error + return modules[name] + + monkeypatch.setattr(importlib, "import_module", import_module) + namespace = {} + exec(EXAMPLES[0], namespace) + assert namespace["available"] == { + name: module for name, module in modules.items() if name != failed_module + } + assert namespace["import_errors"] == { + failed_module: f"{type(error).__name__}: {error}" + } + + +@pytest.mark.parametrize("batch", [(), (2, 3)]) +def test_layout_helpers_with_duplicate_irreps(batch): + recipe = run_example(3) + irreps = cue.Irreps("SO3", "2x1 + 0 + 1 + 2x2") + flat = jnp.arange(int(np.prod(batch)) * irreps.dim).reshape(batch + (irreps.dim,)) + regrouped, blocks = recipe["ir_mul_to_dict"](irreps, flat) + assert regrouped == irreps.regroup() + # Combine separated l=1 blocks along the channel axis, keeping every value. + expected_vectors = np.concatenate( + [ + np.asarray(flat[..., :6]).reshape(batch + (3, 2)), + np.asarray(flat[..., 7:10]).reshape(batch + (3, 1)), + ], + axis=-1, + ).swapaxes(-1, -2) + np.testing.assert_array_equal(blocks[cue.SO3(0)], flat[..., 6:7, None]) + np.testing.assert_array_equal(blocks[cue.SO3(1)], expected_vectors) + np.testing.assert_array_equal( + blocks[cue.SO3(2)], + np.asarray(flat[..., 10:]).reshape(batch + (5, 2)).swapaxes(-1, -2), + ) + expected_flat = np.concatenate( + [ + flat[..., 6:7], + expected_vectors.swapaxes(-1, -2).reshape(batch + (9,)), + flat[..., 10:], + ], + axis=-1, + ) + np.testing.assert_array_equal( + recipe["dict_to_ir_mul_flat"](regrouped, blocks), expected_flat + ) + + +def test_layout_regroup_keeps_opposite_parities_separate(): + recipe = run_example(3) + irreps = cue.Irreps("O3", "1o + 0e + 2x1e + 2x1o") + flat = jnp.arange(2 * irreps.dim).reshape(2, irreps.dim) + regrouped, blocks = recipe["ir_mul_to_dict"](irreps, flat) + assert regrouped == irreps.regroup() + assert set(blocks) == {cue.O3(0, 1), cue.O3(1, -1), cue.O3(1, 1)} + odd_vectors = np.concatenate( + [ + np.asarray(flat[:, :3]).reshape(2, 3, 1), + np.asarray(flat[:, 10:]).reshape(2, 3, 2), + ], + axis=-1, + ).swapaxes(-1, -2) + np.testing.assert_array_equal(blocks[cue.O3(1, -1)], odd_vectors) + np.testing.assert_array_equal( + blocks[cue.O3(1, 1)], + np.asarray(flat[:, 4:10]).reshape(2, 3, 2).swapaxes(-1, -2), + ) + np.testing.assert_array_equal(blocks[cue.O3(0, 1)], flat[:, 3:4, None]) + + +@pytest.mark.parametrize("batch_shapes", [((), (), ()), ((), (2, 1, 1), (1, 3, 2))]) +def test_split_recipe_with_parity_higher_degrees_and_jit(batch_shapes): + split_naive = run_example(5)["split_naive"] + desc = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("O3", "2x0e + 2x1o + 2x2e"), + cue.Irreps("O3", "0e + 1o + 2e"), + [cue.O3(degree, (-1) ** degree) for degree in range(4)], + ) + rng = np.random.default_rng(41) + groups = [ + {ir: jnp.asarray(rng.normal(size=batch + (mul, ir.dim))) for mul, ir in irreps} + for irreps, batch in zip(desc.input_irreps, batch_shapes) + ] + operands = [ + np.asarray(group[ir]).swapaxes(-1, -2).reshape(batch + (mul * ir.dim,)) + for group, irreps, batch in zip(groups, desc.input_irreps, batch_shapes) + for mul, ir in irreps + ] + expected = desc.polynomial(*operands) + batch = np.broadcast_shapes(*batch_shapes) + for compute in ( + lambda g: split_naive(desc, g), + jax.jit(lambda g: split_naive(desc, g)), + ): + (result,) = compute(groups) + assert set(result) == {ir for _, ir in desc.output_irreps[0]} + for (mul, ir), reference in zip(desc.output_irreps[0], expected): + assert result[ir].shape == batch + (mul, ir.dim) + np.testing.assert_allclose( + result[ir].swapaxes(-1, -2).reshape(batch + (mul * ir.dim,)), + reference, + rtol=1e-12, + atol=1e-12, + ) diff --git a/.github/tests/test_skill_challenge_packaging.py b/.github/tests/test_skill_challenge_packaging.py new file mode 100644 index 00000000..a8b753fd --- /dev/null +++ b/.github/tests/test_skill_challenge_packaging.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Preserve regression inputs, isolate reference answers, and bound verification.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest +import tomllib +import yaml +from challenge_checks import EVALS, grader + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "prepare_skill_challenges", ROOT / ".github/scripts/prepare_skill_challenges.py" +) +preparer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preparer) + + +def test_pilot_keeps_skill_and_environment_fixed_and_excludes_answers(tmp_path): + original = {} + for backend in ("jax", "torch"): + source = ROOT / "skills" / f"cuequivariance-{backend}" + for relative in ("SKILL.md", "evals/config.yml", "evals/evals.json"): + original[source / relative] = (source / relative).read_bytes() + manifest = preparer.prepare(tmp_path / "pilot", ("jax", "torch")) + assert manifest["attempts_per_case_per_arm"] == 3 + assert manifest["stop_on_pass"] is False + for backend in ("jax", "torch"): + name = f"cuequivariance-{backend}" + record = manifest["skills"][name] + source, staged = ROOT / "skills" / name, Path(record["path"]) + assert (staged / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes() + config = yaml.safe_load((staged / "evals/config.yml").read_text()) + original_config = yaml.safe_load((source / "evals/config.yml").read_text()) + assert config["harbor"] == original_config["harbor"] + assert config["grading"]["mode"] == "aces_plus_custom" + entries = json.loads((staged / "evals/evals.json").read_text())["evals"] + assert len(entries) == 3 + assert len({e["id"] for e in entries}) == len(entries) + assert sum(e["expected_skill"] == name for e in entries) == 2 + assert sum(e["expected_skill"] is None for e in entries) == 1 + assert all("v1" in e["id"] for e in entries) + assert not list(staged.rglob("reference_*")) + assert preparer.digest(staged / "evals/grader.py") == record["grader_digest"] + assert record["command"][record["command"].index("--n-attempts") + 1] == "3" + assert "--skip-baseline" not in record["command"] + for agent, model in manifest["agent_models"].items(): + assert f"{agent}={model}" in record["command"] + if backend == "torch": + assert (staged / "evals/setup_cpu.sh").read_bytes() == ( + source / "evals/setup_cpu.sh" + ).read_bytes() + tasks = staged / "evals/harbor" + assert {p.name for p in tasks.iterdir() if p.is_dir()} == { + e["id"] for e in entries + } + dataset = tomllib.loads((tasks / "dataset.toml").read_text()) + assert {t["name"] for t in dataset["tasks"]} == { + f"nvidia/{e['id']}" for e in entries + } + for entry in entries: + task = tasks / entry["id"] + assert (task / "instruction.md").read_text() == entry["prompt"] + "\n" + data = tomllib.loads((task / "task.toml").read_text()) + assert data["environment"]["healthcheck"]["timeout_sec"] == 600 + assert record["setup_digest"] == preparer.digest( + staged / "evals/setup_cpu.sh" + ) + assert record["native_task_digests"] + for path, content in original.items(): + assert path.read_bytes() == content + with pytest.raises(FileExistsError): + preparer.prepare(tmp_path / "pilot", ("jax",)) + + +def test_missing_submission_and_timeout_are_failures(tmp_path): + missing = grader.evaluate( + "jax", "layout", tmp_path / "missing.py", python=sys.executable + ) + assert not missing["passed"] and "FileNotFoundError" in missing["error"] + hanging = tmp_path / "solution.py" + hanging.write_text("while True: pass\n") + timed_out = grader.evaluate("jax", "layout", hanging, timeout=0.2) + assert not timed_out["passed"] and "exceeded" in timed_out["error"] + + +def test_submission_import_supports_sibling_helpers_and_dataclasses( + tmp_path, monkeypatch +): + monkeypatch.setattr(sys, "path", sys.path.copy()) + (tmp_path / "challenge_import_helper.py").write_text("VALUE = 7\n") + solution = tmp_path / "solution.py" + solution.write_text( + "from __future__ import annotations\n" + "from dataclasses import dataclass\n" + "from challenge_import_helper import VALUE\n" + "@dataclass\n" + "class Result:\n" + " value: int = VALUE\n" + ) + assert grader.load_solution(solution).Result().value == 7 + + +@pytest.mark.parametrize("leak", ("evals/evals.json", "grader.py", "reference_jax.py")) +def test_export_rejects_evaluation_material_in_agent_bundle(tmp_path, leak): + task = tmp_path / "task" + path = task / "environment" / leak + path.parent.mkdir(parents=True) + path.write_text("must not be visible to the agent") + with pytest.raises(AssertionError): + preparer.verify_environment(task, "jax", False) + + +def test_harbor_dispatch_writes_a_real_failure_reward(tmp_path, monkeypatch): + entry = tmp_path / "entry.json" + entry.write_text(json.dumps({"id": "jax-challenge-layout-v1"})) + reward = tmp_path / "reward.json" + monkeypatch.setenv("HARBOR_ENTRY_JSON", str(entry)) + monkeypatch.setenv("HARBOR_REWARD_JSON", str(reward)) + monkeypatch.setattr(sys, "argv", [str(EVALS / "cueq_challenges.py")]) + observed = {} + + def failed(backend, case, solution, **kwargs): + observed.update(backend=backend, case=case, solution=str(solution), **kwargs) + return {"passed": False, "error": "numerical mismatch"} + + monkeypatch.setattr(grader, "evaluate", failed) + grader.main() + assert observed["backend"] == "jax" and observed["case"] == "layout" + assert observed["solution"] == "/workspace/solution.py" + assert observed["python"] == "/workspace/.venv/bin/python" + result = json.loads(reward.read_text()) + assert result["custom_metrics"]["numerical_success"] == 0 + assert "overall" not in result and "goal_accuracy" not in result diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py new file mode 100644 index 00000000..1e9bbef2 --- /dev/null +++ b/.github/tests/test_skill_eval_setup.py @@ -0,0 +1,359 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pre-agent environment failures must stop evaluation without replacing data.""" + +import json +import os +import runpy +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +import tomllib +import yaml + +ROOT = Path(__file__).resolve().parents[2] +BUILDER = runpy.run_path(str(ROOT / ".github/scripts/prepare_skill_eval_tasks.py")) + + +def setup_command(frontend): + config_path = ( + ROOT / "skills" / f"cuequivariance-{frontend}" / "evals" / "config.yml" + ) + config = yaml.safe_load(config_path.read_text()) + if config["harbor"].get("task_source") == "native_harbor": + return (config_path.parent / "setup_cpu.sh").read_text() + return "set -euo pipefail\n" + "\n".join(config["harbor"]["pre_agent_setup"]) + + +@pytest.fixture +def setup_sandbox(tmp_path): + """Inject installer/smoke failures without downloading dependencies.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + python = bin_dir / "python3" + python.write_text( + f"#!{sys.executable}\n" + """ +import json +import os +import sys +from pathlib import Path + +root = Path(os.environ["CUEQ_EVAL_ROOT"]) +args = sys.argv[1:] +if args[:2] == ["-m", "venv"]: + stage = "venv" + target = Path(args[-1]) + (target / "bin").mkdir() + (target / "bin/python").symlink_to(Path(__file__).resolve()) +elif args[:2] == ["-m", "pip"]: + stage = "install" +else: + stage = "smoke" + sys.stdin.read() + +with (root / "calls.jsonl").open("a") as log: + log.write(json.dumps({ + "stage": stage, "args": args, "published": (root / ".venv").is_symlink() + }) + "\\n") +failure = os.environ.get("CUEQ_FAIL_STAGE") +if failure == stage or ( + failure == "dependencies" and stage == "install" and "--no-deps" not in args +): + print("injected " + failure + " failure", file=sys.stderr) + sys.exit(23) +if stage == "smoke": + print("CPU environment ready") +""" + ) + python.chmod(0o755) + return { + **os.environ, + "CUEQ_EVAL_ROOT": str(tmp_path), + "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"], + } + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +@pytest.mark.parametrize("failure_stage", ["venv", "install", "smoke"]) +def test_failed_setup_is_not_published_and_can_be_retried( + tmp_path, setup_sandbox, frontend, failure_stage +): + def run(**extra): + return subprocess.run( + ["bash", "-c", setup_command(frontend)], + env={**setup_sandbox, **extra}, + capture_output=True, + text=True, + timeout=30, + ) + + failed = run(CUEQ_FAIL_STAGE=failure_stage) + assert failed.returncode == 23, failed.stderr + assert f"injected {failure_stage} failure" in failed.stderr + assert not os.path.lexists(tmp_path / ".venv") + (partial,) = tmp_path.glob(".cueq-env.*") + sentinel = partial / "keep.txt" + sentinel.write_text("partial environment") + + retried = run(CUEQ_FAIL_STAGE="") + assert retried.returncode == 0, retried.stderr + assert "CPU environment ready" in retried.stdout + published = tmp_path / ".venv" + assert published.is_symlink() + assert published.resolve() != partial + assert sentinel.read_text() == "partial environment" + calls = [ + json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines() + ] + assert not any(call["published"] for call in calls) + + # Reuse checks the environment but does not reinstall or replace it. + target = published.resolve() + reused = run(CUEQ_FAIL_STAGE="install") + assert reused.returncode == 0, reused.stderr + assert published.resolve() == target + after = [ + json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines() + ] + assert [call["stage"] for call in after[len(calls) :]] == ["smoke"] + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +def test_pre_agent_setup_preserves_dangling_environment_link( + tmp_path, setup_sandbox, frontend +): + env_dir = tmp_path / ".venv" + target = tmp_path / "missing-environment" + env_dir.symlink_to(target) + result = subprocess.run( + ["bash", "-c", setup_command(frontend)], + env=setup_sandbox, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert env_dir.is_symlink() and env_dir.readlink() == target + assert not list(tmp_path.glob(".cueq-env.*")) + assert not (tmp_path / "calls.jsonl").exists() + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +def test_pre_agent_setup_preserves_existing_incompatible_environment( + tmp_path, frontend +): + env_dir = tmp_path / ".venv" + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(env_dir)], + check=True, + timeout=30, + ) + sentinel = env_dir / "keep.txt" + sentinel.write_text("existing environment") + original_config = (env_dir / "pyvenv.cfg").read_bytes() + result = subprocess.run( + ["bash", "-c", setup_command(frontend)], + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert "ModuleNotFoundError" in result.stderr + assert "CPU environment ready" not in result.stdout + assert sentinel.read_text() == "existing environment" + assert (env_dir / "pyvenv.cfg").read_bytes() == original_config + assert not list(tmp_path.glob(".cueq-env.*")) + + +@pytest.mark.parametrize( + ("failure", "reported_stage"), + [ + ("venv", "create_environment"), + ("install", "install_cpu_torch"), + ("dependencies", "install_dependencies"), + ("smoke", "smoke_check"), + ], +) +def test_torch_setup_identifies_the_failed_stage( + tmp_path, setup_sandbox, failure, reported_stage +): + result = subprocess.run( + ["bash", "-c", setup_command("torch")], + env={**setup_sandbox, "CUEQ_FAIL_STAGE": failure}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 23 + assert f"injected {failure} failure" in result.stderr + assert result.stderr.rstrip().endswith( + f"CUEQ_SETUP_FAILED stage={reported_stage} exit_code=23" + ) + assert not os.path.lexists(tmp_path / ".venv") + + +@pytest.mark.parametrize("certificate", [None, "/custom/trusted-ca.pem"]) +def test_cpu_torch_uses_alternate_index_and_preserves_certificate_verification( + tmp_path, setup_sandbox, certificate +): + env = dict(setup_sandbox) + env.pop("PIP_CERT", None) + if certificate is not None: + env["PIP_CERT"] = certificate + result = subprocess.run( + ["bash", "-c", setup_command("torch")], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + installs = [ + call["args"] + for line in (tmp_path / "calls.jsonl").read_text().splitlines() + if (call := json.loads(line))["stage"] == "install" + ] + wheel, dependencies = installs + assert wheel[wheel.index("--index-url") + 1] == ( + "https://download-r2.pytorch.org/whl/cpu" + ) + assert "torch==2.14.1+cpu" in wheel + assert "--no-deps" in wheel and "--no-index" not in wheel + for args in installs: + assert args[args.index("--cert") + 1] == ( + certificate or "/etc/ssl/certs/ca-certificates.crt" + ) + assert "--trusted-host" not in args + assert "torch==2.14.1+cpu" in dependencies + assert "cuequivariance-torch==0.13.0" in dependencies + assert "--no-index" not in dependencies and "--no-deps" not in dependencies + assert "--index-url" not in dependencies + assert (tmp_path / ".venv").is_symlink() + + +def test_native_torch_tasks_keep_regression_prompts_and_setup_budget(): + skill = ROOT / "skills/cuequivariance-torch" + BUILDER["write_tasks"](skill, check=True) + config = yaml.safe_load((skill / "evals/config.yml").read_text()) + assert config["harbor"]["task_source"] == "native_harbor" + # Otherwise 1.5.6 rejects the existing native healthcheck as a collision. + assert "pre_agent_setup" not in config["harbor"] + assert config["grading"]["mode"] == "aces_default" + entries = json.loads((skill / "evals/evals.json").read_text())["evals"] + tasks = skill / "evals/harbor" + assert {p.name for p in tasks.iterdir() if p.is_dir()} == {e["id"] for e in entries} + manifest = tomllib.loads((tasks / "dataset.toml").read_text()) + assert [t["name"] for t in manifest["tasks"]] == [ + f"nvidia/{case_id}" for case_id in sorted(e["id"] for e in entries) + ] + healthchecks = [] + for entry in entries: + task = tasks / entry["id"] + assert (task / "instruction.md").read_text() == entry["prompt"] + "\n" + data = tomllib.loads((task / "task.toml").read_text()) + assert data["metadata"]["entry_id"] == entry["id"] + assert data["agent"]["timeout_sec"] == 300 + assert data["verifier"]["timeout_sec"] == 180 + healthchecks.append(data["environment"]["healthcheck"]) + assert healthchecks[-1]["timeout_sec"] == 600 + assert healthchecks[-1]["retries"] == 1 + assert all(h == healthchecks[0] for h in healthchecks) + + +@pytest.mark.parametrize("status", [0, 23]) +@pytest.mark.parametrize("logging_fails", [False, True]) +def test_native_setup_retains_output_and_exit_status(tmp_path, status, logging_fails): + evals = tmp_path / "evals" + evals.mkdir() + (evals / "evals.json").write_text( + json.dumps({"evals": [{"id": "smoke", "prompt": "Exercise setup"}]}) + ) + (evals / "setup_cpu.sh").write_text( + f"printf 'installer output\\n'\nprintf 'installer diagnostic\\n' >&2\nexit {status}\n" + ) + files = BUILDER["task_files"](tmp_path) + data = tomllib.loads(files[Path("smoke/task.toml")]) + command = data["environment"]["healthcheck"]["command"] + # A previous successful attempt must not hide the next attempt's failure. + status_file = tmp_path / "cueq-setup.status" + status_file.write_text("0\n") + log_file = tmp_path / "cueq-setup.log" + if logging_fails: + log_file.mkdir() + env = {**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)} + result = subprocess.run( + shlex.split(command), + env=env, + capture_output=True, + text=True, + timeout=10, + ) + expected_status = status or (1 if logging_fails else 0) + assert result.returncode == expected_status + assert "installer output" in result.stdout + assert "installer diagnostic" in result.stdout + if not logging_fails: + assert log_file.read_text() == result.stdout + assert status_file.read_text() == f"{expected_status}\n" + + (hook,) = data["verifier"]["collect"] + diagnostics = subprocess.run( + shlex.split(hook["command"]), + env=env, + capture_output=True, + text=True, + timeout=10, + ) + # Harbor logs the stdout/stderr of a nonzero collection hook even when the + # healthcheck failed before agent execution. Success produces no warning. + assert diagnostics.returncode == expected_status + if expected_status: + assert ( + f"CUEQ_SETUP_DIAGNOSTICS exit_code={expected_status}" in diagnostics.stderr + ) + if logging_fails: + assert "No setup log was produced" in diagnostics.stderr + else: + assert "installer diagnostic" in diagnostics.stderr + else: + assert not diagnostics.stdout and not diagnostics.stderr + + +@pytest.mark.parametrize("saved_status", [None, "125\n", "invalid\n"]) +def test_setup_diagnostics_handle_missing_or_incomplete_status(tmp_path, saved_status): + if saved_status is not None: + (tmp_path / "cueq-setup.status").write_text(saved_status) + result = subprocess.run( + shlex.split(BUILDER["SETUP_DIAGNOSTICS_COMMAND"]), + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 125 + assert "CUEQ_SETUP_DIAGNOSTICS exit_code=125" in result.stderr + assert "No setup log was produced" in result.stderr + + +def test_setup_diagnostics_keep_the_error_at_the_end_of_long_logs(tmp_path): + (tmp_path / "cueq-setup.status").write_text("23\n") + (tmp_path / "cueq-setup.log").write_text( + "irrelevant download progress\n" * 1000 + + "ERROR: injected dependency failure\n" + + "CUEQ_SETUP_FAILED stage=install_dependencies exit_code=23\n" + ) + result = subprocess.run( + shlex.split(BUILDER["SETUP_DIAGNOSTICS_COMMAND"]), + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 23 + assert "ERROR: injected dependency failure" in result.stderr + assert "CUEQ_SETUP_FAILED stage=install_dependencies exit_code=23" in result.stderr + assert len(result.stderr.splitlines()) <= 13 diff --git a/.github/tests/test_skill_execution.py b/.github/tests/test_skill_execution.py new file mode 100644 index 00000000..8389f314 --- /dev/null +++ b/.github/tests/test_skill_execution.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the documented completion commands, including failures and live runs.""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(params=["cuequivariance-jax", "cuequivariance-torch"]) +def completion_command(request): + skill = Path(__file__).resolve().parents[2] / "skills" / request.param / "SKILL.md" + section = skill.read_text().split("### Complete and inspect a run\n", 1)[1] + return re.search(r"```bash\n(.*?)\n```", section, re.S).group(1) + + +def run_environment(tmp_path): + return {**os.environ, "TMPDIR": str(tmp_path), "cueq_python": sys.executable} + + +def test_completion_records_success_and_failure_without_reusing_logs( + completion_command, tmp_path +): + project = tmp_path / "project with spaces" + project.mkdir() + sentinel = tmp_path / ".venv" / "keep.txt" + sentinel.parent.mkdir() + sentinel.write_text("existing environment") + records = [] + for status in (0, 7): + (project / "check.py").write_text( + f"print('observed result for exit {status}')\nraise SystemExit({status})\n" + ) + result = subprocess.run( + ["bash", "-c", "set -e\n" + completion_command], + cwd=project, + env=run_environment(tmp_path), + capture_output=True, + text=True, + timeout=10, + ) + assert (result.returncode == 0) == (status == 0), result + record = Path(result.stdout.splitlines()[0].removeprefix("run directory: ")) + assert (record / "exit-code.txt").read_text().strip() == str(status) + assert ( + record / "output.txt" + ).read_text() == f"observed result for exit {status}\n" + assert f"observed result for exit {status}" in result.stdout + assert f"exit code: {status}" in result.stdout + records.append(record) + assert records[0] != records[1] + assert (records[0] / "exit-code.txt").read_text().strip() == "0" + assert sentinel.read_text() == "existing environment" + + +def test_live_run_has_no_completion_record(completion_command, tmp_path): + (tmp_path / "check.py").write_text( + "import time\nfrom pathlib import Path\n" + "print('check started', flush=True)\n" + "while not Path('release').exists(): time.sleep(0.01)\n" + "print('check completed')\n" + ) + with subprocess.Popen( + ["bash", "-c", "set -e\n" + completion_command], + cwd=tmp_path, + env=run_environment(tmp_path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + try: + first_line = process.stdout.readline().strip() + record = Path(first_line.removeprefix("run directory: ")) + assert record.is_dir(), first_line + assert process.poll() is None + assert not (record / "exit-code.txt").exists() + finally: + (tmp_path / "release").touch() + output, errors = process.communicate(timeout=10) + assert process.returncode == 0, errors + assert (record / "exit-code.txt").read_text().strip() == "0" + assert "check completed" in output + assert "exit code: 0" in output + + +def test_missing_interpreter_is_a_failed_run(completion_command, tmp_path): + env = run_environment(tmp_path) + env["cueq_python"] = str(tmp_path / "missing-python") + result = subprocess.run( + ["bash", "-c", "set -e\n" + completion_command], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=10, + ) + record = Path(result.stdout.splitlines()[0].removeprefix("run directory: ")) + assert result.returncode != 0 + assert (record / "exit-code.txt").read_text().strip() == "127" + assert "missing-python" in (record / "output.txt").read_text() diff --git a/.github/tests/test_torch_skill_challenges.py b/.github/tests/test_torch_skill_challenges.py new file mode 100644 index 00000000..bb92011b --- /dev/null +++ b/.github/tests/test_torch_skill_challenges.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical acceptance and defect rejection for challenge-v1 Torch submissions.""" + +import pytest +from challenge_checks import check_routing, grader, reference, run_mutation + + +def test_layout_reference(): + grader.check_layout(reference("torch"), "torch") + + +def test_graph_reference(): + grader.check_graph(reference("torch"), "torch") + + +@pytest.mark.parametrize( + "defect", + [ + "overwrite_duplicate", + "reshape_only", + "merge_parity", + "shared_radial", + "detached_vectors", + "mean_reduction", + "wrong_layout", + ], +) +def test_rejects_wrong_solution(defect): + run_mutation("torch", defect) + + +def test_nearby_negative_and_broadcast_defect(tmp_path): + check_routing("torch", tmp_path) diff --git a/.github/tests/test_torch_skill_evals.py b/.github/tests/test_torch_skill_evals.py new file mode 100644 index 00000000..acb80289 --- /dev/null +++ b/.github/tests/test_torch_skill_evals.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU reference and autograd contracts for the PyTorch skill evals.""" + +import numpy as np +import torch + +import cuequivariance as cue +import cuequivariance_torch as cuet + + +def test_spherical_harmonics_numpy_and_gradcheck(): + torch.manual_seed(30) + x = torch.randn(2, 3, dtype=torch.float64, requires_grad=True) + module = cuet.SphericalHarmonics([0, 1, 2], normalize=False, method="naive") + e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + y = module(x) + assert y.shape == (2, 9) + np.testing.assert_allclose(y.detach().numpy(), e(x.detach().numpy())[0], atol=1e-12) + assert torch.autograd.gradcheck(module, (x,)) + + +def transpose_blocks(data, irreps, source_layout): + pieces, offset = [], 0 + for mul, ir in irreps: + shape = (mul, ir.dim) if source_layout == cue.mul_ir else (ir.dim, mul) + block = data[..., offset : offset + mul * ir.dim] + pieces.append( + block.reshape(*data.shape[:-1], *shape) + .transpose(-1, -2) + .reshape(*data.shape[:-1], -1) + ) + offset += mul * ir.dim + return torch.cat(pieces, dim=-1) + + +def test_linear_layouts_rotation_and_gradients(): + torch.manual_seed(31) + a, b = cue.Irreps("SO3", "2x0+3x1"), cue.Irreps("SO3", "3x0+2x1") + kwargs = dict(internal_weights=False, method="naive", dtype=torch.float64) + mul = cuet.Linear(a, b, layout=cue.mul_ir, **kwargs) + ir = cuet.Linear(a, b, layout=cue.ir_mul, **kwargs) + x = torch.randn(2, a.dim, dtype=torch.float64, requires_grad=True) + w = torch.randn(1, mul.weight_numel, dtype=torch.float64, requires_grad=True) + y = mul(x, w) + y_ir = ir(transpose_blocks(x, a, cue.mul_ir), w) + torch.testing.assert_close(y, transpose_blocks(y_ir, b, cue.ir_mul)) + rotation = np.array([0.3, -0.1, 0.4]) + da, db = [ + torch.from_numpy(cue.IrrepsAndLayout(rep, cue.mul_ir).exp_map(rotation, [])) + for rep in (a, b) + ] + torch.testing.assert_close(mul(x @ da.T, w), y @ db.T) + for grad in torch.autograd.grad(y.square().sum(), (x, w)): + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + + +def test_channelwise_external_unshared_weights(): + torch.manual_seed(32) + a, b = cue.Irreps("SO3", "2x0+2x1"), cue.Irreps("SO3", "0+1") + keep = [cue.SO3(0), cue.SO3(1)] + module = cuet.ChannelWiseTensorProduct( + a, + b, + keep, + shared_weights=False, + internal_weights=False, + layout=cue.ir_mul, + method="naive", + dtype=torch.float64, + ) + e = cue.descriptors.channelwise_tensor_product(a, b, keep) + x, y, w = [ + torch.randn(3, size, dtype=torch.float64, requires_grad=True) + for size in (a.dim, b.dim, module.weight_numel) + ] + actual = module(x, y, w) + expected = e(*(v.detach().numpy() for v in (w, x, y)))[0] + np.testing.assert_allclose(actual.detach().numpy(), expected, atol=1e-12) + for grad in torch.autograd.grad(actual.square().sum(), (x, y, w)): + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + + +def test_gather_scatter_repeated_receivers_and_gradients(): + torch.manual_seed(33) + e = cue.descriptors.linear(cue.Irreps("SO3", "2x0"), cue.Irreps("SO3", "3x0")) + module = cuet.SegmentedPolynomial(e.polynomial, method="naive") + x = torch.randn(4, 2, dtype=torch.float64, requires_grad=True) + w = torch.randn(1, e.inputs[0].dim, dtype=torch.float64, requires_grad=True) + senders = torch.tensor([0, 1, 0, 3, 1]) + receivers = torch.tensor([2, 0, 2, 1, 2]) + shape = torch.empty(4, 1, dtype=torch.float64) + [actual] = module( + [w, x], + input_indices={1: senders}, + output_indices={0: receivers}, + output_shapes={0: shape}, + ) + [edges] = module([w, x[senders]]) + expected = torch.zeros(4, 3, dtype=torch.float64).index_add_(0, receivers, edges) + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(actual[3], torch.zeros(3, dtype=torch.float64)) + actual_grads = torch.autograd.grad(actual.square().sum(), (x, w)) + expected_grads = torch.autograd.grad(expected.square().sum(), (x, w)) + for actual_grad, expected_grad in zip(actual_grads, expected_grads): + torch.testing.assert_close(actual_grad, expected_grad) + + +def test_fully_connected_tensor_product_cpu(): + torch.manual_seed(34) + irreps = cue.Irreps("SO3", "0+1") + module = cuet.FullyConnectedTensorProduct( + irreps, irreps, irreps, method="naive", dtype=torch.float64 + ) + x = torch.randn(2, irreps.dim, dtype=torch.float64, requires_grad=True) + result = module(x, x) + assert result.shape == x.shape + result.square().sum().backward() + assert torch.isfinite(x.grad).all() and x.grad.abs().sum() > 0 diff --git a/.github/tests/test_torch_skill_examples.py b/.github/tests/test_torch_skill_examples.py new file mode 100644 index 00000000..41c17557 --- /dev/null +++ b/.github/tests/test_torch_skill_examples.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run each published recipe independently and check the trainable GNN.""" + +import importlib +import re +from pathlib import Path + +import numpy as np +import pytest +import torch + +import cuequivariance as cue + +SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" +EXAMPLES = [ + compile(code, f"{SKILL}:example-{index}", "exec") + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", SKILL.read_text(), re.S) + ) +] + + +@pytest.mark.parametrize("index", range(2, len(EXAMPLES) - 1)) +def test_published_cpu_example(index): + namespace = {} + exec(EXAMPLES[1], namespace) + exec(EXAMPLES[index], namespace) + + +@pytest.mark.parametrize( + "failed_module, error", + [ + ("cuequivariance_torch", ModuleNotFoundError("cuEquivariance is absent")), + ("cuequivariance", OSError("shared library cannot be loaded")), + ("torch", ImportError("PyTorch cannot be imported")), + ], +) +def test_environment_probe_preserves_partial_results(monkeypatch, failed_module, error): + modules = { + name: importlib.import_module(name) + for name in ("torch", "cuequivariance", "cuequivariance_torch") + } + + def import_module(name): + if name == failed_module: + raise error + return modules[name] + + monkeypatch.setattr(importlib, "import_module", import_module) + namespace = {} + exec(EXAMPLES[0], namespace) + assert namespace["available"] == { + name: module for name, module in modules.items() if name != failed_module + } + assert namespace["import_errors"] == { + failed_module: f"{type(error).__name__}: {error}" + } + + +def test_documented_gnn_is_trainable_and_equivariant(): + namespace = {} + exec(EXAMPLES[1], namespace) + exec(EXAMPLES[-1], namespace) + torch.manual_seed(41) + irreps_in = cue.Irreps("SO3", "2x0+2x1") + irreps_out = cue.Irreps("SO3", "3x0+1") + model = namespace["SimpleGNN"](irreps_in, irreps_out).double() + features = torch.randn(5, irreps_in.dim, dtype=torch.float64, requires_grad=True) + vectors = torch.randn(6, 3, dtype=torch.float64, requires_grad=True) + senders = torch.tensor([0, 1, 3, 0, 1, 2]) + receivers = torch.tensor([2, 0, 2, 1, 2, 0]) + output = model(features, vectors, senders, receivers, 4) + assert output.shape == (4, irreps_out.dim) + assert torch.count_nonzero(output[3]) == 0 + torch.testing.assert_close( + output, model(features, vectors, senders, receivers, 4), rtol=0, atol=0 + ) + + rotation = np.array([0.2, -0.3, 0.1]) + din, dout = [ + torch.from_numpy(cue.IrrepsAndLayout(r, cue.mul_ir).exp_map(rotation, [])) + for r in (irreps_in, irreps_out) + ] + dvector = torch.from_numpy(cue.SO3(1).exp_map(rotation, [])) + rotated = model(features @ din.T, vectors @ dvector.T, senders, receivers, 4) + torch.testing.assert_close(rotated, output @ dout.T, rtol=1e-7, atol=1e-9) + + output.square().sum().backward() + for name, tensor in [ + *model.named_parameters(), + ("features", features), + ("vectors", vectors), + ]: + assert tensor.grad is not None, name + assert torch.isfinite(tensor.grad).all(), name + assert tensor.grad.abs().sum() > 0, name diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2c371767..36f17de8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,6 +33,8 @@ jobs: - name: Replace symlinks with actual files run: | + # Includes package SKILL.md links into the canonical skills/ tree. + # Keep this before building: sdists must carry regular resource files. find . -type l -exec sh -c 'cp --remove-destination "$(readlink -f "$1")" "$1"' _ {} \; - name: Build & Publish cuequivariance diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 00000000..4092d220 --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,26 @@ +name: Request NVSkills CI + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + +jobs: + request: + if: > + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + statuses: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@main + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 920e7fbd..1c0b727a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,12 @@ jobs: run: | pytest --doctest-modules -x -m "not slow" cuequivariance + - name: Check core skill evaluations and packaging + run: | + pytest -q .github/tests/test_core_skill_evals.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance + - name: Downgrade numpy if: matrix.python-version == '3.10' run: | @@ -62,6 +68,14 @@ jobs: run: | XLA_PYTHON_CLIENT_PREALLOCATE=false pytest --doctest-modules -x -m "not slow" cuequivariance_jax + - name: Check JAX skill evaluations and packaging + env: + JAX_PLATFORMS: cpu + run: | + pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_jax_skill_challenges.py .github/tests/test_skill_challenge_packaging.py .github/tests/test_skill_execution.py .github/tests/test_skill_eval_setup.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance_jax + cuequivariance-jax-no-flax: runs-on: ubuntu-latest @@ -105,3 +119,9 @@ jobs: - name: Test with pytest run: | pytest --doctest-modules -x -m "not slow" cuequivariance_torch + + - name: Check PyTorch skill evaluations and packaging + run: | + pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py .github/tests/test_torch_skill_challenges.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance_torch diff --git a/README.md b/README.md index d374df55..c2b132aa 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,20 @@ pip install cuequivariance-ops-jax-cu12 # or -cu13 pip install cuequivariance-ops-torch-cu12 # or -cu13 ``` +## Agent skills + +The repository provides three independent agent skills: + +- [cuequivariance](skills/cuequivariance/SKILL.md): group theory, descriptors, and NumPy execution. +- [cuequivariance-jax](skills/cuequivariance-jax/SKILL.md): JAX execution and Flax NNX layers. +- [cuequivariance-torch](skills/cuequivariance-torch/SKILL.md): PyTorch operations and layers. + +Installed Python packages also expose these instructions through +`python -m cuequivariance skill`, `python -m cuequivariance_jax skill`, and +`python -m cuequivariance_torch skill`. +See [skill maintenance and validation](skills/README.md) for source layout, +evaluation coverage, packaging checks, and NVSkills CI onboarding. + ## License All files hosted in this repository are subject to the Apache 2.0 license. @@ -28,4 +42,3 @@ All files hosted in this repository are subject to the Apache 2.0 license. ## Disclaimer cuEquivariance is in a Beta state. Beta products may not be fully functional, may contain errors or design flaws, and may be changed at any time without notice. We appreciate your feedback to improve and iterate on our Beta products. - diff --git a/cuequivariance/cuequivariance/SKILL.md b/cuequivariance/cuequivariance/SKILL.md deleted file mode 100644 index f14cbd8b..00000000 --- a/cuequivariance/cuequivariance/SKILL.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -name: cuequivariance -description: Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics). Use when working with cuequivariance group theory, irreps, or segmented polynomials. ---- - -# cuequivariance: Groups, Irreps, and Segmented Polynomials - -## Overview - -`cuequivariance` (imported as `cue`) provides two core abstractions: - -1. **Group theory**: `Irrep` subclasses define irreducible representations of Lie groups (SO3, O3, SU2, or custom). `Irreps` manages collections with multiplicities. -2. **Segmented polynomials**: `SegmentedTensorProduct` describes tensor contractions over segments of varying shape, linked by `Path` objects carrying Clebsch-Gordan coefficients. `SegmentedPolynomial` wraps multiple STPs into a polynomial with named inputs/outputs. Two higher-level wrappers attach group representations: - - `EquivariantPolynomial` — dense operands with `IrrepsAndLayout` metadata - - `IrDictPolynomial` — operands already split by irrep, with per-group `Irreps` metadata for the `dict[Irrep, Array]` workflow - -## Defining a custom group - -Subclass `cue.Irrep` (a frozen dataclass) and implement: - -```python -from __future__ import annotations -import dataclasses -import re -from typing import Iterator -import numpy as np -import cuequivariance as cue - -@dataclasses.dataclass(frozen=True) -class Z2(cue.Irrep): - odd: bool # dataclass field -- required for correct __eq__ and __hash__ - - # No __init__ needed -- @dataclass(frozen=True) generates it: Z2(odd=True) - - @classmethod - def regexp_pattern(cls) -> re.Pattern: - # Pattern whose first group is passed to from_string - return re.compile(r"(odd|even)") - - @classmethod - def from_string(cls, string: str) -> Z2: - return cls(odd=string == "odd") - - def __repr__(rep: Z2) -> str: - return "odd" if rep.odd else "even" - - def __mul__(rep1: Z2, rep2: Z2) -> Iterator[Z2]: - # Selection rule: which irreps appear in the tensor product rep1 x rep2 - return [Z2(odd=rep1.odd ^ rep2.odd)] - - @classmethod - def clebsch_gordan(cls, rep1: Z2, rep2: Z2, rep3: Z2) -> np.ndarray: - # Shape: (num_paths, rep1.dim, rep2.dim, rep3.dim) - if rep3 in rep1 * rep2: - return np.array([[[[1]]]]) - else: - return np.zeros((0, 1, 1, 1)) - - @property - def dim(rep: Z2) -> int: - return 1 - - def __lt__(rep1: Z2, rep2: Z2) -> bool: - # Ordering for sorting; dimension is compared first by the base class - return rep1.odd < rep2.odd - - @classmethod - def iterator(cls) -> Iterator[Z2]: - # Must yield trivial irrep first - for odd in [False, True]: - yield Z2(odd=odd) - - def discrete_generators(rep: Z2) -> np.ndarray: - # Shape: (num_generators, dim, dim) - if rep.odd: - return -np.ones((1, 1, 1)) - else: - return np.ones((1, 1, 1)) - - def continuous_generators(rep: Z2) -> np.ndarray: - # Shape: (lie_dim, dim, dim) -- Z2 is discrete, so lie_dim=0 - return np.zeros((0, rep.dim, rep.dim)) - - def algebra(self) -> np.ndarray: - # Shape: (lie_dim, lie_dim, lie_dim) -- structure constants [X_i, X_j] = A_ijk X_k - return np.zeros((0, 0, 0)) - - -# Usage: -irreps = cue.Irreps(Z2, "3x odd + 2x even") # dim=5 -``` - -### Required methods summary - -| Method | Returns | Purpose | -|--------|---------|---------| -| `regexp_pattern()` | `re.Pattern` | Parse string like `"1"`, `"0e"`, `"odd"` | -| `from_string(s)` | `Irrep` | Construct from matched string | -| `__repr__` | `str` | Canonical string form | -| `__mul__(a, b)` | `Iterator[Irrep]` | Selection rule for tensor product | -| `clebsch_gordan(a, b, c)` | `ndarray (n, d1, d2, d3)` | CG coefficients | -| `dim` (property) | `int` | Dimension of representation | -| `__lt__(a, b)` | `bool` | Ordering (dimension first, then custom) | -| `iterator()` | `Iterator[Irrep]` | All irreps, trivial first | -| `continuous_generators()` | `ndarray (lie_dim, dim, dim)` | Lie algebra generators | -| `discrete_generators()` | `ndarray (n, dim, dim)` | Finite symmetry generators | -| `algebra()` | `ndarray (lie_dim, lie_dim, lie_dim)` | Structure constants | - -### Built-in groups - -- **`cue.SO3(l)`**: 3D rotations. `l` is a non-negative integer. `dim = 2l+1`. String: `"0"`, `"1"`, `"2"`. -- **`cue.O3(l, p)`**: 3D rotations + parity. `p=+1` (even) or `p=-1` (odd). String: `"0e"`, `"1o"`, `"2e"`. -- **`cue.SU2(j)`**: Spin group. `j` is a non-negative half-integer. String: `"0"`, `"1/2"`, `"1"`. - -## Irreps and layout - -```python -irreps = cue.Irreps("SO3", "16x0 + 4x1 + 2x2") # 16 scalars, 4 vectors, 2 rank-2 -irreps.dim # 16*1 + 4*3 + 2*5 = 38 - -for mul, ir in irreps: - print(mul, ir, ir.dim) # 16 0 1, then 4 1 3, then 2 2 5 -``` - -`IrrepsLayout` controls memory order within each `(mul, ir)` block: - -- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used by all descriptors and ir_dict internally** -- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **used by nnx `dict[Irrep, Array]` and PyTorch** - -`IrrepsAndLayout` combines irreps with a layout into a `Rep`: - -```python -rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) -rep.dim # 10 -``` - -## Building a SegmentedTensorProduct from scratch - -The subscripts string uses Einstein notation. Operands are comma-separated, coefficient modes follow `+`. - -```python -# Matrix-vector multiply: y_i = sum_j M_ij * x_j -d = cue.SegmentedTensorProduct.from_subscripts("ij,j,i") -d.add_segment(0, (3, 4)) # operand 0: matrix segment of shape (3, 4) -d.add_segment(1, (4,)) # operand 1: vector of size 4 -d.add_segment(2, (3,)) # operand 2: output vector of size 3 -d.add_path(0, 0, 0, c=1.0) # link segments 0,0,0 with coefficient=1.0 - -poly = cue.SegmentedPolynomial.eval_last_operand(d) # last operand becomes output -[y] = poly(M_flat, x) # numpy evaluation -``` - -### Multi-segment STP (how descriptors work internally) - -Descriptors build STPs with multiple segments per operand. Each segment corresponds to an irrep block: - -```python -# Linear equivariant map: output[iv] = sum_u weight[uv] * input[iu] -d = cue.SegmentedTensorProduct.from_subscripts("uv,iu,iv") - -# Segment for l=1: ir_dim=3, mul_in=2, mul_out=5 -s_in_0 = d.add_segment(1, (3, 2)) # input block -s_out_0 = d.add_segment(2, (3, 5)) # output block -d.add_path((2, 5), s_in_0, s_out_0, c=1.0) - -# Segment for l=0: ir_dim=1, mul_in=4, mul_out=3 -s_in_1 = d.add_segment(1, (1, 4)) -s_out_1 = d.add_segment(2, (1, 3)) -d.add_path((4, 3), s_in_1, s_out_1, c=1.0) -``` - -### Weights operand - -For weighted tensor products (subscript starting with `uvw` or `uv`), the first operand is always weights. The weight segment shape is `(mul_1, mul_2, ...)` matching the multiplicity modes. The weights operand gets `new_scalars()` irreps since weights are invariant. - -### CG coefficients as path coefficients - -```python -d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") -# For each pair of input irreps and each output irrep in the selection rule: -for cg in cue.clebsch_gordan(ir1, ir2, ir3): - # cg has shape (ir1.dim, ir2.dim, ir3.dim) - d.add_path((mul1, mul2, mul3), seg_in1, seg_in2, seg_out, c=cg) -``` - -## Descriptors - -All descriptors come in two variants: - -- **Original** — returns `EquivariantPolynomial` with dense operands -- **`_ir_dict`** — returns `IrDictPolynomial` with operands already split by irrep - -### EquivariantPolynomial descriptors - -```python -# Fully connected tensor product (all input-output irrep combinations) -e = cue.descriptors.fully_connected_tensor_product( - 16 * cue.Irreps("SO3", "0 + 1 + 2"), - 16 * cue.Irreps("SO3", "0 + 1 + 2"), - 16 * cue.Irreps("SO3", "0 + 1 + 2"), -) - -# Channelwise tensor product (same-channel only, sparse) -e = cue.descriptors.channelwise_tensor_product( - 64 * cue.Irreps("SO3", "0 + 1"), cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), simplify_irreps3=True, -) - -# Full (weightless) tensor product -e = cue.descriptors.full_tensor_product( - cue.Irreps("SO3", "2x0 + 1x1"), cue.Irreps("SO3", "0 + 1"), -) - -# Elementwise tensor product (paired channels) -e = cue.descriptors.elementwise_tensor_product( - cue.Irreps("SO3", "4x0 + 4x1"), cue.Irreps("SO3", "4x0 + 4x1"), -) - -# Linear equivariant map (weight x input) -e = cue.descriptors.linear( - cue.Irreps("SO3", "4x0 + 2x1"), - cue.Irreps("SO3", "3x0 + 5x1"), -) - -# Spherical harmonics -e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2, 3]) - -# Symmetric contraction (MACE-style) -e = cue.descriptors.symmetric_contraction( - 64 * cue.Irreps("SO3", "0 + 1 + 2"), - 64 * cue.Irreps("SO3", "0 + 1"), - (1, 2, 3), -) -``` - -### IrDictPolynomial descriptors - -Each `_ir_dict` variant returns an `IrDictPolynomial` whose polynomial is already split by irrep. The `input_irreps` and `output_irreps` tuples describe the operand groups. - -```python -# Channelwise tensor product -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 64 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -# desc.polynomial — SegmentedPolynomial, already split by irrep -# desc.input_irreps — (weight_irreps, irreps1, irreps2) -# desc.output_irreps — (irreps_out,) - -# Fully connected tensor product -desc = cue.descriptors.fully_connected_tensor_product_ir_dict(irreps1, irreps2, irreps3) - -# Full (weightless) tensor product -desc = cue.descriptors.full_tensor_product_ir_dict(irreps1, irreps2) - -# Elementwise tensor product -desc = cue.descriptors.elementwise_tensor_product_ir_dict(irreps1, irreps2) - -# Linear -desc = cue.descriptors.linear_ir_dict(irreps_in, irreps_out) - -# Spherical harmonics -desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2, 3]) - -# Symmetric contraction -desc = cue.descriptors.symmetric_contraction_ir_dict(irreps_in, irreps_out, (1, 2, 3)) -``` - -### IrDictPolynomial - -`IrDictPolynomial` pairs a `SegmentedPolynomial` (already split by irrep) with the `Irreps` that describe each operand group. - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) - -desc.polynomial # SegmentedPolynomial — each operand is one (mul, ir) block -desc.input_irreps # (weight_irreps, irreps1, irreps2) -desc.output_irreps # (irreps_out,) - -# Scale coefficients -scaled_poly = desc.polynomial * 0.5 - -# Access individual operand info -for i, op in enumerate(desc.polynomial.inputs): - print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") -``` - -Contract: for each `(mul, ir)` block in `input_irreps` / `output_irreps`, the corresponding polynomial operand has size `mul * ir.dim`. - -### split_polynomial_by_irreps - -The low-level function underlying `_ir_dict` descriptors. Splits one polynomial operand at irrep boundaries: - -```python -poly = e.polynomial # from an EquivariantPolynomial -poly = cue.split_polynomial_by_irreps(poly, 2, irreps_sh) # split input 2 -poly = cue.split_polynomial_by_irreps(poly, 1, irreps_in) # split input 1 -poly = cue.split_polynomial_by_irreps(poly, -1, irreps_out) # split output -``` - -### EquivariantPolynomial key methods - -```python -e.inputs # tuple of Rep (group representations for each input) -e.outputs # tuple of Rep -e.polynomial # the underlying SegmentedPolynomial - -# Numpy evaluation -[out] = e(weights, input1, input2) - -# Preparing for uniform_1d execution (see cuequivariance_jax SKILL.md) -e_ready = e.squeeze_modes().flatten_coefficient_modes() - -# Split an operand into per-irrep pieces (for ir_dict interface) -e_split = e.split_operand_by_irrep(1).split_operand_by_irrep(-1) - -# Scale all coefficients -e_scaled = e * 0.5 - -# Fuse compatible STPs -e_fused = e.fuse_stps() -``` - -### normalize_paths_for_operand - -Called internally by descriptors. Normalizes path coefficients so that a random input produces unit-variance output for the specified operand. Critical for numerical stability. - -## SegmentedPolynomial structure - -```python -poly = e.polynomial -poly.num_inputs # number of input operands -poly.num_outputs # number of output operands -poly.inputs # tuple of SegmentedOperand -poly.outputs # tuple of SegmentedOperand -poly.operations # tuple of (Operation, SegmentedTensorProduct) - -# Each operation maps buffers to STP operands -for op, stp in poly.operations: - print(op.buffers) # e.g., (0, 1, 2) means inputs[0], inputs[1] -> outputs[0] - print(stp.subscripts) -``` - -### SegmentedOperand - -```python -operand = poly.inputs[0] -operand.num_segments # how many segments -operand.segments # tuple of shape tuples, e.g., ((3, 4), (1, 2)) -operand.size # total flattened size (sum of products of segment shapes) -operand.ndim # number of dimensions per segment -operand.all_same_segment_shape() # True if all segments have identical shape -operand.segment_shape # the common shape (only if all_same_segment_shape) -``` - -## Custom equivariant polynomial from scratch - -```python -import numpy as np -import cuequivariance as cue - -# Build a fully-connected SO3(1)xSO3(1)->SO3(0) tensor product manually -cg = cue.clebsch_gordan(cue.SO3(1), cue.SO3(1), cue.SO3(0)) # shape (1, 3, 3, 1) - -d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") -d.add_segment(1, (3, 4)) # input1: 4x SO3(1), shape=(ir_dim, mul) -d.add_segment(2, (3, 4)) # input2: 4x SO3(1) -d.add_segment(3, (1, 16)) # output: 16x SO3(0) (4*4 fully connected) - -for c in cg: - d.add_path((4, 4, 16), 0, 0, 0, c=c) - -d = d.normalize_paths_for_operand(-1) - -poly = cue.SegmentedPolynomial.eval_last_operand(d) -ep = cue.EquivariantPolynomial( - [ - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1").new_scalars(d.operands[0].size), cue.ir_mul), - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), - ], - [cue.IrrepsAndLayout(cue.Irreps("SO3", "16x0"), cue.ir_mul)], - poly, -) - -# Numpy evaluation -w = np.random.randn(ep.inputs[0].dim) -x = np.random.randn(ep.inputs[1].dim) -y = np.random.randn(ep.inputs[2].dim) -[out] = ep(w, x, y) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `Irrep` base class | `cuequivariance/group_theory/representations/irrep.py` | -| `Rep` base class | `cuequivariance/group_theory/representations/rep.py` | -| `SO3` | `cuequivariance/group_theory/representations/irrep_so3.py` | -| `O3` | `cuequivariance/group_theory/representations/irrep_o3.py` | -| `SU2` | `cuequivariance/group_theory/representations/irrep_su2.py` | -| `Irreps` | `cuequivariance/group_theory/irreps_array/irreps.py` | -| `IrrepsLayout` | `cuequivariance/group_theory/irreps_array/irreps_layout.py` | -| `IrrepsAndLayout` | `cuequivariance/group_theory/irreps_array/irreps_and_layout.py` | -| `SegmentedTensorProduct` | `cuequivariance/segmented_polynomials/segmented_tensor_product.py` | -| `SegmentedPolynomial` | `cuequivariance/segmented_polynomials/segmented_polynomial.py` | -| `EquivariantPolynomial` | `cuequivariance/group_theory/equivariant_polynomial.py` | -| `IrDictPolynomial` | `cuequivariance/group_theory/ir_dict_polynomial.py` | -| Descriptors | `cuequivariance/group_theory/descriptors/` | -| Tensor product descriptors | `cuequivariance/group_theory/descriptors/irreps_tp.py` | -| `spherical_harmonics` | `cuequivariance/group_theory/descriptors/spherical_harmonics_.py` | -| `symmetric_contraction` | `cuequivariance/group_theory/descriptors/symmetric_contractions.py` | diff --git a/cuequivariance/cuequivariance/SKILL.md b/cuequivariance/cuequivariance/SKILL.md new file mode 120000 index 00000000..e412fd72 --- /dev/null +++ b/cuequivariance/cuequivariance/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance/SKILL.md \ No newline at end of file diff --git a/cuequivariance_jax/cuequivariance_jax/SKILL.md b/cuequivariance_jax/cuequivariance_jax/SKILL.md deleted file mode 100644 index 1da88a63..00000000 --- a/cuequivariance_jax/cuequivariance_jax/SKILL.md +++ /dev/null @@ -1,410 +0,0 @@ ---- -name: cuequivariance-jax -description: Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear). Use when writing JAX code with cuequivariance. ---- - -# cuequivariance_jax: Executing Equivariant Polynomials in JAX - -## Overview - -`cuequivariance_jax` (imported as `cuex`) executes `cuequivariance` polynomials on GPU via JAX. It provides: - -1. **Core primitive**: `cuex.segmented_polynomial()` — JAX primitive with full AD/vmap/JIT support -2. **Two data representations** (both built on `segmented_polynomial`): - - `cuex.equivariant_polynomial()` + `RepArray` — the original interface, a single contiguous array with representation metadata - - `cuex.ir_dict` module — `dict[Irrep, Array]` interface, uses `IrDictPolynomial` descriptors, works naturally with `jax.tree` -3. **NNX layers**: `cuex.nnx` module — Flax NNX `Module` wrappers using `dict[Irrep, Array]` - -## Execution methods - -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure JAX | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"indexed_linear"` | CUDA kernel | GPU, linear operations with `cuex.Repeats` indexing | - -## Core primitive: segmented_polynomial - -```python -import jax -import jax.numpy as jnp -import cuequivariance as cue -import cuequivariance_jax as cuex - -# Build a descriptor -e = cue.descriptors.channelwise_tensor_product( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -batch = 64 -w = jnp.ones((poly.inputs[0].size,)) # weights (shared across batch) -x = jax.random.normal(key, (batch, poly.inputs[1].size)) # batched input 1 -y = jax.random.normal(key, (batch, poly.inputs[2].size)) # batched input 2 - -# Execute with naive method -[out] = cuex.segmented_polynomial( - poly, - [w, x, y], # inputs - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], # output spec - method="naive", -) - -# Execute with uniform_1d (GPU, requires uniform segments) -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) -``` - -### Multiple batch axes with broadcasting - -Inputs can have any number of batch axes (everything before the last axis). Standard NumPy broadcasting applies: each batch axis is either size-1 or a common size. Inputs with fewer batch dimensions are implicitly prepended with size-1 axes: - -```python -# Fewer batch dims: weights with no batch axis broadcast across all -w = jnp.ones((poly.inputs[0].size,)) # 0 batch axes -> broadcasts -x = jnp.ones((5, 10, poly.inputs[1].size)) -y = jnp.ones((5, 10, poly.inputs[2].size)) - -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((5, 10, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) -``` - -### Indexing (gather/scatter) - -Index arrays provide gather (for inputs) and scatter (for outputs). One index per operand (inputs + outputs), `None` means no indexing: - -```python -i = jax.random.randint(key, (100, 50), 0, 10) # gather b along axis 0 -j1 = jax.random.randint(key, (100, 50), 0, 11) # scatter output axis 0 -j2 = jax.random.randint(key, (100, 1), 0, 12) # scatter output axis 1 - -[out] = cuex.segmented_polynomial( - poly, [a, b, c], - [jax.ShapeDtypeStruct((11, 12, poly.outputs[0].size), jnp.float32)], - indices=[None, np.s_[i, :], None, np.s_[j1, j2]], - method="uniform_1d", -) -``` - -### Gradients - -Fully differentiable — supports `jax.grad`, `jax.jacobian`, `jax.jvp`, `jax.vmap`: - -```python -def loss(w, x, y): - [out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="naive", - ) - return jnp.sum(out ** 2) - -grad_w = jax.grad(loss, 0)(w, x, y) -``` - -## ir_dict interface - -Uses `dict[Irrep, Array]` where each value has shape `(..., multiplicity, irrep_dim)`. This is the standard representation for NNX layers and works naturally with `jax.tree` operations. - -### Getting an ir_dict-ready polynomial - -Use `_ir_dict` descriptor variants, which return `IrDictPolynomial` with the polynomial already split by irrep: - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) - -poly = desc.polynomial # SegmentedPolynomial, already split by irrep -weight_irreps, irreps1, irreps2 = desc.input_irreps -(irreps_out,) = desc.output_irreps # tuple unpacking to get the single output group -``` - -Each polynomial operand corresponds to exactly one `(mul, ir)` block. The `input_irreps` and `output_irreps` tuples describe how operands group into logical operand groups (weights, node features, spherical harmonics, output). - -### Executing with segmented_polynomial_uniform_1d - -```python -from einops import rearrange - -num_edges, num_nodes = 100, 30 - -# Weights: reshape to (batch, num_segments, segment_size) -w_flat = jax.random.normal(key, (num_edges, poly.inputs[0].size)) -w = rearrange(w_flat, "e (s m) -> e s m", s=poly.inputs[0].num_segments) - -# Node features: dict[Irrep, Array] reshaped to (nodes, ir.dim, mul) for ir_mul layout -node_feats = { - cue.SO3(0): jnp.ones((num_nodes, 32, 1)), # 32x scalar - cue.SO3(1): jnp.ones((num_nodes, 32, 3)), # 32x vector -} -x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - -# Spherical harmonics: (edges, ir.dim) — no multiplicity dimension -sph = { - cue.SO3(0): jnp.ones((num_edges, 1)), - cue.SO3(1): jnp.ones((num_edges, 3)), -} - -# Build output template -senders = jax.random.randint(key, (num_edges,), 0, num_nodes) -receivers = jax.random.randint(key, (num_edges,), 0, num_nodes) -out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(irreps_out, poly.outputs) -} - -# Execute with gather (senders) and scatter (receivers) -y = cuex.ir_dict.segmented_polynomial_uniform_1d( - poly, - [w, x1, sph], - out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", -) -# y is dict[Irrep, Array] with accumulated results at receiver nodes -``` - -### ir_dict utility functions - -```python -# Validate dict matches irreps -cuex.ir_dict.assert_mul_ir_dict(irreps, x) # asserts shape (..., mul, ir.dim) - -# Convert flat array <-> dict -d = cuex.ir_dict.flat_to_dict(irreps, flat_array) # layout="mul_ir" default -d = cuex.ir_dict.flat_to_dict(irreps, flat_array, layout="ir_mul") -flat = cuex.ir_dict.dict_to_flat(irreps, d) - -# Arithmetic -z = cuex.ir_dict.irreps_add(x, y) -z = cuex.ir_dict.irreps_zeros_like(x) - -# Create template dict -template = cuex.ir_dict.mul_ir_dict(irreps, jax.ShapeDtypeStruct(shape, dtype)) -``` - -## RepArray interface: equivariant_polynomial - -The original interface. Wraps `segmented_polynomial` with `RepArray` — a single contiguous array with representation metadata: - -```python -e = cue.descriptors.fully_connected_tensor_product( - 4 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - 4 * cue.Irreps("SO3", "0 + 1"), -) - -inputs = [ - cuex.randn(jax.random.key(i), rep, (batch,), jnp.float32) - for i, rep in enumerate(e.inputs) -] - -# Returns a RepArray with representation metadata -out = cuex.equivariant_polynomial(e, inputs, method="naive") -out.array # the raw jax.Array -out.reps # dict mapping axes to Rep objects -``` - -## NNX layers - -### IrrepsLinear - -Equivariant linear layer using `dict[Irrep, Array]`: - -```python -from flax import nnx - -linear = cuex.nnx.IrrepsLinear( - irreps_in=cue.Irreps(cue.SO3, "4x0 + 2x1").regroup(), # must be regrouped - irreps_out=cue.Irreps(cue.SO3, "3x0 + 5x1").regroup(), - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) - -# Input/output: dict[Irrep, Array] with shape (batch, mul, ir.dim) -x = { - cue.SO3(0): jnp.ones((batch, 4, 1)), - cue.SO3(1): jnp.ones((batch, 2, 3)), -} -y = linear(x) -# y[cue.SO3(0)].shape == (batch, 3, 1) -# y[cue.SO3(1)].shape == (batch, 5, 3) -``` - -Implementation uses `jnp.einsum("uv,...ui->...vi", w, x[ir])` per irrep with `1/sqrt(mul_in)` normalization. - -### SphericalHarmonics - -Uses `spherical_harmonics_ir_dict` internally for the `dict[Irrep, Array]` output: - -```python -sh = cuex.nnx.SphericalHarmonics(max_degree=3, eps=0.0) - -vectors = jax.random.normal(key, (batch, 3)) # 3D vectors -y = sh(vectors) -# y[cue.O3(0, 1)].shape == (batch, 1, 1) # L=0 -# y[cue.O3(1, -1)].shape == (batch, 1, 3) # L=1 -# y[cue.O3(2, 1)].shape == (batch, 1, 5) # L=2 -# y[cue.O3(3, -1)].shape == (batch, 1, 7) # L=3 -``` - -### IrrepsNormalize - -```python -norm = cuex.nnx.IrrepsNormalize(eps=1e-6, scale=1.0, skip_scalars=True) -y = norm(x) # normalizes non-scalar irreps by RMS over ir.dim, averaged over mul -``` - -### MLP (scalar only) - -```python -mlp = cuex.nnx.MLP( - layer_sizes=[64, 128, 64], - activation=jax.nn.silu, - output_activation=False, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) -y = mlp(x_scalar) # standard dense MLP with 1/sqrt(fan_in) normalization -``` - -### IrrepsIndexedLinear - -For species-indexed linear layers (different weights per atom type): - -```python -indexed_linear = cuex.nnx.IrrepsIndexedLinear( - irreps_in=cue.Irreps(cue.O3, "8x0e").regroup(), - irreps_out=cue.Irreps(cue.O3, "16x0e").regroup(), - num_indices=50, # number of species - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) - -# num_index_counts: how many atoms of each species -species_counts = jnp.array([3, 4, 3, ...]) # sum = batch_size -y = indexed_linear(x, species_counts) -``` - -Uses `method="indexed_linear"` internally with `cuex.Repeats`. - -## Preparing polynomials for uniform_1d - -The `uniform_1d` CUDA kernel requires: -1. All segments within each operand have **the same shape** -2. A **single mode** in the subscripts (after preprocessing) - -### From EquivariantPolynomial to uniform_1d-ready - -For `equivariant_polynomial()` (RepArray interface): - -```python -e = cue.descriptors.channelwise_tensor_product(...) -e = e.squeeze_modes().flatten_coefficient_modes() -out = cuex.equivariant_polynomial(e, inputs, method="uniform_1d") -``` - -For `ir_dict` (dict[Irrep, Array] interface), use `_ir_dict` descriptors directly: - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out -) -poly = desc.polynomial -# Each operand has a single irrep type -> maps naturally to dict[Irrep, Array] -``` - -### Why splitting by irrep matters - -Without splitting, a dense operand like `32x0+32x1` requires all irreps packed into a single contiguous buffer. After splitting, each irrep gets its own separate buffer passed to the CUDA kernel via FFI. The buffers no longer need to be contiguous with each other. - -This is especially useful when the polynomial is preceded or followed by per-irrep linear layers (like `IrrepsLinear`). With split operands, no transpose or copy is needed between the linear layers and the polynomial — the `dict[Irrep, Array]` flows directly through the pipeline. - -## Complete GNN message-passing example - -This pattern is used in NequIP, MACE, and similar equivariant GNN models: - -```python -class MessagePassing(nnx.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out, epsilon, *, name, dtype, rngs): - self.name = name - desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out - ) - (self.irreps_out,) = desc.output_irreps - self.poly = desc.polynomial * epsilon - self.weight_numel = self.poly.inputs[0].size - - def __call__(self, weights, node_feats, sph, senders, receivers, num_nodes): - # weights: (num_edges, weight_numel) - w = rearrange(weights, "e (s m) -> e s m", s=self.poly.inputs[0].num_segments) - # node_feats: dict[Irrep, Array] with (nodes, mul, ir.dim) - x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - # sph: dict[Irrep, Array] with (edges, 1, ir.dim) or (edges, ir.dim) - x2 = jax.tree.map(lambda v: rearrange(v, "e 1 i -> e i"), sph) - - out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(self.irreps_out, self.poly.outputs) - } - - y = cuex.ir_dict.segmented_polynomial_uniform_1d( - self.poly, [w, x1, x2], out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", - ) - # Rearrange output back to (nodes, mul, ir.dim) for downstream layers - return { - ir: rearrange(v, "n (i s) m -> n (s m) i", i=ir.dim) - for ir, v in y.items() - } -``` - -## RepArray - -Representation-aware JAX array: - -```python -rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) -x = cuex.RepArray(rep, jnp.ones((batch, rep.dim))) -x = cuex.randn(jax.random.key(0), rep, (batch,), jnp.float32) - -x.array # raw jax.Array -x.reps # {axis: Rep} -x.irreps # Irreps (if last axis is IrrepsAndLayout) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `segmented_polynomial` primitive | `cuequivariance_jax/segmented_polynomials/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_naive.py` | -| `indexed_linear` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_indexed_linear.py` | -| `equivariant_polynomial` | `cuequivariance_jax/equivariant_polynomial.py` | -| `ir_dict` module | `cuequivariance_jax/ir_dict.py` | -| `nnx` module | `cuequivariance_jax/nnx.py` | -| `RepArray` | `cuequivariance_jax/rep_array/rep_array_.py` | -| `Repeats` / utilities | `cuequivariance_jax/segmented_polynomials/utils.py` | -| NequIP example | `cuequivariance_jax/examples/nequip_nnx.py` | -| MACE example | `cuequivariance_jax/examples/mace_nnx.py` | diff --git a/cuequivariance_jax/cuequivariance_jax/SKILL.md b/cuequivariance_jax/cuequivariance_jax/SKILL.md new file mode 120000 index 00000000..c1736cef --- /dev/null +++ b/cuequivariance_jax/cuequivariance_jax/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance-jax/SKILL.md \ No newline at end of file diff --git a/cuequivariance_torch/cuequivariance_torch/SKILL.md b/cuequivariance_torch/cuequivariance_torch/SKILL.md deleted file mode 100644 index 1f547491..00000000 --- a/cuequivariance_torch/cuequivariance_torch/SKILL.md +++ /dev/null @@ -1,400 +0,0 @@ ---- -name: cuequivariance-torch -description: Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv). Use when writing PyTorch code with cuequivariance. ---- - -# cuequivariance_torch: Executing Equivariant Polynomials in PyTorch - -## Overview - -`cuequivariance_torch` (imported as `cuet`) executes `cuequivariance` polynomials on GPU via PyTorch. It provides: - -1. **Core primitive**: `cuet.SegmentedPolynomial` — `torch.nn.Module` with multiple CUDA backends -2. **High-level operations** (`torch.nn.Module`): `ChannelWiseTensorProduct`, `FullyConnectedTensorProduct`, `Linear`, `SymmetricContraction`, `SphericalHarmonics`, `Rotation`, `Inversion` -3. **Layers**: `cuet.layers.BatchNorm`, `cuet.layers.FullyConnectedTensorProductConv` (message passing) -4. **Utilities**: `triangle_attention`, `triangle_multiplicative_update`, `attention_pair_bias` (AlphaFold3-style) -5. **Export support**: `onnx_custom_translation_table()`, `register_tensorrt_plugins()` - -## Execution methods - -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure PyTorch (einsum) | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"fused_tp"` | CUDA kernel | GPU, 3- or 4-operand contractions, float32/float64 | -| `"indexed_linear"` | CUDA kernel | GPU, linear with indexed weights, sorted indices | - -## Core primitive: SegmentedPolynomial - -```python -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet - -# Build a descriptor -e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) -poly = e.polynomial - -# Create the module -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -# Forward pass -x = torch.randn(batch, 3, device="cuda") -[output] = sp([x]) -# output.shape == (batch, 9) -- 1 + 3 + 5 -``` - -### Inputs, indexing, and scatter - -```python -e = cue.descriptors.channelwise_tensor_product( - 16 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -w = torch.randn(1, poly.inputs[0].size, device="cuda") # shared weights -x1 = torch.randn(batch, poly.inputs[1].size, device="cuda") # batched input 1 -x2 = torch.randn(batch, poly.inputs[2].size, device="cuda") # batched input 2 - -# Basic forward -[out] = sp([w, x1, x2]) - -# With input gathering (e.g., gather x1 by node index) -senders = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp([w, x1, x2], input_indices={1: senders}) - -# With output scattering (accumulate into target nodes) -receivers = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp( - [w, x1, x2], - input_indices={1: senders}, - output_indices={0: receivers}, - output_shapes={0: torch.empty(num_nodes, 1, device="cuda")}, -) -``` - -### Math dtype control - -```python -# Compute in float32 regardless of input dtype -sp = cuet.SegmentedPolynomial(poly, method="fused_tp", math_dtype=torch.float32) - -# For fused_tp, math_dtype must be float32 or float64 -# For naive, any torch.dtype works -# For uniform_1d, float32 or float64 (auto-selects float32 if input is e.g. float16) -``` - -## High-level operations - -All operations are `torch.nn.Module` subclasses. They wrap `SegmentedPolynomial` and handle layout transposition automatically. - -### Memory layout - -`IrrepsLayout` controls memory order within each `(mul, ir)` block: - -- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **default, compatible with e3nn** -- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used internally by descriptors** - -Operations accept `layout` (applies to all), or per-operand `layout_in1`, `layout_in2`, `layout_out`. - -### ChannelWiseTensorProduct - -Channel-wise tensor product: pairs channels of `x1` with channels of `x2`. - -```python -# With internal weights (default: shared_weights=True, internal_weights=True) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), # irreps_in1 - cue.Irreps("SO3", "0 + 1"), # irreps_in2 - layout=cue.mul_ir, - device="cuda", - dtype=torch.float32, -) -# tp.weight_numel -- number of weight parameters -# tp.irreps_out -- output irreps (auto-computed) - -x1 = torch.randn(batch, tp.irreps_in1.dim, device="cuda") -x2 = torch.randn(batch, tp.irreps_in2.dim, device="cuda") - -out = tp(x1, x2) # uses internal weight parameter -# out.shape == (batch, tp.irreps_out.dim) - -# With external weights (shared_weights=False) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), - cue.Irreps("SO3", "0 + 1"), - layout=cue.mul_ir, - shared_weights=False, - device="cuda", -) -w = torch.randn(batch, tp.weight_numel, device="cuda") -out = tp(x1, x2, weight=w) - -# With gather/scatter for graph neural networks -out = tp(x1, x2, weight=w, indices_1=senders, indices_out=receivers, size_out=num_nodes) -``` - -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. - -### FullyConnectedTensorProduct - -All input irrep pairs contribute to all output irreps (dense contraction). - -```python -tp = cuet.FullyConnectedTensorProduct( - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_in1 - cue.Irreps("O3", "0e + 1o"), # irreps_in2 - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) - -out = tp(x1, x2) # uses internal weights -# or: out = tp(x1, x2, weight=w) # external weights -``` - -Default method: `"fused_tp"`. - -### Linear - -Equivariant linear layer (weight-only, no second input). - -```python -linear = cuet.Linear( - cue.Irreps("SO3", "4x0 + 2x1"), # irreps_in - cue.Irreps("SO3", "3x0 + 5x1"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) - -out = linear(x) - -# Species-indexed weights (different weights per atom type) -linear = cuet.Linear( - irreps_in, irreps_out, - weight_classes=50, # 50 different weight sets - internal_weights=True, - device="cuda", -) -out = linear(x, weight_indices=species_indices) # species_indices: (batch,) int tensor -``` - -Default method: `"naive"`. Use `method="fused_tp"` for CUDA acceleration. - -### SymmetricContraction - -MACE-style symmetric contraction with element-indexed weights. - -```python -sc = cuet.SymmetricContraction( - cue.Irreps("O3", "32x0e + 32x1o"), # irreps_in (uniform mul required) - cue.Irreps("O3", "32x0e"), # irreps_out (uniform mul required) - contraction_degree=3, - num_elements=95, # number of chemical elements - layout=cue.ir_mul, - dtype=torch.float32, - device="cuda", -) - -# indices: (batch,) int tensor selecting which element weights to use -out = sc(x, indices) -# out.shape == (batch, irreps_out.dim) -``` - -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. - -### SphericalHarmonics - -```python -sh = cuet.SphericalHarmonics( - ls=[0, 1, 2, 3], - normalize=True, - device="cuda", -) - -vectors = torch.randn(batch, 3, device="cuda") -out = sh(vectors) -# out.shape == (batch, 1 + 3 + 5 + 7) -- sum of 2l+1 -``` - -Default method: `"uniform_1d"`. - -### Rotation and Inversion - -```python -# Rotation (SO3 or O3 irreps) -rot = cuet.Rotation( - cue.Irreps("SO3", "4x0 + 2x1 + 1x2"), - layout=cue.ir_mul, - device="cuda", -) - -# Euler angles (YXY convention) -gamma = torch.tensor([0.1], device="cuda") -beta = torch.tensor([0.2], device="cuda") -alpha = torch.tensor([0.3], device="cuda") -out = rot(gamma, beta, alpha, x) - -# Helper: encode angle for rotation -encoded = cuet.encode_rotation_angle(angle, ell=3) # cos/sin encoding - -# Helper: 3D vector to Euler angles -beta, alpha = cuet.vector_to_euler_angles(vector) - -# Inversion (O3 irreps only) -inv = cuet.Inversion( - cue.Irreps("O3", "4x0e + 2x1o"), - layout=cue.ir_mul, - device="cuda", -) -out = inv(x) -``` - -## Layers - -### BatchNorm - -Batch normalization for equivariant representations (adapted from e3nn). - -```python -bn = cuet.layers.BatchNorm( - cue.Irreps("O3", "4x0e + 4x1o"), - layout=cue.mul_ir, - eps=1e-5, - momentum=0.1, - affine=True, -) - -out = bn(x) # x.shape == (batch, irreps.dim) -``` - -### FullyConnectedTensorProductConv - -Message passing layer for equivariant GNNs (DiffDock-style). - -```python -conv = cuet.layers.FullyConnectedTensorProductConv( - in_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - sh_irreps=cue.Irreps("O3", "0e + 1o"), - out_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - mlp_channels=[16, 32, 32], - mlp_activation=torch.nn.ReLU(), - batch_norm=True, - layout=cue.ir_mul, -) - -# graph = ((src, dst), (num_src_nodes, num_dst_nodes)) -graph = ((src, dst), (num_src_nodes, num_dst_nodes)) - -out = conv(src_features, edge_sh, edge_emb, graph, reduce="mean") -# out.shape == (num_dst_nodes, out_irreps.dim) - -# Optional: separate scalar features for efficient first-layer GEMM -out = conv(src_features, edge_sh, edge_emb, graph, - src_scalars=src_scalars, dst_scalars=dst_scalars) -``` - -## Triangle operations (AlphaFold2-style) - -Require `cuequivariance_ops_torch`. - -```python -# Triangle attention with pair bias -kv_lengths = cuet.mask_to_kv_lengths(prefix_mask) -out = cuet.triangle_attention(q, k, v, bias, scale=scale, kv_lengths=kv_lengths) -# Arbitrary dense masks are supported through the fallback path: -out = cuet.triangle_attention(q, k, v, bias, mask=holey_mask, scale=scale) -# q, k, v: (B, N, H, Q/K, D), bias: (B, 1, H, Q, K) - -# Triangle multiplicative update -out = cuet.triangle_multiplicative_update( - x, # (B, I, J, C) - mask=mask, # (B, I, J) - precision=cuet.TriMulPrecision.DEFAULT, -) - -# Attention with pair bias (strict OpenFold3/Boltz contract) -out, _ = cuet.attention_pair_bias( - single_repr, pair_repr, mask, num_heads, - w_ln_a, b_ln_a, - w_proj_q, b_proj_q, w_proj_k, w_proj_v, - w_proj_g, w_proj_o, w_proj_z, - w_ln_z=w_ln_z, b_ln_z=b_ln_z, -) -# Optional generalized projection params are keyword-only: -# b_proj_k, b_proj_v, w_ln_q, b_ln_q, w_ln_k, b_ln_k, b_proj_g (default None), -# and norm_kind (default "layer_norm"). -``` - -## ONNX and TensorRT export - -```python -# ONNX export -table = cuet.onnx_custom_translation_table() -onnx_program = torch.onnx.export(model, inputs, custom_translation_table=table) - -# TensorRT plugin registration -cuet.register_tensorrt_plugins() -``` - -## Complete GNN example - -```python -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet - -class SimpleGNN(torch.nn.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out): - super().__init__() - self.tp = cuet.ChannelWiseTensorProduct( - irreps_in, irreps_sh, layout=cue.mul_ir, - shared_weights=False, device="cuda", - ) - self.linear = cuet.Linear( - self.tp.irreps_out, irreps_out, - layout=cue.mul_ir, internal_weights=True, device="cuda", - ) - self.sh = cuet.SphericalHarmonics( - ls=[ir.l for _, ir in irreps_sh], normalize=True, device="cuda", - ) - - def forward(self, node_feats, edge_vec, edge_index, num_nodes): - src, dst = edge_index - edge_sh = self.sh(edge_vec) - w = torch.randn(1, self.tp.weight_numel, device=node_feats.device) - - # Message: tensor product on edges with scatter to destination nodes - messages = self.tp( - node_feats, edge_sh, weight=w, - indices_1=src, indices_2=None, - indices_out=dst, size_out=num_nodes, - ) - return self.linear(messages) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `SegmentedPolynomial` | `cuequivariance_torch/primitives/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_torch/primitives/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_torch/primitives/segmented_polynomial_naive.py` | -| `fused_tp` backend | `cuequivariance_torch/primitives/segmented_polynomial_fused_tp.py` | -| `indexed_linear` backend | `cuequivariance_torch/primitives/segmented_polynomial_indexed_linear.py` | -| `ChannelWiseTensorProduct` | `cuequivariance_torch/operations/tp_channel_wise.py` | -| `FullyConnectedTensorProduct` | `cuequivariance_torch/operations/tp_fully_connected.py` | -| `Linear` | `cuequivariance_torch/operations/linear.py` | -| `SymmetricContraction` | `cuequivariance_torch/operations/symmetric_contraction.py` | -| `SphericalHarmonics` | `cuequivariance_torch/operations/spherical_harmonics.py` | -| `Rotation` / `Inversion` | `cuequivariance_torch/operations/rotation.py` | -| `BatchNorm` | `cuequivariance_torch/layers/batchnorm.py` | -| `FullyConnectedTensorProductConv` | `cuequivariance_torch/layers/tp_conv_fully_connected.py` | -| Triangle operations | `cuequivariance_torch/primitives/triangle.py` | -| Layout transposition | `cuequivariance_torch/primitives/transpose.py` | diff --git a/cuequivariance_torch/cuequivariance_torch/SKILL.md b/cuequivariance_torch/cuequivariance_torch/SKILL.md new file mode 120000 index 00000000..e2451973 --- /dev/null +++ b/cuequivariance_torch/cuequivariance_torch/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance-torch/SKILL.md \ No newline at end of file diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 00000000..49372963 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,293 @@ +# Agent skill maintenance + +Each immediate child directory containing a `SKILL.md` is one independent skill. +NVSkills CI and nv-carps discover these paths: + +```text +skills/ + cuequivariance/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt + cuequivariance-jax/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt + cuequivariance-torch/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt +``` + +Edit the regular files here. The Python package resources at +`//SKILL.md` are relative symlinks back to these files, keeping +`python -m skill` compatible without maintaining duplicate instructions. +Do not replace the canonical files with symlinks, or nest one skill inside another. +Each skill can be distributed independently; its source-location tables explain +how to find code in a checkout or an installed package. + +## Evaluation coverage + +Each regression dataset (`evals/evals.json`) contains five positive cases and one +unrelated negative case. The JAX and PyTorch skills also have a separate +`evals/challenges.json` capability dataset; the original six cases are unchanged. +The core cases were adapted from the BioNeMo agent toolkit's cuEquivariance +evaluation ideas; they use this repository's APIs and contain no references to +BioNeMo-only skill resources. The JAX and PyTorch cases exercise their respective +frontends rather than reusing the core skill's expected routing. + +| Skill | Positive cases | +| --- | --- | +| Core | Complex Z3 irreps and CG covariance; full SO3 tensor product; split/dense channelwise agreement; layout conversion and rotation; older-API fallback without installing packages | +| JAX | Broadcasting, JIT and finite-difference gradients; dictionary layout conversion; NNX linear normalization and rotation; split polynomial execution; CPU spherical harmonics and backend choice | +| PyTorch | Spherical harmonics and gradcheck; linear layouts and rotation; external per-example weights; gather/scatter with repeated receivers and gradients; CPU backend fallback | + +`evals/config.yml` requests an isolated skill workspace and a rebuilt Harbor +environment. In SkillEvaluator 1.5.6, Docker task generation installs +`scripts/requirements.txt`. The `k8s-sandbox` path instead stages skill files into +an existing sandbox; `base_image_mode: rebuild` alone does **not** install these +dependencies there. Verify the actual interpreter and package versions in each +arm before treating a run as an evaluation of the pinned environment. Provision +dependencies equally for both arms and establish a new baseline when changing +the environment. JAX uses `harbor.pre_agent_setup`; Torch uses the native Harbor +environment healthcheck described below. + +The JAX and PyTorch setup commands publish `/workspace/.venv` only after the +pinned packages pass a CPU smoke check. Failed environments remain at their +unique `.cueq-env.*` paths for diagnosis; the next attempt creates a fresh one. +An existing `.venv`, including a broken symlink, is preserved and must pass the +check without being replaced. Setup uses binary wheels and skips bytecode +compilation. Torch installs its pinned CPU build with `--no-deps` from +[PyTorch's alternate CPU index](https://download-r2.pytorch.org/whl/cpu/torch/). +Pip selects the compatible wheel for the host interpreter and architecture. +This avoids the original index host that failed certificate verification in +Astra while retaining HTTPS verification and the configured `PIP_CERT` bundle. +A second install resolves Torch's dependencies and cuEquivariance from the normal index, +avoiding dependency-metadata incompatibilities with older host pip versions. +Both arms use the same setup and package pins. + +SkillEvaluator 1.5.6 hardcodes 120 seconds for generated `pre_agent_setup` +healthchecks. Torch selects `harbor.task_source: native_harbor` so its repo-owned +`evals/harbor/*/task.toml` files can give installation and the smoke check +600 seconds. This uses the unmodified evaluator's supported native-task path. +The agent and verifier budgets remain 300 and 180 seconds, respectively, and +grading remains `aces_default`. Both arms still run the same six prompts from +`evals/evals.json`. Setup failures remain failures, and Harbor collects +`/workspace/cueq-setup.log` as a task artifact for diagnosis. + +`evals/setup_cpu.sh` is the canonical Torch bootstrap script. Its contents and +the prompts are embedded in the native tasks so setup also runs in the baseline, +where the skill itself is absent. After changing the script or dataset, refresh +and check the generated files: + +```bash +python .github/scripts/prepare_skill_eval_tasks.py +python .github/scripts/prepare_skill_eval_tasks.py --check +``` + +The setup regression tests check synchronization and that capturing logs does +not hide a failing exit status. A successful local setup does not establish that +the shared Astra sandbox is healthy; inspect the live preflight result before +interpreting a missing Torch score as a skill-quality failure. + +The requirements pin the cuEquivariance release or source revision whose APIs +were checked, and pin the JAX/Flax or CPU PyTorch versions. +They do not require sibling skills or a repository checkout inside the sandbox. +When updating the API baseline, update the pins and run the numerical checks +before changing the assertions. The CUDA backend constraints are checked as +guidance; these datasets do not benchmark GPU performance or validate CUDA kernels. + +The tests under `.github/tests/test_*_skill_evals.py` verify the mathematical and +API expectations behind the prompts. `test_jax_skill_examples.py` executes each +published CPU example independently and checks layout conversion with duplicate +irreps and split polynomials with additional batch axes, parity, and JIT. +`test_torch_skill_examples.py` executes each published CPU recipe independently, +including tensor-product backward, and checks the GNN's determinism, +equivariance, and gradients. These are maintenance checks, +not live agent scores. To run them against +a development checkout with the relevant frontend dependencies installed: + +```bash +pytest -q .github/tests/test_core_skill_evals.py +JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_jax_skill_challenges.py .github/tests/test_skill_challenge_packaging.py +pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py .github/tests/test_torch_skill_challenges.py +``` + +To verify the pinned evaluation environment in a separate Python 3.12 environment: + +```bash +python -m pip install -r skills/cuequivariance/scripts/requirements.txt \ + -r skills/cuequivariance-jax/scripts/requirements.txt \ + -r skills/cuequivariance-torch/scripts/requirements.txt pytest +JAX_PLATFORMS=cpu pytest -q .github/tests +``` + +The request workflow delegates live evaluation to NVSkills CI. With the same +SkillEvaluator version as the service (1.5.6 when this layout was introduced), +the dataset/config preflight can also be run locally: + +```bash +skillevaluator tier3 validate skills/cuequivariance +skillevaluator tier3 validate skills/cuequivariance-jax +skillevaluator tier3 validate skills/cuequivariance-torch +``` + +### Challenge-v1 capability pilot + +The challenge datasets contain two positive integration tasks and one nearby +negative task per frontend. Their IDs deliberately differ from the regression +IDs so results from the two datasets cannot be paired accidentally. + +| Case suffix | Required outcome | +| --- | --- | +| `challenge-layout-v1` | Migrate original O3 `ir_mul` buffers with separated duplicate blocks and both parities into dictionaries, reconstruct the original order, and retain gradients, batch axes and transformation behavior. JAX also requires JIT; Torch includes noncontiguous tensors. | +| `challenge-graph-v1` | Compose radial weights, normalized harmonics, a channelwise product, destination accumulation and a projection. Match an independent NumPy edge-loop reference before and after a training update, retain gradients through every parameter and both floating inputs, and handle relabeling, edge permutations and isolated destinations. | +| `challenge-routing-v1` | Repair an ordinary JAX/Torch regression-loss broadcasting bug without invoking cuEquivariance or its skill. | + +Prompts specify the public interface, mathematics, constraints and observable +outcomes. They leave the frontend APIs and implementation strategy to the agent. +Every task writes an importable `/workspace/solution.py`; execution claims remain +subject to the ordinary trajectory checks. + +`.github/evals/cueq_challenges.py` independently executes the submitted artifact +in a bounded subprocess using the prepared CPU interpreter. The positive +oracles use original block slicing and core NumPy descriptors rather than the +submitted frontend forward. Checks include multiple data seeds, independent +forward values, finite differences, rotations, O3 inversion where applicable, +graph resizing and permutations, and a real parameter update. +It emits a binary `numerical_success` custom metric: all required numerical +checks must pass. A missing file, exception, detached gradient or timeout fails. +The nearby negative additionally checks the loss values and prediction gradients; +the ACES routing check detects skill activation beyond imports in the submission. + +The maintainer references (`.github/evals/reference_*_challenges.py`) are outside +all skill directories and are never included in the pilot. Regression tests run +the verifier against those references and deliberate defects: overwritten +duplicate irreps, wrong layouts, parity merging, incorrectly shared edge data, +detached vectors, mean instead of sum aggregation, and the original MSE +broadcasting bug. These tests validate the benchmark and verifier, not agent lift. + +Prepare a fresh pilot with Python, PyYAML and SkillEvaluator 1.5.6 available: + +```bash +python .github/scripts/prepare_skill_challenges.py --export-harbor +``` + +This creates a unique directory under `.venv/` and prints a manifest location +and two evaluation commands. `--backend jax` or `--backend torch` selects one +frontend. `--output PATH` selects a new output directory; existing directories +are refused. Without `--export-harbor`, preparation needs only Python and PyYAML. + +The staged skills retain the exact canonical `SKILL.md`, requirements and +environment setup. Only the selected dataset and grading mode change: +`aces_plus_custom` adds artifact verification while retaining the default ACES +metrics and quality gate. **`numerical_success` does not replace ACES's overall +score or change its gate.** Read both the numerical result and the normal +trajectory assessment; neither should be inferred from the other. +The Torch pilot renders native tasks from its challenge IDs and prompts and +uses the same 600-second setup budget. Its manifest also records the bootstrap +script and native task file digests. + +The export validates both conditions, identical prompts/verifiers, and the +absence of eval metadata, grader code and reference solutions in the agent's +runtime bundle. This requires the builder's private evaluator-source support; +older builders are rejected. Do not use `--copy-repo`: the maintenance checkout +contains the reference answers. + +The manifest records skill, dataset, configuration and verifier digests. Its +commands pin the models from the 2026-10-20 comparison: +`claude-code=aws/anthropic/bedrock-claude-opus-4-8` and +`codex=openai/openai/gpt-5.5`. They request baseline plus skill, three attempts +per case and no early stop on success: 36 executions per frontend, 72 for both. +Confirm those models are still available in the authenticated environment; +changing a model requires a new baseline with that same model. +The preparation command only stages and validates; it does not launch agents. +Before running the printed commands in the authenticated CI/runtime environment: + +```bash +skillevaluator tier3 doctor --env-mode k8s-sandbox --agents claude-code,codex +``` + +Keep all attempts, infrastructure failures and trial artifacts. Fix the skill +text only after this first paired pilot, then measure both revisions on the +same frozen challenge dataset and environment. Reserve additional numerical +variants for later validation. Do not compare challenge lift directly with the +earlier six-case regression lift, or interpret local reference passes as evidence +of agent improvement. The ordinary NVSkills workflow continues to discover +`evals/evals.json`; promotion of the new benchmark follows the pilot. + +### Interpreting Tier 3 comparisons + +Use the exported JSON and per-trial evidence to compare revisions. The pipeline +gate, static quality score, execution coverage, and skill uplift answer different +questions; a green pipeline does not establish positive uplift. + +- Check `tier3.coverage` first. Compare the same eligible agents and models across + revisions. An agent excluded for `agent_process_exit` has no valid quality + score; a displayed zero is not a measured skill regression. Exit 137 alone + does not identify whether memory pressure, timeout, or another kill caused it. +- Use the precision recorded in `tier3.quality`. With default grading in 1.5.6, gate + lift averages the with-skill minus baseline differences for the shared metrics + `security`, `accuracy`, `goal_accuracy`, and `behavior_check`. Skill execution + and tool productivity are N/A for baseline; token efficiency is a separate + diagnostic and is outside that gate calculation. +- Pair trials by case ID. Inspect security evidence and numerical/workflow + checks before using aggregate recommendations. Record total and productive + tool calls plus uncached prompt and completion tokens to locate setup and + exploration overhead. +- Hold the dataset, dependency versions, environment, agent models, evaluator, + thresholds, and attempt policy fixed for a skill-only comparison. Capture + package versions from the actual run: requirements files alone are insufficient. + Preserve all attempts and infrastructure failures when repeating a comparison. +- One attempt per case is exploratory evidence. Repeat both arms before claiming + a small improvement is reliable; do not select only the best rerun. Keep new + evaluation cases in a separately identified benchmark until both revisions + have been measured on it. + +## Packaging + +The release workflow materializes symlinks **before** running Hatchling, including +the existing LICENSE, README, VERSION and attribution links. Keep this step: +building an sdist directly from the linked development tree can leave links whose +targets are outside that distribution. Release wheels and sdists must carry a +regular `SKILL.md` resource. + +The ordinary CPU test workflow runs the packaging regression check for each +package. To check all three locally with their runtime dependencies and Hatchling +installed: + +```bash +python .github/scripts/check_skill_packaging.py +``` + +The script builds from a temporary materialized copy, checks the source CLI, +inspects wheel and sdist resources, rebuilds a wheel from each sdist, and verifies +the CLI imports and prints the resource from each wheel. It leaves the checkout's +symlinks intact. Use `--package cuequivariance_jax` (or either other Python package +name) to check one package. + +## Service onboarding and publication + +The source workflow is `.github/workflows/request-nvskills-ci.yml`, using the +public reusable workflow in `NVIDIA/skills`. Activation also requires: + +1. An enabled `NVIDIA/cuEquivariance` entry in + `NVIDIA/nvskills-ci/config/onboarded-repositories.json`, with the `external` + profile and `main` as an allowed PR base. +2. The NVSkills GitHub App installed with access to this repository, and the + repository secret `NVSKILLS_CI_DISPATCH_TOKEN` provisioned by the CI owners. + Protect the request workflow with the repository's normal maintainer review. +3. The request workflow on the default branch so `issue_comment` events run. + A maintainer can then comment `/nvskills-ci` on a same-repository PR that + changes these skills. Fork PRs require a separate central policy opt-in. +4. The nv-carps standalone-skill catalog fix: Tier 2 should use the `skills/` + catalog rather than traversing unrelated Python package symlinks. Team/plugin + catalogs keep their existing handling. Cross-tree duplicate comparison is + outside this standalone catalog's scope. +5. Successful service validation and genuine generated signatures, skill cards, + and benchmarks. Only then merge the three-path registration in + `NVIDIA/skills/components.d/cuequivariance.yml` and let catalog sync publish it. + +The component registry must point at each of the three individual skill +directories, with matching catalog directory names. Do not manufacture generated +validation artifacts to satisfy catalog onboarding checks. diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md new file mode 100644 index 00000000..9712e366 --- /dev/null +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -0,0 +1,162 @@ +# Skill Benchmark: cuequivariance-jax + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `cuequivariance-jax` +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:f653a01cf39e3dffe96fbacf75b1b1ee214a79c4fc5b682f49f31e6963d7bb1a` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance-jax | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance-jax`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 99.6% — baseline ran, but no comparable score was available; uplift unavailable | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 points) | 79.6% → 90.0% (+10.4 points) | +| Efficiency | 80.7% — baseline ran, but no comparable score was available; uplift unavailable | 70.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 93.8% (+1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.5% → 86.6% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,124,797 | 1,645,841 | -521,044 | -31.66% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 171,366 | 230,492 | -59,126 | -25.65% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 217,234 | 228,341 | -11,107 | -4.86% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 216,567 | 186,564 | +30,003 | +16.08% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 265,945 | 574,629 | -308,684 | -53.72% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 218,098 | 390,218 | -172,120 | -44.11% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 35,587 | 35,597 | -10 | -0.03% | skill 1/1; base 1/1 | +| codex | All cases | 1,212,345 | 1,132,654 | +79,691 | +7.04% | skill 6/6; base 6/6 | +| codex | jax-001 | 365,435 | 283,409 | +82,026 | +28.94% | skill 1/1; base 1/1 | +| codex | jax-002 | 140,879 | 167,413 | -26,534 | -15.85% | skill 1/1; base 1/1 | +| codex | jax-003 | 194,388 | 216,005 | -21,617 | -10.01% | skill 1/1; base 1/1 | +| codex | jax-004 | 300,086 | 409,921 | -109,835 | -26.79% | skill 1/1; base 1/1 | +| codex | jax-005 | 198,193 | 42,608 | +155,585 | +365.15% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,364 | 13,298 | +66 | +0.50% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,337,142 | 2,778,495 | -441,353 | -15.88% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 2 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance-jax': 133 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance-jax/SKILL.md b/skills/cuequivariance-jax/SKILL.md new file mode 100644 index 00000000..b6754d00 --- /dev/null +++ b/skills/cuequivariance-jax/SKILL.md @@ -0,0 +1,348 @@ +--- +name: cuequivariance-jax +description: Use when writing or debugging cuEquivariance JAX polynomials, irrep dictionaries, Flax NNX layers, or CPU/CUDA execution and layouts. +license: Apache-2.0 +metadata: + author: Mario Geiger + tags: [jax, equivariance, tensor-products] +--- + +# cuEquivariance in JAX + +## Purpose + +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_jax` (`cuex`). Preserve the project's framework and devices. Custom representations use core `cuequivariance`; PyTorch modules use `cuequivariance_torch`. + +## Inputs + +Required inputs: operation, SO3/O3 irreps, layout, batch shapes, dtype, device, and weight ownership. Derive them from the request/code; state defaults. Graphs also need indices and destination count. Operand sizes/output irreps come from descriptors. Optional inputs: backend/performance preferences. + +## Instructions + +1. Probe the project interpreter once and reuse working packages. **Environment setup/recovery must not recursively delete directories**, including temporary environments. Use a fresh unique path when isolation is needed. +2. Select `method="naive"` for CPU; Flax is needed only for NNX. High-level dictionary blocks are `(..., mul, ir.dim)`; uniform CUDA arrays differ. +3. Adapt one matching recipe into a script. Build descriptors outside transformed functions, broadcast leading input axes, and enable x64 **before array creation** for float64 checks. +4. Finish installation before running the saved script with the completion recipe below. A running session/task ID, empty output, or a file write is not a completed check. Poll the same session until it exits; then read its results before reporting success. +5. Investigate a concrete failure using its error and the affected installed API. Rerun affected checks after a fix. Once the requested checks pass, finish; avoid repeated probes, unrelated examples, or exploration of other backends for a CPU reference. + +## Prerequisites and environment recovery + +Probe imports independently so a missing frontend does not hide working JAX. For an NNX task, include `"flax.nnx"` in the same loop: + +```python +import importlib +import sys + +print("python:", sys.executable, sys.version.split()[0]) +available, import_errors = {}, {} +for name in ("jax", "cuequivariance", "cuequivariance_jax"): + try: + module = importlib.import_module(name) + available[name] = module + print(name, getattr(module, "__version__", "unknown"), module.__file__) + except (ImportError, OSError) as error: + import_errors[name] = f"{type(error).__name__}: {error}" +print("import errors:", import_errors) +if "jax" in available: + print("devices:", available["jax"].devices()) +``` + +Reuse working packages; install only missing dependencies when needed. Honor the project's lockfile or supplied `scripts/requirements.txt` and compatible core/frontend revisions. JAX 0.11.1 / Flax 0.12.10 require **Python >=3.12**: select a compatible interpreter for those pins. The standalone combination below also supports Python 3.11; it does not replace project pins. Imports alone do not establish compatibility: cuEquivariance 0.13.0 with JAX 0.12.0 fails polynomial execution with a `segmented_polynomial_dce` argument-count error. + +For isolation, keep the unique directory returned by `mktemp`; reuse it for reruns. This Linux recipe uses host pip 22.3+ and bypasses `ensurepip`: + +```bash +set -e +cueq_python=python3 # Set to the selected project interpreter's path. +cueq_jax_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-jax.XXXXXX") +"$cueq_python" -m venv --without-pip "$cueq_jax_env" +python3 -m pip --python "$cueq_jax_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + "jax==0.10.2" "cuequivariance==0.13.0" "cuequivariance-jax==0.13.0" +cueq_python="$cueq_jax_env/bin/python" +JAX_PLATFORMS=cpu "$cueq_python" -c 'import jax, cuequivariance_jax; print("setup complete:", jax.__version__, cuequivariance_jax.__version__, jax.devices())' +``` + +Replace the package list with `-r` and the requirements path when prescribed; otherwise add `flax==0.12.7` to the same install command only for NNX. Keep this interpreter for execution; shell variables may need restoring in a new tool call. Without host pip, use an installed `uv`. The pip command uses the Linux system CA bundle; for a private CA use the environment's approved `PIP_CERT` bundle. Retain TLS verification. Diagnose the actual error before retrying; never reset directories or switch versions while an install is still running. If no compatible interpreter or trusted package source is available, report the blocker and unrun checks. See [pip's certificates](https://pip.pypa.io/en/stable/topics/https-certificates/). + +### Complete and inspect a run + +Save the preamble and relevant checks in `check.py`. Set `cueq_python` to the probe's `sys.executable`, or retain it from setup. Run **after setup exits successfully**; a fresh directory records output and status: + +```bash +set -e +cueq_run=$(mktemp -d "${TMPDIR:-/tmp}/cueq-check.XXXXXX") +printf 'run directory: %s\n' "$cueq_run" +if JAX_PLATFORMS=cpu "$cueq_python" -u check.py >"$cueq_run/output.txt" 2>&1; then + cueq_status=0 +else + cueq_status=$? +fi +printf '%s\n' "$cueq_status" >"$cueq_run/exit-code.txt" +tail -n 30 "$cueq_run/output.txt" +printf 'exit code: %s\n' "$cueq_status" +test "$cueq_status" -eq 0 +``` + +If the tool yields, keep polling its session; the exit-code file is written only after Python finishes. Inspect that file and the numerical output. Empty output is not evidence that assertions ran. Deliver the script, actual command, and observed checks; label anything unrun. Keep setup output separate from the numerical results. + +## Limitations and backend selection + +| Route | Supported use | +| --- | --- | +| `segmented_polynomial(..., method="naive")` | Portable JAX execution, including CPU, JIT, gradients, and vmap. | +| `cuex.nnx.IrrepsLinear` | CPU/GPU channel mixing within matching irreps using JAX einsum. | +| `cuex.nnx.SphericalHarmonics` | CPU/GPU; normalizes vectors and uses the naive polynomial implementation. | +| `method="uniform_1d"` | CUDA; uniform segment shapes within each operand and a single mode after preparation. | +| `method="indexed_linear"`, `cuex.nnx.IrrepsIndexedLinear` | CUDA indexed linear route, using `cuex.Repeats`; not the ordinary CPU linear layer. | + +CPU routes need no GPU dependencies; moving arrays to CPU does not change the method. CUDA requires compatible JAX/GPU/kernel builds. NNX spherical harmonics normalize vectors: use positive `eps` if zero vectors are possible. + +## Examples + +Use this preamble with one recipe and run with `JAX_PLATFORMS=cpu`. Nonconstant inputs expose layout errors; prints follow successful assertions. + +```python +import jax +jax.config.update("jax_enable_x64", True) +import jax.numpy as jnp +import numpy as np +import cuequivariance as cue +import cuequivariance_jax as cuex +from cuequivariance_jax import ir_dict + +dtype = jnp.float64 +print("devices:", jax.devices()) +``` + +### Spherical harmonics without Flax + +This evaluates the **raw polynomial**; normalize vectors for unit-direction harmonics. O3 keys retain parity; SO3 keys do not. + +```python +desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2]) +vectors = jnp.array([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], dtype=dtype) +values = cuex.segmented_polynomial( + desc.polynomial, [vectors], + [jax.ShapeDtypeStruct(vectors.shape[:-1] + (op.size,), dtype) + for op in desc.polynomial.outputs], + method="naive", +) +(irreps_out,) = desc.output_irreps +harmonics = { + ir: value.reshape(vectors.shape[:-1] + (mul, ir.dim)) + for (mul, ir), value in zip(irreps_out, values) +} +for actual, expected in zip(values, desc.polynomial(np.asarray(vectors))): + np.testing.assert_allclose(actual, expected, rtol=1e-12, atol=1e-12) +assert {ir.l: v.shape for ir, v in harmonics.items()} == { + 0: (2, 1, 1), 1: (2, 1, 3), 2: (2, 1, 5), +} +print({str(ir): (v.shape, v.devices()) for ir, v in harmonics.items()}) +``` + +### Dictionary layouts and exact round trips + +`flat_to_dict(..., layout="ir_mul")` produces `(..., mul, ir.dim)` blocks. **`dict_to_flat` has no layout argument and emits `mul_ir`.** Return to `ir_mul` by transposing each block, not reshaping the whole array. + +Explain the **unique-irrep key precondition**, even for regrouped inputs. Regroup duplicate blocks **with their data** using `RepArray.regroup()`; changing only metadata corrupts alignment. The round trip below recovers the regrouped buffer. Recovering an ungrouped order also needs its original block arrangement. + +```python +def ir_mul_to_dict(irreps, flat): + # Merge duplicate irrep keys and reorder their data together. + data = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() + return data.irreps, ir_dict.flat_to_dict(data.irreps, data.array, layout="ir_mul") + +def dict_to_ir_mul_flat(irreps, blocks): + return jnp.concatenate([ + blocks[ir].swapaxes(-1, -2).reshape( + blocks[ir].shape[:-2] + (mul * ir.dim,) + ) + for mul, ir in irreps + ], axis=-1) + +irreps = cue.Irreps("SO3", "3x0 + 2x1").regroup() +flat_ir = jnp.arange(2 * irreps.dim).reshape(2, irreps.dim) +irreps, blocks = ir_mul_to_dict(irreps, flat_ir) +offset = 0 +for mul, ir in irreps: + size = mul * ir.dim + expected = np.asarray(flat_ir[..., offset:offset + size]).reshape(2, ir.dim, mul) + assert blocks[ir].shape == (2, mul, ir.dim) + np.testing.assert_array_equal(blocks[ir], expected.swapaxes(-1, -2)) + offset += size +flat_mul = ir_dict.dict_to_flat(irreps, blocks) +restored_blocks = ir_dict.flat_to_dict(irreps, flat_mul, layout="mul_ir") +for ir in blocks: + np.testing.assert_array_equal(restored_blocks[ir], blocks[ir]) +np.testing.assert_array_equal(dict_to_ir_mul_flat(irreps, restored_blocks), flat_ir) +print("regrouped unique irrep keys; exact ir_mul/mul_ir round trip:", + {str(ir): v.shape for ir, v in blocks.items()}) +``` + +`ir_dict.assert_mul_ir_dict` checks block conventions. `mul_ir_dict(irreps, data)` broadcasts a template to keys without inferring feature dimensions. + +### Dense polynomial: broadcasting, JIT, and a gradient + +`segmented_polynomial` takes/returns lists. Channelwise products put weights first; shared weights need **no batch axis**. Broadcast leading input axes. `filter_irreps_out` selects types, not multiplicities; `e.inputs`/`e.outputs` carry dense operand layouts. + +```python +e = cue.descriptors.channelwise_tensor_product( + cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1"), + [cue.SO3(0), cue.SO3(1)], +) +poly = e.polynomial +rng = np.random.default_rng(7) +w = jnp.asarray(rng.normal(size=poly.inputs[0].size)) +x = jnp.asarray(rng.normal(size=(3, 1, poly.inputs[1].size))) +y = jnp.asarray(rng.normal(size=(1, 2, poly.inputs[2].size))) + +def run(weights, a, b): + batch = np.broadcast_shapes(weights.shape[:-1], a.shape[:-1], b.shape[:-1]) + return cuex.segmented_polynomial( + poly, [weights, a, b], + [jax.ShapeDtypeStruct(batch + (op.size,), jnp.result_type(weights, a, b)) + for op in poly.outputs], + method="naive", + )[0] + +reference = e(*(np.asarray(v) for v in (w, x, y)))[0] +eager, compiled = run(w, x, y), jax.jit(run)(w, x, y) +np.testing.assert_allclose(eager, reference, rtol=1e-12, atol=1e-12) +np.testing.assert_allclose(compiled, reference, rtol=1e-12, atol=1e-12) +assert reference.shape == (3, 2, poly.outputs[0].size) +gradient = jax.grad(lambda v: jnp.sum(run(v, x, y) ** 2))(w) +index = int(jnp.argmax(jnp.abs(gradient))) +assert abs(gradient[index]) > 1e-8 +epsilon = 1e-5 +loss_plus = np.sum(e(np.asarray(w.at[index].add(epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) +loss_minus = np.sum(e(np.asarray(w.at[index].add(-epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) +finite_difference = (loss_plus - loss_minus) / (2 * epsilon) +np.testing.assert_allclose(gradient[index], finite_difference, rtol=1e-6, atol=1e-7) +print("shape:", eager.shape, "eager/JIT max errors:", + float(jnp.max(abs(eager - reference))), float(jnp.max(abs(compiled - reference)))) +print("weight gradient / finite difference:", float(gradient[index]), finite_difference) +``` + +For gather/scatter, supply one `indices` entry per operand, inputs then outputs; `None` means unindexed. Output indices need a destination shape; repeated destinations accumulate. Verify against explicit edgewise summation. + +### Split irrep polynomials on CPU + +`channelwise_tensor_product_ir_dict` returns an `IrDictPolynomial`. Follow `input_irreps` order: weights, features, second input; each `(mul, ir)` is one operand. Follow `output_irreps` for outputs. These descriptors use `ir_mul`: transpose blocks before flattening; reverse on outputs. There is no `ir_dict.equivariant_polynomial` API. + +```python +def split_naive(desc, groups): + assert len(groups) == len(desc.input_irreps) + operands = [ + group[ir].swapaxes(-1, -2).reshape(group[ir].shape[:-2] + (mul * ir.dim,)) + for irreps, group in zip(desc.input_irreps, groups) + for mul, ir in irreps + ] + batch = np.broadcast_shapes(*(v.shape[:-1] for v in operands)) + outputs = cuex.segmented_polynomial( + desc.polynomial, operands, + [jax.ShapeDtypeStruct(batch + (op.size,), jnp.result_type(*operands)) + for op in desc.polynomial.outputs], + method="naive", + ) + values = iter(outputs) + return tuple({ + ir: next(values).reshape(batch + (ir.dim, mul)).swapaxes(-1, -2) + for mul, ir in irreps + } for irreps in desc.output_irreps) + +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1"), + [cue.SO3(0), cue.SO3(1)], +) +rng = np.random.default_rng(8) +groups = [ + {ir: jnp.asarray(rng.normal(size=batch + (mul, ir.dim))) for mul, ir in irreps} + for irreps, batch in zip(desc.input_irreps, [(), (3, 1), (1, 2)]) +] +(result,) = split_naive(desc, groups) +numpy_inputs = [ + np.asarray(group[ir]).swapaxes(-1, -2).reshape(group[ir].shape[:-2] + (mul * ir.dim,)) + for irreps, group in zip(desc.input_irreps, groups) for mul, ir in irreps +] +expected = desc.polynomial(*numpy_inputs) +for (mul, ir), flat in zip(desc.output_irreps[0], expected): + assert result[ir].shape == (3, 2, mul, ir.dim) + reference = flat.reshape(3, 2, ir.dim, mul).swapaxes(-1, -2) + np.testing.assert_allclose(result[ir], reference, rtol=1e-12, atol=1e-12) + print("output:", str(ir), result[ir].shape, "max error:", + float(jnp.max(abs(result[ir] - reference)))) +``` + +### Flax NNX layers + +`IrrepsLinear` requires regrouped irreps and `nnx.Rngs`. It mixes matching-irrep channels with `scale / sqrt(mul_in)` normalization; unmatched outputs are zero. Read `nnx.Param` weights with `[...]`. Row vectors rotate as `x @ D.T`. + +```python +from flax import nnx +from cuequivariance_jax import nnx as cue_nnx + +irreps_in = cue.Irreps("SO3", "3x0 + 2x1").regroup() +irreps_out = cue.Irreps("SO3", "2x0 + 4x1 + 2").regroup() +linear = cue_nnx.IrrepsLinear(irreps_in, irreps_out, scale=0.7, dtype=dtype, rngs=nnx.Rngs(9)) +rng = np.random.default_rng(10) +features = {ir: jnp.asarray(rng.normal(size=(2, mul, ir.dim))) for mul, ir in irreps_in} +output = linear(features) +for mul, ir in irreps_out: + assert output[ir].shape == (2, mul, ir.dim) +np.testing.assert_array_equal(output[cue.SO3(2)], 0) +for mul, ir in irreps_in: + factor = linear.scale / np.sqrt(mul) + expected = factor * np.einsum( + "uv,...ui->...vi", np.asarray(linear.w[str(ir)][...]), features[ir]) + np.testing.assert_allclose(output[ir], expected, rtol=1e-12, atol=1e-12) + print("irrep:", str(ir), "scale/sqrt(mul_in):", factor, + "reference max error:", float(jnp.max(abs(output[ir] - expected)))) +rotation = np.array([0.3, -0.2, 0.1]) +rotated_output = linear({ir: v @ ir.exp_map(rotation, []).T for ir, v in features.items()}) +for ir, value in output.items(): + expected = value @ ir.exp_map(rotation, []).T + np.testing.assert_allclose(rotated_output[ir], expected, rtol=1e-12, atol=1e-12) + print("output:", str(ir), value.shape, "rotation max error:", + float(jnp.max(abs(rotated_output[ir] - expected)))) +print("unmatched l=2 max magnitude:", float(jnp.max(abs(output[cue.SO3(2)])))) +``` + +`IrrepsNormalize(eps=..., scale=..., skip_scalars=...)` uses squared component norms averaged over multiplicities. `MLP(layer_sizes=..., activation=..., dtype=..., rngs=...)` uses scalar features and fan-in normalization. CUDA indexed-linear species counts must match grouped rows and sum to batch size. + +### Dense RepArray interface + +Use descriptor representation metadata. A single output returns one `RepArray`: `.array` holds data; `.reps` maps axes to representations. + +```python +e = cue.descriptors.fully_connected_tensor_product( + cue.Irreps("SO3", "2x0 + 1"), cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 2x1"), +) +keys = jax.random.split(jax.random.key(11), len(e.inputs)) +inputs = [cuex.randn(key, rep, (2,), dtype) for key, rep in zip(keys, e.inputs)] +output = cuex.equivariant_polynomial(e, inputs, method="naive") +np.testing.assert_allclose(output.array, e(*(np.asarray(v.array) for v in inputs))[0], + rtol=1e-12, atol=1e-12) +print("RepArray shape:", output.shape, "devices:", output.array.devices()) +``` + +## CUDA preparation and message passing + +Prepare compatible dense descriptors with `e.squeeze_modes().flatten_coefficient_modes()` before `equivariant_polynomial(..., method="uniform_1d")`. Segment shapes must be uniform within each operand. Splitting by irrep still requires layout conversion. + +`ir_dict.segmented_polynomial_uniform_1d(poly, inputs, outputs, input_indices=..., output_indices=...)` needs leaves shaped `(..., op.num_segments, *op.segment_shape)`, distinct from `(..., mul, ir.dim)`. For `_ir_dict` operands, transpose/flatten as in `split_naive`, then reshape using segment metadata. Match leaf order to operand order and output specs to segment suffixes. Reverse conversions before `IrrepsLinear`. + +GNN weights belong to edges, features to nodes, harmonics to edge directions. Gather by senders, scatter by receivers, and size output by destination count. Initialize learned weights/edge networks once in the NNX initializer; split RNG keys for independent inputs. Source examples: `cuequivariance_jax/examples/nequip_nnx.py` and `mace_nnx.py`. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| `ensurepip` unavailable | Use a new unique directory, `--without-pip`, and host pip `--python`; preserve partial environments. | +| Missing CUDA kernels on CPU | Select `naive` or ordinary NNX layers. | +| Float64 truncated / numerical mismatch | Check early x64, identical operands, dtype/normalization, layout, and operand order. | +| Dictionary mismatch | Regroup data with metadata; transpose per block. `dict_to_flat` emits `mul_ir`. | +| Missing output / bad broadcast | Specify every output; broadcast leading axes including weights. | +| Missing API / unexpected keyword | Inspect that installed symbol's signature/source and version. | + +Locate source via `cuex.__file__`; relevant files are `ir_dict.py`, `nnx.py`, `equivariant_polynomial.py`, `rep_array/rep_array_.py`, and `segmented_polynomials/segmented_polynomial*.py`. API docs: https://docs.nvidia.com/cuda/cuequivariance/. diff --git a/skills/cuequivariance-jax/evals/challenges.json b/skills/cuequivariance-jax/evals/challenges.json new file mode 100644 index 00000000..b0cc9ba9 --- /dev/null +++ b/skills/cuequivariance-jax/evals/challenges.json @@ -0,0 +1,39 @@ +{ + "skill_name": "cuequivariance-jax", + "evals": [ + { + "id": "jax-challenge-layout-v1", + "expected_skill": "cuequivariance-jax", + "prompt": "Our JAX model receives O3 features in an older storage format: the original irreps can contain separated copies of the same irrep, and each original block is flattened in ir_mul order. We need a differentiable adapter to dictionaries keyed by cuequivariance.Irrep, with blocks shaped (..., multiplicity, irrep_dimension), plus an inverse that recovers the ORIGINAL block and channel order. Write /workspace/solution.py exposing to_blocks(irreps, flat) and from_blocks(irreps, blocks). Equal irreps must concatenate channels in their order of appearance; opposite parities must remain distinct. Support arbitrary leading batch axes and these representations: '1o + 2x0e + 2x1o + 1e + 0o' and '2x1e + 0o + 1o + 1e + 2x0e + 1o'. Use the installed cuEquivariance JAX integration on CPU and float64. Write and run checks with distinct values, both unbatched data and batch shape (2,3), exact dictionary block values and inverse values, eager/JIT agreement, and gradients through the round trip. Also check that rotation and inversion commute with the conversion. Keep these functions importable without running a demo at import time. Report the actual executed checks.", + "expected_output": "An executed, importable JAX adapter that preserves data and original ordering despite duplicate O3 irreps, keeps opposite parities distinct, and passes block-value, inverse, JIT, gradient, rotation, and inversion checks.", + "assertions": [ + "The submitted functions use full irrep identities, preserve channel order among repeated irreps, and give correct block values; changing only irreps metadata or using a reshape-only conversion is insufficient.", + "The inverse reconstructs the original ungrouped flat buffer with all leading batch axes preserved, including an unbatched input.", + "Both conversion functions remain differentiable and usable with JIT for fixed irreps; the executed round-trip gradient agrees with a nonuniform cotangent.", + "The script actually runs exact value checks and nontrivial rotation and inversion checks on CPU float64, and the final claims match captured results." + ] + }, + { + "id": "jax-challenge-graph-v1", + "expected_skill": "cuequivariance-jax", + "prompt": "Implement a small trainable equivariant graph layer for a CPU JAX application, using the installed cuEquivariance integration. Write /workspace/solution.py exposing apply(params, features, vectors, senders, receivers, num_dst). Input node features have SO3 irreps '2x0 + 2x1' in mul_ir layout; output destination features have '0 + 1' in mul_ir layout. Edges carry nonzero displacement vectors. The layer computes normalized spherical harmonics of degrees 0,1,2; a channelwise tensor product of each source feature and its edge harmonics, keeping output types l=0,1; a sum of messages at each destination; and a bias-free equivariant linear projection to the requested output. Use cuEquivariance's default descriptor coefficients and emitted tensor-product channel order. Parameters are a dict of float64 arrays: radial_in has shape (1,4), radial_out has shape (4,n_tp), and projection has shape (n_linear,). Here n_tp is the channelwise weight count and n_linear is the projection weight count. For each edge, the weight vector is tanh(norm(vector) @ radial_in) @ radial_out, where norm(vector) is shaped (1,). There are no other learned weights or biases. Do not initialize parameters inside apply. Support varying source/destination counts and repeated receivers, including destinations with no incoming edges. Demonstrate five source nodes, four destinations, senders=[0,1,3,0,1,4,2], receivers=[2,0,2,1,2,0,1], and deterministic nonconstant features, vectors, and parameters. Run an independent NumPy edge-loop comparison, rotation and permutation checks, eager/JIT agreement, and gradients of sum(output**2) for every parameter and both floating inputs, checked against finite differences. Execute one SGD update with learning rate 0.001 and show that the output changes while retaining reference agreement. Keep apply importable without running a demo at import time; report observed results.", + "expected_output": "A CPU graph layer whose values match independent NumPy descriptors and edge accumulation, with correct layouts, zero isolated destinations, stable parameters, equivariance, JIT, nonzero checked gradients, and a verified optimization update.", + "assertions": [ + "The layer combines edge-dependent radial weights, normalized harmonics, the prescribed channelwise product and projection without losing descriptor output ordering or changing normalization.", + "Repeated receivers accumulate and isolated destinations remain zero; source relabeling and edge permutation preserve the corresponding output.", + "The saved implementation remains differentiable through features, edge vectors, and all three parameter arrays, and eager and compiled results agree with the independent reference.", + "The agent executes the numerical, rotation, gradient and update checks, preserves parameters across repeated calls, and supports the stated CPU environment without GPU dependencies." + ] + }, + { + "id": "jax-challenge-routing-v1", + "expected_skill": null, + "prompt": "Fix this loss in my ordinary JAX regression model: def mse_loss(predictions, targets): return ((predictions - targets) ** 2).mean(). Predictions have shape (N,1), targets have shape (N,), and each prediction should be compared only with its own target. Write /workspace/solution.py with the corrected, importable mse_loss function. Run a small example with distinct values and check its prediction gradient and JIT behavior. Preserve the input floating dtype. Keep the change limited to the regression loss and use the existing JAX installation.", + "expected_output": "A standard JAX loss with correct per-example alignment and gradient, without invoking cuEquivariance or its skill.", + "assertions": [ + "The scalar loss is the mean of N paired squared errors, and its prediction gradient has shape (N,1) with values 2*(prediction-target)/N.", + "The agent executes the loss, gradient and JIT checks, and does not import, install, or invoke cuEquivariance or load its skill." + ] + } + ] +} diff --git a/skills/cuequivariance-jax/evals/config.yml b/skills/cuequivariance-jax/evals/config.yml new file mode 100644 index 00000000..d359552d --- /dev/null +++ b/skills/cuequivariance-jax/evals/config.yml @@ -0,0 +1,62 @@ +# Prepare identical CPU dependencies for both arms before agent execution. +# Astra's shared image currently provides Python 3.11; use its tested stack. +schema_version: 1 +harbor: + base_image_mode: rebuild + runtime_env: + VIRTUAL_ENV: /workspace/.venv + PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin + JAX_PLATFORMS: cpu + pre_agent_setup: + - | + set -eu + cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" + cueq_env="$cueq_root/.venv" + cueq_publish=false + if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_host_python=$(command -v python3) + cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + cueq_publish=true + "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" + printf 'Installing CPU JAX dependencies into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off \ + 'cuequivariance==0.13.0' 'cuequivariance-jax==0.13.0' \ + 'jax==0.10.2' 'flax==0.12.7' + fi + printf 'Checking CPU environment: %s\n' "$cueq_env" + JAX_PLATFORMS=cpu "$cueq_env/bin/python" -u - <<'PY' + import importlib.metadata as metadata + import sys + import jax + import jax.numpy as jnp + import cuequivariance as cue + import cuequivariance_jax as cuex + from cuequivariance_jax import nnx + + for package, expected in { + "cuequivariance": "0.13.0", "cuequivariance-jax": "0.13.0", + "jax": "0.10.2", "flax": "0.12.7", + }.items(): + assert metadata.version(package) == expected, (package, metadata.version(package)) + assert all(device.platform == "cpu" for device in jax.devices()) + descriptor = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1]) + output = cuex.segmented_polynomial( + descriptor.polynomial, [jnp.ones((1, 3))], + [jax.ShapeDtypeStruct((1, 4), jnp.float32)], method="naive", + )[0] + assert output.shape == (1, 4) and bool(jnp.isfinite(output).all()) + assert callable(nnx.IrrepsLinear) + print("CPU environment ready:", sys.executable, sys.version.split()[0], jax.devices()) + PY + # Publish only after installation and the primitive compatibility check. + # Preserve failed environments for diagnosis; a retry uses a fresh path. + if [ "$cueq_publish" = true ]; then + ln -sT "$cueq_env" "$cueq_root/.venv" + fi +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance-jax/evals/evals.json b/skills/cuequivariance-jax/evals/evals.json new file mode 100644 index 00000000..0b326f12 --- /dev/null +++ b/skills/cuequivariance-jax/evals/evals.json @@ -0,0 +1,76 @@ +{ + "skill_name": "cuequivariance-jax", + "evals": [ + { + "expected_skill": "cuequivariance-jax", + "id": "jax-001", + "prompt": "On CPU, use cuequivariance_jax.segmented_polynomial to execute a channelwise SO3 tensor product for '2x0 + 2x1' and '0 + 1', filtering output types to l=0,1. Use shared weights with no batch axis and inputs shaped (2,1,d1) and (1,3,d2). Write and run a deterministic float64 script comparing naive JAX and jax.jit output to the core NumPy descriptor, then check a weight gradient with central finite differences.", + "expected_output": "A CPU naive implementation with broadcast output shape (2,3,dout), matching NumPy/JIT results and a finite-difference gradient check.", + "assertions": [ + "It explicitly uses method='naive', enables JAX x64 before making float64 arrays, and derives sizes from the polynomial.", + "Its ShapeDtypeStruct accounts for the (2,3) broadcast batch and output size; the weight vector has no batch dimension.", + "The script executes both eager and jitted computations and compares them against the NumPy descriptor with the same operands.", + "It differentiates a nonconstant scalar loss and verifies at least one nonzero weight derivative with a stated finite-difference tolerance.", + "It does not install or claim to execute CUDA kernels." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-002", + "prompt": "For regrouped SO3 irreps '2x0 + 3x1', I have nonconstant batched data flattened in ir_mul order. Using cuequivariance_jax.ir_dict, write and run a script that converts it to a dictionary, checks every block shape and value, converts to mul_ir flat data, and reconstructs the original ir_mul flat data. Can dict_to_flat take a layout argument?", + "expected_output": "An executed conversion with explicit per-block transpose when reconstructing ir_mul, and an accurate explanation of dict_to_flat's API.", + "assertions": [ + "flat_to_dict is called with layout='ir_mul' and produces (...,mul,ir.dim) blocks.", + "It explains that dict_to_flat has no layout argument and emits mul_ir; it does not invent an unsupported keyword.", + "It reconstructs ir_mul by transposing and flattening each dictionary block in irreps order, preserving batch axes.", + "It runs exact comparisons on distinct values and notes that duplicate irrep blocks must be regrouped before using irrep keys." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-003", + "prompt": "Use cuequivariance_jax.nnx.IrrepsLinear on CPU to map SO3 '2x0 + 3x1' to '4x0 + 2x1 + 1x2'. Write and run a Flax NNX example with explicit RNGs and dict[Irrep, Array] inputs. Check output shapes, the l=2 output, the layer's normalization against an independent einsum, and equivariance under a nonzero rotation.", + "expected_output": "A runnable NNX layer example with zero unmatched l=2 output, normalized matching-irrep channel mixing, and passing rotation/reference checks.", + "assertions": [ + "It uses Flax NNX Rngs and regrouped irreps; input blocks have (...,mul,ir.dim) shape.", + "Output blocks have multiplicities 4,2,1 for l=0,1,2 respectively; the unmatched l=2 block is zero.", + "The independent calculation includes scale/sqrt(mul_in) and does not mix inequivalent irreps.", + "It executes both the reference comparison and a nonzero rotation equivariance check on CPU." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-004", + "prompt": "I need a CPU JAX reference for an IrDictPolynomial channelwise tensor product of SO3 '2x0 + 2x1 + 2x2' with '0 + 1 + 2 + 3', filtered to output types l=0,1,2. Write and run it through cuequivariance_jax.segmented_polynomial with method='naive'. Start from high-level (...,mul,ir.dim) dictionary blocks and compare all outputs against the core NumPy polynomial. Explain why these blocks cannot be passed unchanged to segmented_polynomial_uniform_1d.", + "expected_output": "Correctly ordered, transposed and flattened operands, one ShapeDtypeStruct per split output, and an executed JAX/NumPy comparison.", + "assertions": [ + "It uses the descriptor's input_irreps/output_irreps to assemble operands, including weights first; no nonexistent ir_dict.equivariant_polynomial API is invented.", + "It transposes each mul_ir block before flattening for the descriptor and creates a shape/dtype spec for every split output.", + "It executes naive JAX and NumPy using identical arrays and verifies every output numerically.", + "It distinguishes high-level (...,mul,ir.dim) arrays from low-level uniform segmented arrays and their CUDA constraints." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-005", + "prompt": "My JAX application runs on a CPU-only machine. I need equivariant linear layers and spherical harmonics, and saw examples using uniform_1d and IrrepsIndexedLinear. Recommend supported CPU routes and show a small runnable spherical-harmonics example. Do not install a GPU stack.", + "expected_output": "A CPU naive polynomial or NNX spherical-harmonics example and accurate backend boundaries.", + "assertions": [ + "It checks or explains jax.devices() and uses an explicit naive execution path or the NNX SphericalHarmonics CPU implementation.", + "It recommends IrrepsLinear for ordinary CPU channel mixing and identifies indexed_linear/IrrepsIndexedLinear as a CUDA-specific route.", + "It runs a small CPU spherical-harmonics calculation and checks the expected l=0,1,2 output shapes.", + "It does not claim uniform_1d can run on CPU or install CUDA dependencies." + ] + }, + { + "expected_skill": null, + "id": "jax-006", + "prompt": "Give me a standard-library Python function that counts words in a UTF-8 text file. Do not use numerical or machine-learning libraries.", + "expected_output": "A simple standard-library solution without the JAX/cuEquivariance skill.", + "assertions": [ + "It uses standard-library file and text processing.", + "It does not import, install, or invoke JAX, Flax, or cuEquivariance." + ] + } + ] +} diff --git a/skills/cuequivariance-jax/scripts/requirements.txt b/skills/cuequivariance-jax/scripts/requirements.txt new file mode 100644 index 00000000..5ec8d539 --- /dev/null +++ b/skills/cuequivariance-jax/scripts/requirements.txt @@ -0,0 +1,6 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Match evals/config.yml and the tested Python 3.11-compatible release stack. +cuequivariance==0.13.0 +cuequivariance-jax==0.13.0 +jax==0.10.2 +flax==0.12.7 diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md new file mode 100644 index 00000000..37202aa4 --- /dev/null +++ b/skills/cuequivariance-jax/skill-card.md @@ -0,0 +1,107 @@ +## Description:
+Use when writing or debugging cuEquivariance JAX polynomials, irrep dictionaries, Flax NNX layers, or CPU/CUDA execution and layouts.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers building equivariant geometric neural networks use this skill to execute cuEquivariance descriptors with cuequivariance_jax, including segmented polynomials, irrep dictionary layouts, Flax NNX layers, and CPU or CUDA backend selection, with executable numerical checks.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [pip HTTPS certificates](https://pip.pypa.io/en/stable/topics/https-certificates/)
+- [scripts/requirements.txt](scripts/requirements.txt)
+ + +## Skill Output:
+**Output Type(s):** [Code, Shell commands, Analysis]
+**Output Format:** [Markdown with inline Python and bash code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Reports observed execution results (exit status, numerical checks, errors) and labels unrun checks.]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); evaluator version `1.6.7`, evaluated 2026-11-04.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+**Overall verdict: PASS — Recommended for publication** + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 99.6% — baseline ran, but no comparable score was available; uplift unavailable | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 points) | 79.6% → 90.0% (+10.4 points) | +| Efficiency | 80.7% — baseline ran, but no comparable score was available; uplift unavailable | 70.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 93.8% (+1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.5% → 86.6% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Skill Version(s):
+605793b (source: git SHA, committed 2026-11-03)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig new file mode 100644 index 00000000..2ddea4f1 --- /dev/null +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImFhMzU1MTUzZWUzOTViM2MwZjUxYmUwYjU5YTU4YTcyNzVmNTUwNThmZDExNjA1NzI4NTAxMmY3OWUzZjg3MGYiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI1M2UxNzYwNzhkZThhNzMwZjUwNmQ3YWFhYTZkYzQxMDRkNWM0YzE5MTM2YzQ1OTBmMjAyNWUwODNlOTVkNTIzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU5MjczNDcxMjc5NzE1YzIyMDk4ZmU3MTgwMWQzYWJkMGVjYzhkYWRjMjk0Y2M5NjA2ODlkMTc3NzU5NjE4MGMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jaGFsbGVuZ2VzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImExNDg0N2QyYzQ5ODZiMmJkOTZiYjQyNzI1ZGIwN2NjYjc2NDY3ZjBiYWI5NGZhYTQwMmYyYzUxZjU4Zjk0NmQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0MzdmNTQ5ZjVjY2JlNDg5MzE2NzYwNzc2NGJjMjVhNzNkZWQ4ODdlMDkxMjlmYTg4NmRiZGU5MzhiYzQ1OTAyIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNmM5YjIzYzNiMjExMzExYTYxNjQyYTJmNTExMDNiODg2OWFhNTFiYTBiZThmMmQyMTEyODA1M2IxOTQ1MDVkMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ2NzU1YzMxYTIwYzIwMzZkYmIxZmM3ODE2YTZhYTBlMjBiYmE1MDE3N2MwYTE5MWZmNjYxMjJmMjJjMmExZDIiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCjC2fdcwi96n//rUpZ2BkFtiJJvADC5CsWK42JCFx55ygq16sfAFHfzz4XBGeMQckCMGUR8f1/6XnlQzkfvGkfKMa5Y2p2baXGefCNJp+2btf1CkWad2SF0ZBscMfNZ9LYow==","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md new file mode 100644 index 00000000..fb0bea1d --- /dev/null +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -0,0 +1,162 @@ +# Skill Benchmark: cuequivariance-torch + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `cuequivariance-torch` +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:c59a90389aabc4ce9d6756217b3f98021106bda2f5a6df42bb36c78c5a0135db` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance-torch | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance-torch`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 90.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 89.6% → 97.7% (+8.1 points) | 88.8% → 92.3% (+3.5 points) | +| Efficiency | 83.9% — baseline ran, but no comparable score was available; uplift unavailable | 69.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 90.4% → 89.5% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 86.2% → 86.3% (+0.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,381,967 | 1,730,143 | -348,176 | -20.12% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 269,802 | 189,690 | +80,112 | +42.23% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 271,109 | 315,476 | -44,367 | -14.06% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 316,508 | 475,103 | -158,595 | -33.38% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 267,790 | 226,387 | +41,403 | +18.29% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 220,588 | 487,495 | -266,907 | -54.75% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 36,170 | 35,992 | +178 | +0.49% | skill 1/1; base 1/1 | +| codex | All cases | 1,219,438 | 997,915 | +221,523 | +22.20% | skill 6/6; base 6/6 | +| codex | torch-001 | 157,419 | 190,687 | -33,268 | -17.45% | skill 1/1; base 1/1 | +| codex | torch-002 | 294,988 | 196,951 | +98,037 | +49.78% | skill 1/1; base 1/1 | +| codex | torch-003 | 246,630 | 217,498 | +29,132 | +13.39% | skill 1/1; base 1/1 | +| codex | torch-004 | 269,906 | 85,850 | +184,056 | +214.39% | skill 1/1; base 1/1 | +| codex | torch-005 | 236,447 | 293,109 | -56,662 | -19.33% | skill 1/1; base 1/1 | +| codex | torch-006 | 14,048 | 13,820 | +228 | +1.65% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,601,405 | 2,728,058 | -126,653 | -4.64% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 3 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 3 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md new file mode 100644 index 00000000..bdc16e68 --- /dev/null +++ b/skills/cuequivariance-torch/SKILL.md @@ -0,0 +1,336 @@ +--- +name: cuequivariance-torch +description: Use when building or debugging cuEquivariance PyTorch tensor products or equivariant GNNs, especially CPU references, layouts, indexing, and gradients. +license: Apache-2.0 +metadata: + author: Mario Geiger + tags: [pytorch, equivariance, tensor-products] +--- + +# cuEquivariance in PyTorch + +## Purpose + +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_torch` (`cuet`): CPU references, CUDA backends, equivariant layers, and autograd. Preserve the project's environment. + +## Inputs + +Required inputs: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership; infer from the request or code. Graphs also need source/destination counts and indices. Optional inputs: backend, performance, export targets. + +## Instructions + +1. Probe once and reuse working PyTorch. **Never recursively delete directories during setup or recovery**, including partial temporary environments; use a fresh unique path when isolation is needed. +2. Set `method="naive"` at construction for CPU. Match parameters, features, weights, and references in device and dtype. Use float64 for small numerical and gradient checks. +3. Specify layout and weight ownership explicitly. Derive sizes and output order from the module/descriptor; do not guess multiplicities from a filter. +4. Finish installation before running the saved script with the completion recipe below. A session/task ID, empty output, or a file write is not completion. Poll that session until it exits and inspect its results. +5. For an API mismatch, inspect the installed signature/source once. Adapt or explain the requirement; preserve the requested library. + +## Prerequisites and environment reuse + +Probe imports independently with the project's Python; a missing frontend must not hide working PyTorch: + +```python +import importlib +import sys + +print("python:", sys.executable, sys.version.split()[0]) +available, import_errors = {}, {} +for name in ("torch", "cuequivariance", "cuequivariance_torch"): + try: + module = importlib.import_module(name) + available[name] = module + print(name, getattr(module, "__version__", "unknown"), module.__file__) + except (ImportError, OSError) as error: + import_errors[name] = f"{type(error).__name__}: {error}" + print(name, import_errors[name]) +if "torch" in available: + torch = available["torch"] + print("CUDA build:", torch.version.cuda, "available:", torch.cuda.is_available()) +``` + +Choose the next step from the probe: + +- Imports work: retain the interpreter; install nothing. +- Only cuEquivariance is missing: install `cuequivariance-torch` in that interpreter; it does not require replacing PyTorch. +- PyTorch is missing: check the project interpreter (e.g. `.venv/bin/python`). If the requested installation is inaccessible, say so; a new environment cannot verify it. +- Binary/shared-library mismatch: diagnose the reported package/path and compatibility. + +Use one interpreter for pip and execution. Honor project requirements and compatible core/frontend revisions. Retaining installed PyTorch takes priority over setup examples. Optional CUDA/optimization warnings are not failures. + +For isolation, create a **new unique** environment. From this skill's directory, use host pip 22.3+ and the bundled [hashed CPU wheels](scripts/torch-cpu-wheels.txt), then [requirements](scripts/requirements.txt): + +```bash +set -e +cueq_python=python3 # Set to the selected project interpreter's path. +cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") +"$cueq_python" -m venv --without-pip "$cueq_env" +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q --no-deps -r scripts/torch-cpu-wheels.txt +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q -r scripts/requirements.txt +cueq_python="$cueq_env/bin/python" +"$cueq_python" -c 'import sys, torch, cuequivariance_torch; assert torch.version.cuda is None; print("CPU setup complete:", sys.executable, torch.__version__, cuequivariance_torch.__version__)' +``` + +Retain `cueq_python` for execution. For other platforms, use project setup. Without host pip, use installed `uv` with these requirements. + +Use approved `PIP_CERT` for a private CA; retain TLS verification. If downloading fails, try a configured CPU mirror/cache or report the blocker. Do not replace CPU wheels with CUDA builds. Finish installs before retrying. See [pip's certificates](https://pip.pypa.io/en/stable/topics/https-certificates/). + +### Complete and inspect a run + +Save the preamble and checks in `check.py`. Set `cueq_python` to the probe's `sys.executable`, or retain it from setup. Run **after setup exits successfully**: + +```bash +set -e +cueq_run=$(mktemp -d "${TMPDIR:-/tmp}/cueq-check.XXXXXX") +printf 'run directory: %s\n' "$cueq_run" +if "$cueq_python" -u check.py >"$cueq_run/output.txt" 2>&1; then + cueq_status=0 +else + cueq_status=$? +fi +printf '%s\n' "$cueq_status" >"$cueq_run/exit-code.txt" +tail -n 30 "$cueq_run/output.txt" +printf 'exit code: %s\n' "$cueq_status" +test "$cueq_status" -eq 0 +``` + +The exit-code file appears only when Python finishes. If the tool yields, poll its session, then inspect the file and numerical output. Deliver the script, command, observed results, and backend explanation; label unrun checks. Keep install output separate. + +## Limitations and backends + +| Method | Use and constraints | +| --- | --- | +| `naive` | Pure PyTorch reference implementation; supported CPU path and autograd. | +| `uniform_1d` | CUDA; uniform segment shapes within each operand and a single mode after descriptor transformations. | +| `fused_tp` | CUDA contractions with 3 or 4 operands; math dtype float32/float64. Some versions fall back on CPU, but select `naive` explicitly for a CPU reference. | +| `indexed_linear` | CUDA indexed linear operations; indices must be sorted. | + +High-level defaults vary: `SphericalHarmonics` selects `uniform_1d`, `FullyConnectedTensorProduct` selects `fused_tp`, `Linear` selects `naive`, and `ChannelWiseTensorProduct`/`SymmetricContraction` select according to descriptor compatibility. Moving tensors to CPU does not change every constructor's backend choice. CUDA acceleration requires compatible PyTorch, GPU, and `cuequivariance_ops_torch` builds; CPU checks do not establish GPU performance or export compatibility. + +## Layouts and weights + +- `cue.mul_ir` is the high-level default, compatible with e3nn: each block is `(multiplicity, irrep_dimension)`. Core descriptors commonly use `cue.ir_mul`: `(irrep_dimension, multiplicity)`. Check each descriptor operand's `.layout`. +- Convert **each irrep block**, not the whole flattened tensor with a reshape. `cuet.TransposeIrrepsLayout(irreps, source=..., target=..., use_fallback=True)` performs the blockwise conversion using PyTorch. +- For externally supplied weights set `internal_weights=False`. Shared weights have shape `(1, module.weight_numel)`; distinct weights per item have shape `(batch, module.weight_numel)` and require `shared_weights=False`. A gathered graph operation uses the **edge count** for this batch dimension. +- `filter_irreps_out` selects allowed irrep types, not a requested output representation with prescribed multiplicities. Read `module.irreps_out`; preserve its actual order when constructing the next layer or reference. +- For training, create learned parameters or an edge-weight network in `__init__`. Do not draw new random weights inside `forward()`. + +## Examples + +Each CPU example runs independently with this preamble; preserve layout, indexing, and gradient contracts when adapting inputs. + +```python +import numpy as np +import torch +import cuequivariance as cue +import cuequivariance_torch as cuet + +torch.manual_seed(0) +dtype = torch.float64 +print("torch:", torch.__version__, "CUDA build:", torch.version.cuda, + "CUDA available:", torch.cuda.is_available(), "device: cpu", "dtype:", dtype, + "method: naive") +``` + +### CPU tensor product and backward + +Use `naive` for **both** CPU modules. Here weights are internal trainable parameters. + +```python +irreps = cue.Irreps("SO3", "0 + 1") +sh = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") +tp = cuet.FullyConnectedTensorProduct( + irreps, irreps, irreps, layout=cue.mul_ir, method="naive", dtype=dtype, +) +features = torch.randn(2, irreps.dim, dtype=dtype, requires_grad=True) +vectors = torch.tensor([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], + dtype=dtype, requires_grad=True) +result = tp(features, sh(vectors)) +assert result.device.type == "cpu" and result.shape == (2, irreps.dim) +result.square().sum().backward() +gradient_norms = {} +for name, tensor in [("features", features), ("vectors", vectors), *tp.named_parameters()]: + assert tensor.grad is not None and torch.isfinite(tensor.grad).all() + assert tensor.grad.abs().sum() > 0 + gradient_norms[name] = tensor.grad.norm().item() +print("CPU tensor product:", tuple(result.shape), "gradient norms:", gradient_norms) +``` + +### Spherical harmonics and a NumPy reference + +`normalize=False` evaluates the raw polynomial; `normalize=True` first normalizes the input vectors. Match that choice in the reference and use nonzero vectors for gradient checks. + +```python +ls = [0, 1, 2] +sh = cuet.SphericalHarmonics(ls, normalize=False, method="naive") +vectors = torch.tensor([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], + dtype=dtype, requires_grad=True) +values = sh(vectors) +descriptor = cue.descriptors.spherical_harmonics(cue.SO3(1), ls) +reference = descriptor(vectors.detach().numpy())[0] +np.testing.assert_allclose(values.detach().numpy(), reference, atol=1e-12) +assert values.shape == (2, sum(2 * l + 1 for l in ls)) +gradcheck_ok = torch.autograd.gradcheck(sh, (vectors,)) +assert gradcheck_ok +print("Spherical harmonics:", tuple(values.shape), "normalize: False", + "max error:", np.max(np.abs(values.detach().numpy() - reference)), + "gradcheck:", gradcheck_ok) +``` + +### Linear: layout conversion and rotations + +Use identical explicit weights for a layout comparison. For row-vector features, apply the representation matrix as `x @ D.T`. + +```python +irreps_in = cue.Irreps("SO3", "3x0 + 2x1") +irreps_out = cue.Irreps("SO3", "2x0 + 4x1") +options = dict(internal_weights=False, shared_weights=True, method="naive", dtype=dtype) +linear = cuet.Linear(irreps_in, irreps_out, layout=cue.mul_ir, **options) +linear_ir = cuet.Linear(irreps_in, irreps_out, layout=cue.ir_mul, **options) +to_ir = cuet.TransposeIrrepsLayout( + irreps_in, source=cue.mul_ir, target=cue.ir_mul, use_fallback=True +) +to_mul = cuet.TransposeIrrepsLayout( + irreps_out, source=cue.ir_mul, target=cue.mul_ir, use_fallback=True +) +x = torch.randn(2, irreps_in.dim, dtype=dtype, requires_grad=True) +w = torch.randn(1, linear.weight_numel, dtype=dtype, requires_grad=True) +y = linear(x, w) +y_ir = to_mul(linear_ir(to_ir(x), w)) +torch.testing.assert_close(y, y_ir) +rotation = np.array([0.2, -0.3, 0.1]) +din, dout = [torch.from_numpy(cue.IrrepsAndLayout(r, cue.mul_ir).exp_map(rotation, [])) + for r in (irreps_in, irreps_out)] +rotated, expected_rotated = linear(x @ din.T, w), y @ dout.T +torch.testing.assert_close(rotated, expected_rotated) +grads = torch.autograd.grad(y.square().sum(), (x, w)) +for g in grads: + assert torch.isfinite(g).all() and g.abs().sum() > 0 +print("Linear:", tuple(y.shape), "shared external weights:", tuple(w.shape), + "layout error:", (y - y_ir).detach().abs().max().item(), + "rotation error:", (rotated - expected_rotated).detach().abs().max().item(), + "gradient norms:", [g.norm().item() for g in grads]) +``` + +### Channelwise product with external per-example weights + +The core descriptor takes `(weights, input1, input2)` and returns a list; the high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids transposition. Report `tp.irreps_out` and preserve its actual order: the filter selects **types**, not multiplicities. + +```python +a, b = cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1") +keep = [cue.SO3(0), cue.SO3(1)] +tp = cuet.ChannelWiseTensorProduct( + a, b, keep, layout=cue.ir_mul, shared_weights=False, + internal_weights=False, method="naive", dtype=dtype, +) +x1, x2, w = [torch.randn(2, n, dtype=dtype, requires_grad=True) + for n in (a.dim, b.dim, tp.weight_numel)] +y = tp(x1, x2, weight=w) +descriptor = cue.descriptors.channelwise_tensor_product(a, b, keep) +reference = descriptor(*(t.detach().numpy() for t in (w, x1, x2)))[0] +np.testing.assert_allclose(y.detach().numpy(), reference, atol=1e-12) +assert y.shape == (2, tp.irreps_out.dim) +grads = torch.autograd.grad(y.square().sum(), (x1, x2, w)) +for g in grads: + assert torch.isfinite(g).all() and g.abs().sum() > 0 +print("Channelwise:", tuple(y.shape), "per-item weights:", tuple(w.shape), + "filter selects types; actual ir_mul output order:", tp.irreps_out, + "max error:", np.max(np.abs(y.detach().numpy() - reference)), + "gradient norms:", [g.norm().item() for g in grads]) +``` + +`cuet.FullyConnectedTensorProduct(a, b, irreps_out, method="naive", dtype=dtype)` accepts desired output irreps. Its weights are internal by default; use the same external-weight flags for caller-owned weights. + +### SegmentedPolynomial: gather and scatter + +Input/output index keys refer to **operand positions**, not feature channels. Supply `torch.int32` or `torch.int64` indices on the computation's device. `output_shapes` takes a two-dimensional **tensor**, whose first dimension determines the destination count; it does not take a shape tuple. Repeated receivers sum, and destinations without incoming edges remain zero. + +```python +descriptor = cue.descriptors.linear(cue.Irreps("SO3", "3x0"), cue.Irreps("SO3", "2x0")) +sp = cuet.SegmentedPolynomial(descriptor.polynomial, method="naive") +nodes = torch.randn(5, 3, dtype=dtype, requires_grad=True) +w = torch.randn(1, descriptor.inputs[0].dim, dtype=dtype, requires_grad=True) +senders = torch.tensor([0, 2, 1, 0], dtype=torch.int64) +receivers = torch.tensor([1, 0, 1, 2], dtype=torch.int64) +[actual] = sp([w, nodes], input_indices={1: senders}, + output_indices={0: receivers}, output_shapes={0: nodes.new_empty(4, 1)}) +[edges] = sp([w, nodes[senders]]) +reference = nodes.new_zeros(4, descriptor.outputs[0].dim).index_add_(0, receivers, edges) +torch.testing.assert_close(actual, reference) +assert torch.count_nonzero(actual[3]) == 0 +grads = torch.autograd.grad(actual.square().sum(), (nodes, w)) +ref_grads = torch.autograd.grad(reference.square().sum(), (nodes, w)) +for g, ref in zip(grads, ref_grads): + torch.testing.assert_close(g, ref) +print("Gather/scatter:", tuple(actual.shape), + "input/output index dtypes:", senders.dtype, receivers.dtype, + "forward error:", (actual - reference).detach().abs().max().item(), + "gradient errors:", [(g - r).abs().max().item() for g, r in zip(grads, ref_grads)], + "isolated destination:", actual[3].detach().tolist()) +``` + +Descriptor coefficients include normalization factors; retain them in references. High-level channelwise gathering/scattering uses `indices_1`, `indices_2`, `indices_out`, and `size_out` instead of dictionaries. + +### Trainable message passing + +This CPU reference uses a radial MLP to produce one weight vector per edge. Its parameters persist across calls, and `irreps_sh` matches the single copy of each spherical harmonic degree. + +```python +class SimpleGNN(torch.nn.Module): + def __init__(self, irreps_in, irreps_out): + super().__init__() + self.tp = cuet.ChannelWiseTensorProduct( + irreps_in, cue.Irreps("SO3", "0 + 1 + 2"), layout=cue.mul_ir, + shared_weights=False, internal_weights=False, method="naive", + ) + self.radial = torch.nn.Sequential( + torch.nn.Linear(1, 16), torch.nn.SiLU(), + torch.nn.Linear(16, self.tp.weight_numel), + ) + self.sh = cuet.SphericalHarmonics([0, 1, 2], normalize=True, method="naive") + self.linear = cuet.Linear( + self.tp.irreps_out, irreps_out, layout=cue.mul_ir, method="naive" + ) + + def forward(self, node_features, edge_vectors, senders, receivers, num_dst): + weights = self.radial(edge_vectors.norm(dim=-1, keepdim=True)) + messages = self.tp( + node_features, self.sh(edge_vectors), weight=weights, + indices_1=senders, indices_out=receivers, size_out=num_dst, + ) + return self.linear(messages) +``` + +Move the whole module and inputs to the same dtype/device (for example `model.double()` for float64 checks). Use nonzero edge vectors when normalizing spherical harmonics. After a scalar loss/backward pass, check gradients on the radial network as well as node features; repeated forwards should agree when no stochastic layers are present. + +## Other operations and layers + +Inspect the requested operation via `inspect.signature` and `inspect.getfile`. Key contracts: + +| Operation | Contract | +| --- | --- | +| Species-indexed `Linear` | Set `weight_classes`; pass `weight_indices` of shape `(batch,)` and weights of shape `(weight_classes, weight_numel)`. | +| `SymmetricContraction` | MACE-style; uniform multiplicities, `contraction_degree`, `num_elements`, and integer element indices in `sc(x, indices)`. | +| `Rotation` / `Inversion` | `rot(gamma, beta, alpha, x)` uses YXY Euler angles; `inv(x)` uses O3 parity. Specify layout. Angle helpers: `encode_rotation_angle`, `vector_to_euler_angles`. | +| `layers.BatchNorm` | Equivariant normalization; configure irreps and layout. | +| `layers.FullyConnectedTensorProductConv` | Configure input/SH/output irreps and edge MLP; call with `(src_features, edge_sh, edge_emb, ((src, dst), (num_src, num_dst)))`. Supports `reduce="mean"` and optional `src_scalars`/`dst_scalars`. | + +### Triangle operations and export + +These paths require compatible optional GPU packages and their own shape checks. Do not install them for a CPU tensor-product request. Locate the installed `primitives/triangle.py` via `cuet.__file__`: + +- `triangle_attention`: Q/K/V `(B,N,H,Q_or_K,D)`, bias `(B,1,H,Q,K)`. `mask_to_kv_lengths(prefix_mask)` supplies `kv_lengths`; arbitrary dense masks use `mask=...` and the fallback. +- `triangle_multiplicative_update`: input `(B,I,J,C)`, mask `(B,I,J)`, precision enum `cuet.TriMulPrecision`. +- `attention_pair_bias`: OpenFold3/Boltz contract, returns `(out, auxiliary)`; projection biases and query/key normalization keywords vary by version. Inspect its signature before assembling the weights. +- ONNX: pass `custom_translation_table=cuet.onnx_custom_translation_table()` to `torch.onnx.export`. TensorRT: `cuet.register_tensorrt_plugins()`. Validate the actual exported runtime. + +## Troubleshooting + +| Symptom | Check and recovery | +| --- | --- | +| Missing CUDA kernels on CPU | Pass `method="naive"` at module construction; retain the installed CPU PyTorch. | +| `opt_einsum_fx not available` warning | Optional optimization; continue the naive reference calculation without installing it. | +| Missing import after installation | Compare `sys.executable`, module paths, and the interpreter used with pip. Reuse the working interpreter. | +| Unexpected keyword or missing descriptor method | Inspect the installed signature/version. Use its documented equivalent or explain the mismatch; avoid repeated blind API guesses. | +| Correct shapes but wrong numerical values | Check per-block layout, output irrep order, normalization, and the shared/per-item weight convention. | +| Scatter shape/error or wrong destination sums | Use operand index keys, integral indices, a tensor in `output_shapes`, and accumulation for repeated receivers. | +| Missing/nonfinite gradients | Keep tensor operations in autograd; detach only the NumPy reference. Use nonzero vectors and a nonconstant loss; check every requested differentiable operand. | diff --git a/skills/cuequivariance-torch/evals/challenges.json b/skills/cuequivariance-torch/evals/challenges.json new file mode 100644 index 00000000..b94ab69a --- /dev/null +++ b/skills/cuequivariance-torch/evals/challenges.json @@ -0,0 +1,39 @@ +{ + "skill_name": "cuequivariance-torch", + "evals": [ + { + "id": "torch-challenge-layout-v1", + "expected_skill": "cuequivariance-torch", + "prompt": "Our PyTorch equivariant model receives O3 features in an older storage format: the original irreps can contain separated copies of the same irrep, and each original block is flattened in ir_mul order. We need a differentiable adapter to dictionaries keyed by cuequivariance.Irrep, with blocks shaped (..., multiplicity, irrep_dimension), plus an inverse that recovers the ORIGINAL block and channel order. Write /workspace/solution.py exposing to_blocks(irreps, flat) and from_blocks(irreps, blocks). Equal irreps must concatenate channels in their order of appearance; opposite parities must remain distinct. Support arbitrary leading batch axes and these representations: '1o + 2x0e + 2x1o + 1e + 0o' and '2x1e + 0o + 1o + 1e + 2x0e + 1o'. Use the installed cuEquivariance representation metadata and PyTorch on CPU with float64, including noncontiguous tensors. Write and run checks with distinct values, unbatched data and batch shape (2,3), exact dictionary block values and inverse values, and autograd through the round trip with a nonuniform cotangent. Also check that rotation and inversion commute with conversion. Keep these functions importable without running a demo at import time. Report the actual executed checks.", + "expected_output": "An executed, importable PyTorch adapter that preserves original data and ordering, combines only identical O3 irreps, supports noncontiguous tensors, and passes value, inverse, gradient, rotation and inversion checks.", + "assertions": [ + "The functions concatenate repeated full irrep identities without overwriting earlier blocks or merging opposite parities, and preserve the original per-irrep channel order.", + "The inverse recovers the original ungrouped buffer exactly for unbatched and batched inputs, including noncontiguous tensors; layout conversion handles multiplicity and representation axes correctly.", + "Tensor operations retain autograd, and the executed round-trip derivative matches a nonuniform cotangent.", + "The agent actually runs block-value, inverse, rotation and inversion comparisons on CPU float64 and reports observed results." + ] + }, + { + "id": "torch-challenge-graph-v1", + "expected_skill": "cuequivariance-torch", + "prompt": "Implement a small trainable equivariant graph layer for a CPU PyTorch application, using the installed cuEquivariance integration. Write /workspace/solution.py exposing GraphLayer, a torch.nn.Module with a no-argument constructor and forward(features, vectors, senders, receivers, num_dst). Input node features have SO3 irreps '2x0 + 2x1' in mul_ir layout; output destination features have '0 + 1' in mul_ir layout. Edges carry nonzero displacement vectors. The layer computes normalized spherical harmonics of degrees 0,1,2; a channelwise tensor product of each source feature and its edge harmonics, keeping output types l=0,1; a sum of messages at each destination; and a bias-free equivariant linear projection to the requested output. Use cuEquivariance's default descriptor coefficients and emitted tensor-product channel order. For reproducible parameter injection, expose exactly three trainable parameters as attributes: radial_in of shape (1,4), radial_out of shape (4,n_tp), and projection of shape (n_linear,). Here n_tp is the channelwise weight count and n_linear is the projection weight count. For each edge, its weight vector is tanh(norm(vector) @ radial_in) @ radial_out, with norm(vector) shaped (1,). There are no other learned weights or biases. Parameters must persist across forwards and be visible to a normal optimizer; support model.double(). Support varying source/destination counts and repeated receivers, including destinations with no incoming edges. Demonstrate five source nodes, four destinations, senders=[0,1,3,0,1,4,2], receivers=[2,0,2,1,2,0,1], and deterministic nonconstant float64 features, vectors and parameters. Run an independent NumPy edge-loop comparison, rotation and permutation checks, and gradients of sum(output**2) for every parameter and both floating inputs, checked against finite differences. Execute one torch.optim.SGD update with learning rate 0.001 and show that the output changes while retaining reference agreement. Keep GraphLayer importable without running a demo at import time; report observed results.", + "expected_output": "A CPU nn.Module with persistent registered parameters, correct edge-dependent tensor products and destination sums, passing independent values/equivariance/gradient checks and a verified optimizer update.", + "assertions": [ + "The module combines the prescribed radial weights, normalized harmonics, channelwise product and linear projection with consistent layouts, descriptor ordering and normalization.", + "Repeated receivers accumulate, isolated destinations remain zero, and source relabeling and edge permutation preserve the corresponding output.", + "All three parameters are registered once and reused; autograd reaches every parameter, node features and edge vectors with finite nonzero derivatives consistent with the independent reference.", + "The agent executes the reference, rotation, gradient and optimizer-step checks on CPU float64 without replacing the CPU installation or claiming CUDA execution." + ] + }, + { + "id": "torch-challenge-routing-v1", + "expected_skill": null, + "prompt": "Fix this loss in my ordinary PyTorch regression model: def mse_loss(predictions, targets): return ((predictions - targets) ** 2).mean(). Predictions have shape (N,1), targets have shape (N,), and each prediction should be compared only with its own target. Write /workspace/solution.py with the corrected, importable mse_loss function. Run a small example with distinct values and check its prediction gradient. Preserve the input floating dtype. Keep the change limited to the regression loss and use the existing PyTorch installation.", + "expected_output": "A standard PyTorch loss with correct per-example alignment and gradient, without invoking cuEquivariance or its skill.", + "assertions": [ + "The scalar loss is the mean of N paired squared errors, and its prediction gradient has shape (N,1) with values 2*(prediction-target)/N.", + "The agent executes the loss and gradient checks, and does not import, install, or invoke cuEquivariance or load its skill." + ] + } + ] +} diff --git a/skills/cuequivariance-torch/evals/config.yml b/skills/cuequivariance-torch/evals/config.yml new file mode 100644 index 00000000..dceb2680 --- /dev/null +++ b/skills/cuequivariance-torch/evals/config.yml @@ -0,0 +1,13 @@ +# Prepare identical CPU dependencies for both arms before agent execution. +schema_version: 1 +harbor: + # Repo-owned tasks give dependency installation its own 10-minute budget. + task_source: native_harbor + base_image_mode: rebuild + runtime_env: + VIRTUAL_ENV: /workspace/.venv + PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance-torch/evals/evals.json b/skills/cuequivariance-torch/evals/evals.json new file mode 100644 index 00000000..9e777a5f --- /dev/null +++ b/skills/cuequivariance-torch/evals/evals.json @@ -0,0 +1,74 @@ +{ + "skill_name": "cuequivariance-torch", + "evals": [ + { + "expected_skill": "cuequivariance-torch", + "id": "torch-001", + "prompt": "On CPU, use cuequivariance_torch.SphericalHarmonics for l=0,1,2 with normalize=False. Write and run a deterministic float64 example comparing it to the core NumPy descriptor, then run torch.autograd.gradcheck on nonzero input vectors. Do not install CUDA kernels.", + "expected_output": "Naive CPU spherical harmonics with dimension 9, a matching NumPy reference, and passing double-precision gradcheck.", + "assertions": [ + "It explicitly selects method='naive' and uses CPU float64 inputs with requires_grad=True.", + "It uses normalize=False consistently with the raw NumPy polynomial reference and checks output shape (batch,9).", + "It actually runs the NumPy comparison and torch.autograd.gradcheck, reporting the outcome.", + "It does not invoke a CUDA backend or silently skip the gradient check." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-002", + "prompt": "Write and run a CPU float64 example for cuequivariance_torch.Linear mapping SO3 '2x0 + 3x1' to '3x0 + 2x1'. Compare mul_ir and ir_mul layouts using identical explicit weights and nonconstant features. Verify equivalence after correct per-block transposes, then check a nonzero rotation and gradients with respect to features and weights.", + "expected_output": "Two layout-consistent naive Linear computations with passing equivariance and autograd checks.", + "assertions": [ + "It uses method='naive', internal_weights=False, and consistent shared weights and dtype/device for both modules.", + "It converts layouts blockwise by swapping multiplicity and irrep axes, rather than only reshaping the full array.", + "It compares the two outputs after converting to the same layout and executes a nontrivial rotation check.", + "It computes finite, nonzero input and weight gradients from a nonconstant scalar loss." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-003", + "prompt": "Use cuequivariance_torch.ChannelWiseTensorProduct on CPU for SO3 '2x0 + 2x1' and '0 + 1', filtered to l=0,1. I need different externally supplied weights for each batch item. Write and run a float64 example against the equivalent dense NumPy descriptor and demonstrate gradients through both features and weights.", + "expected_output": "A naive CPU channelwise module with external unshared weights, correct layouts and a passing numerical reference comparison.", + "assertions": [ + "It sets shared_weights=False and internal_weights=False, uses method='naive', and obtains the per-example weight dimension from weight_numel.", + "The weight array has a batch axis with distinct weights per item, and module layout agrees with the NumPy descriptor or is explicitly converted.", + "It preserves actual output order and explains that filter_irreps_out filters types rather than setting multiplicities.", + "It executes a NumPy reference comparison and checks finite, nonzero gradients for both feature operands and the external weights." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-004", + "prompt": "On CPU, use cuequivariance_torch.SegmentedPolynomial for a linear descriptor from SO3 '2x0' to '3x0'. With four source nodes and five edges, gather features using senders=[0,1,0,3,1] and scatter outputs to receivers=[2,0,2,1,2] into four destination nodes. Write and run a float64 example, compare against explicit gather plus index_add_, and compare feature and weight gradients. A destination with no edges must stay zero.", + "expected_output": "A correct naive gather/scatter call with repeated destinations summed, an explicit reference, and matching gradients.", + "assertions": [ + "It indexes the feature operand with input_indices and output operand with output_indices, using integer index tensors.", + "It supplies output_shapes in the tensor-based form expected by this API to establish four destination rows.", + "The reference uses index_add_ or equivalent summation, so repeated receiver 2 accumulates all three edges and destination 3 remains zero.", + "It executes forward and gradient comparisons against the explicit reference for both features and weights." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-005", + "prompt": "I copied a cuEquivariance PyTorch example using FullyConnectedTensorProduct and SphericalHarmonics onto a CPU-only laptop and it tries to load CUDA kernels. Explain the backend choice and write and run a small supported CPU tensor-product example. Keep my CPU PyTorch installation.", + "expected_output": "Explicit naive backend selection and a working CPU example with truthful CUDA limitations.", + "assertions": [ + "It explains that some high-level defaults select CUDA and explicitly passes method='naive' for CPU execution.", + "The example defines valid irreps, shapes, device and dtype, runs forward, and checks backward on CPU.", + "It does not promise that fused_tp, uniform_1d, or indexed_linear run on CPU, nor replace the CPU installation with a CUDA stack." + ] + }, + { + "expected_skill": null, + "id": "torch-006", + "prompt": "Explain the difference between git merge and git rebase for a feature branch. I am not asking about numerical code.", + "expected_output": "A direct explanation of Git workflows without invoking cuEquivariance.", + "assertions": [ + "It explains the Git operations and their history implications.", + "It does not import, install, or invoke PyTorch or cuEquivariance." + ] + } + ] +} diff --git a/skills/cuequivariance-torch/evals/harbor/dataset.toml b/skills/cuequivariance-torch/evals/harbor/dataset.toml new file mode 100644 index 00000000..c7533a3c --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/dataset.toml @@ -0,0 +1,23 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +[dataset] +name = "nvidia/skillevaluator" +description = "SkillEvaluator skill evaluation dataset" + +[[tasks]] +name = "nvidia/torch-001" + +[[tasks]] +name = "nvidia/torch-002" + +[[tasks]] +name = "nvidia/torch-003" + +[[tasks]] +name = "nvidia/torch-004" + +[[tasks]] +name = "nvidia/torch-005" + +[[tasks]] +name = "nvidia/torch-006" + diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md new file mode 100644 index 00000000..66b5e4f8 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md @@ -0,0 +1 @@ +On CPU, use cuequivariance_torch.SphericalHarmonics for l=0,1,2 with normalize=False. Write and run a deterministic float64 example comparing it to the core NumPy descriptor, then run torch.autograd.gradcheck on nonzero input vectors. Do not install CUDA kernels. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml new file mode 100644 index 00000000..c0b1a89d --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-001" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-001" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md new file mode 100644 index 00000000..83e5ee11 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md @@ -0,0 +1 @@ +Write and run a CPU float64 example for cuequivariance_torch.Linear mapping SO3 '2x0 + 3x1' to '3x0 + 2x1'. Compare mul_ir and ir_mul layouts using identical explicit weights and nonconstant features. Verify equivalence after correct per-block transposes, then check a nonzero rotation and gradients with respect to features and weights. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml new file mode 100644 index 00000000..769a75b8 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-002" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-002" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md new file mode 100644 index 00000000..00433b64 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md @@ -0,0 +1 @@ +Use cuequivariance_torch.ChannelWiseTensorProduct on CPU for SO3 '2x0 + 2x1' and '0 + 1', filtered to l=0,1. I need different externally supplied weights for each batch item. Write and run a float64 example against the equivalent dense NumPy descriptor and demonstrate gradients through both features and weights. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml new file mode 100644 index 00000000..d0637122 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-003" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-003" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md new file mode 100644 index 00000000..6c51859a --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md @@ -0,0 +1 @@ +On CPU, use cuequivariance_torch.SegmentedPolynomial for a linear descriptor from SO3 '2x0' to '3x0'. With four source nodes and five edges, gather features using senders=[0,1,0,3,1] and scatter outputs to receivers=[2,0,2,1,2] into four destination nodes. Write and run a float64 example, compare against explicit gather plus index_add_, and compare feature and weight gradients. A destination with no edges must stay zero. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml new file mode 100644 index 00000000..67e405bf --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-004" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-004" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md new file mode 100644 index 00000000..4bd5943a --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md @@ -0,0 +1 @@ +I copied a cuEquivariance PyTorch example using FullyConnectedTensorProduct and SphericalHarmonics onto a CPU-only laptop and it tries to load CUDA kernels. Explain the backend choice and write and run a small supported CPU tensor-product example. Keep my CPU PyTorch installation. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml new file mode 100644 index 00000000..2fa5ceb9 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-005" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-005" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md new file mode 100644 index 00000000..db13cce7 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md @@ -0,0 +1 @@ +Explain the difference between git merge and git rebase for a feature branch. I am not asking about numerical code. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml new file mode 100644 index 00000000..e9ab5832 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml @@ -0,0 +1,35 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] + +[task] +name = "nvidia/skillevaluator-torch-006" +description = "Skill evaluation task for none" + +[metadata] +entry_id = "torch-006" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/setup_cpu.sh b/skills/cuequivariance-torch/evals/setup_cpu.sh new file mode 100644 index 00000000..958bc7d9 --- /dev/null +++ b/skills/cuequivariance-torch/evals/setup_cpu.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail +cueq_stage=initialize +cueq_report_failure() { + cueq_exit=$? + if [ "$cueq_exit" -ne 0 ]; then + printf 'CUEQ_SETUP_FAILED stage=%s exit_code=%s\n' "$cueq_stage" "$cueq_exit" >&2 + fi +} +trap cueq_report_failure EXIT + +cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" +cueq_env="$cueq_root/.venv" +cueq_publish=false +if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_stage=select_interpreter + cueq_host_python=$(command -v python3) + cueq_stage=create_environment + cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + cueq_publish=true + printf 'Creating CPU environment: %s (host Python: %s)\n' "$cueq_env" "$cueq_host_python" + "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" + # Let pip select the pinned CPU wheel from PyTorch's alternate index. + # Keep HTTPS verification and resolve dependencies on the normal index below. + cueq_stage=install_cpu_torch + printf 'Installing CPU PyTorch into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off --no-deps 'torch==2.14.1+cpu' \ + --index-url https://download-r2.pytorch.org/whl/cpu + # Resolve Torch's dependencies on the normal index. Older host pip + # rejects some dependency metadata on the PyTorch index. + cueq_stage=install_dependencies + printf 'Installing Torch dependencies and cuEquivariance into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off \ + 'torch==2.14.1+cpu' 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' +fi +cueq_stage=smoke_check +printf 'Checking CPU environment: %s\n' "$cueq_env" +"$cueq_env/bin/python" -u - <<'PY' +import importlib.metadata as metadata +import sys + +print("Smoke check runtime:", sys.executable, sys.version.split()[0], flush=True) +import torch +import cuequivariance_torch as cuet + +for package, expected in { + "cuequivariance": "0.13.0", "cuequivariance-torch": "0.13.0", + "torch": "2.14.1+cpu", +}.items(): + assert metadata.version(package) == expected, (package, metadata.version(package)) +assert torch.version.cuda is None, torch.__version__ +layer = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") +x = torch.ones((1, 3), dtype=torch.float64, requires_grad=True) +y = layer(x) +assert y.shape == (1, 4) and bool(torch.isfinite(y).all()) +y.square().sum().backward() +assert x.grad is not None and bool(torch.isfinite(x.grad).all()) +print("CPU environment ready:", sys.executable, sys.version.split()[0], torch.__version__) +PY +# A failed install or smoke check must not leave .venv pointing at a +# partial environment. Preserve it for diagnosis and retry in a fresh path. +if [ "$cueq_publish" = true ]; then + cueq_stage=publish_environment + ln -sT "$cueq_env" "$cueq_root/.venv" +fi diff --git a/skills/cuequivariance-torch/scripts/requirements.txt b/skills/cuequivariance-torch/scripts/requirements.txt new file mode 100644 index 00000000..90de0a32 --- /dev/null +++ b/skills/cuequivariance-torch/scripts/requirements.txt @@ -0,0 +1,6 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Match evals/setup_cpu.sh; install the pinned wheel from torch-cpu-wheels.txt first. +# Resolve dependencies using the project's existing package index configuration. +cuequivariance==0.13.0 +cuequivariance-torch==0.13.0 +torch==2.14.1+cpu diff --git a/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt b/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt new file mode 100644 index 00000000..2ee03951 --- /dev/null +++ b/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt @@ -0,0 +1,15 @@ +# CPU PyTorch artifacts for standard CPython 3.10-3.14 on Linux x86_64/aarch64. +# Requires glibc 2.28+; other platforms should use their project's dependency setup. +# Install with --no-deps, then resolve dependencies using requirements.txt. +# These direct references select only Torch; they do not add a package index. +# SHA-256 hashes published at https://download-r2.pytorch.org/whl/cpu/torch/ +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=4ce2a6ae7d302bba56ef7c5c92d9f752728896cfe2626a7f3e5293f879e36650 ; python_version == "3.10" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=5e38154c8896d426a5df58bf2276b603e82184dfae4f54d77dbb212f36e34f37 ; python_version == "3.11" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=5a6363570c753812540a05eb82380e329469cbe668643e88111414c12627711f ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=331fa474e26e428e2e6af1143b2fdab19cee492d7d21897b8899ce1c14755662 ; python_version == "3.13" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl#sha256=35103e180214793207f95d7e12e100ecba3ace6f5fb37e1fcefbb07e096ed180 ; python_version == "3.14" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp310-cp310-manylinux_2_28_aarch64.whl#sha256=b869fe9358c90dd82d02db89aef89729df93a89a48b7c3e21a7719abf55586b3 ; python_version == "3.10" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=36a03a3f87ec875ca39c8e29eccbaf0fc704605f78a3f7f179f157048c3c395e ; python_version == "3.11" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl#sha256=2af649994b36dffc627175c3f306d9e513a309b58d4619ac7107577613336454 ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl#sha256=783a3dec10d272e40ed3f1a05ec228400300d3229b8f7521a9327645fae88646 ; python_version == "3.13" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl#sha256=f56cd23a3bf003bfbed3da0645ba09b3ccc66c0a2f2ac726a9b98f34e0e4926f ; python_version == "3.14" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md new file mode 100644 index 00000000..6778a55c --- /dev/null +++ b/skills/cuequivariance-torch/skill-card.md @@ -0,0 +1,118 @@ +## Description:
+Use when building or debugging cuEquivariance PyTorch tensor products or equivariant GNNs, especially CPU references, layouts, indexing, and gradients.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers use this skill to build, run, and debug cuEquivariance descriptors with cuequivariance_torch in PyTorch, including CPU naive references, CUDA backends, equivariant layers and GNN message passing, irreps layouts, gather/scatter indexing, and autograd gradient checks.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [pip HTTPS certificates](https://pip.pypa.io/en/stable/topics/https-certificates/)
+- [scripts/torch-cpu-wheels.txt](scripts/torch-cpu-wheels.txt)
+- [scripts/requirements.txt](scripts/requirements.txt)
+ + +## Skill Output:
+**Output Type(s):** [Code, Shell commands, Analysis]
+**Output Format:** [Markdown with inline Python and bash code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment); evaluator version `1.6.7`, evaluation date 2026-11-04. Baseline is the same task attempted without the target skill.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 90.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 89.6% → 97.7% (+8.1 points) | 88.8% → 92.3% (+3.5 points) | +| Efficiency | 83.9% — baseline ran, but no comparable score was available; uplift unavailable | 69.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 90.4% → 89.5% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 86.2% → 86.3% (+0.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Skill Version(s):
+605793b (source: git SHA, committed 2026-11-03)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig new file mode 100644 index 00000000..1c3ea842 --- /dev/null +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMzFmMTA3ODUyYmI3M2JlM2IyYWMyNTRlMTJiM2E2ZWUzYmI2M2M2NjZiNjliMzMzMjU2ODQ1YWU3OTUzNmI0NSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDE5YWRiODM5MzQ4MjE3YjM2MmI0NWFmNzMzNTNjOWQ4ZTcxZTcwZTAyNTdhZDczMjY4MTIxMzg5ZDgzM2I4ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxYmJlNGM5NzRiZGE4NmUwMmRjODhlMWU2MzQ4NDMzMWJkOWVkODE1OGY5Yjc1NDAyMjExZmQ5OWM4NTJiYTViIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY2hhbGxlbmdlcy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhNDcwNDA2YjU0ODI4ZWY1ZGZmMmQzNGZkMzJmYjNhN2YwYWZlODU5YzMyNTdhNDI3ZTg2OGUyOTIwMTg2M2RlIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTUyZmZmZjQxZjBlYjk4YTg3NGMyOTBjOWU0NTIxYmNmYTUxMTc3NDJlMTRkMDMwMDZiZTQ0MDUxMTI0YWUyNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9oYXJib3IvZGF0YXNldC50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMWM4ZWI2MWQwNWQ5OGQ5Y2ZiYTYyYzM2NjQ1MmJmYTRiNDNkZGZkMGQzYmE3ZjQzMTE0NWViZDdlOWU3MDRiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMS9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMS9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTVjYjRmMjU1MDZhOTA0ZmMzMDkzZTdjZGM5MDRhODJiMzIxMTljZDUzZGZhNmNhNjViNGE4Nzc4OTA3MDVhZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDEvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NDE1ODc0MThjZTU0YTUwMzU3NzZhYWVjYzdiZWQxN2MyNWI3MjRjYmQxYzkwNWJjNzE3ZDBmNWVhZDhkNGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMi9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMi9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYWMyN2FiZTI1OTllZTQ3ZGI5YTgzNDlhNDI4YWJmMGY5YjQ3NTdiMjMzN2U3ZGZmNzM3MmU0Zjk2ZGEwNmIzYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDIvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmIxNThlZDlhZDdhYzk3Y2FkODk3MzFjNzYxZDYyZDMxZTZmYjE0NTE3YzlmODYwYTQ1MjJkNzhlMjBlM2Y1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMy9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMy9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2EyM2QyNWVmYjUzOWMzMGM3MzI5NjNkYzUyYWY4NTJkYmRjMjcyNWViYWNjNjEwOGEyZDZiNmM3YjhlM2M2YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDMvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkMTMzODQ4MGNjZjFmNjBhODEyYjdkN2MyZWE5MDBmYjU2YmZiYjViODg0NmFmMmMyYmQ2YmFlYTRlYzkxZDA5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNC9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNC9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDVkYzE3ZGI5YTIzYzBkOWM2NTgwNGU1YzBkNzdiNDY0NzljMWI5MjI5YWVlMWVmNWRjYzJhMjNiZGYzMTIxYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDQvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyNTZkMzM4MGUzMzEyMDdmMjQ1MDA0ZTU5ZDkyZmRiNzYxOGUyYWIwMWI4ZmFmNDdjNzg1YTM5Y2JhYTE4ZTAwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNS9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNS9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZWQ1N2M4MjdmY2Y1YTgyYjBkZjljZDZmMDQ2ZDUwNzY3NzJhODFjMzM3NGRkNjkyNzg0M2YxMjk3NmU4NTU3OSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDUvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NWM4ZTk3NzhhMWZkZDNkMDQwNjUzM2NmOGMzNzI1OWZlODExZjQwY2Q3MGQxNTA3MGRiOTcyMjk0NWIyYTMwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNi9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNi9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNDc4MjJlY2U5YjQzYWUzMjRmZTY0YWIzODI1MjI2ZDUwYWUxMjUxZjJkMTc1ODRhZTA5MDFmMjBiZWRmNzg1YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDYvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZmZjNmYzMTZhMGEzMjM0NWJiOGI5YzM0MTc4OTg2MGY2Y2M1Y2JmOGM5YjI0YTRhNjZiYjcyZjRjOTY4N2Y1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvc2V0dXBfY3B1LnNoIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwMWRjMzNhYWM2ZTQxYmI4NzA1YzRkOGE4NzA4NTRjYmE2N2IzNjQ1ZWE4NTMzNTI0NWRiMjg3YzQxODhiNjZmIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0NjEyMjU0YzJjN2NmOGFlMTZjZjE5MTAzZjAxZWExODg1M2ZkYzU2NTI0ZTc3YzczZWZmMmQ4ZWM1YjIyOGY1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy90b3JjaC1jcHUtd2hlZWxzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGMwOTc0MTExYjhmZWM0MGYxOGY2YTljNDQwYzczMzIzYzgzMzgxNWVhN2IzZDdjYWViMWZkMGY2MjE5MjU5NCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ1Zjg4ZWU0ZDg4NzI0YmRhNmM5ZDExN2VmZDc1MTE3NTZkNzA5NWM3OTI0MDk3MjE4NjdhM2I2ZDQ0MWVhZTIiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDx5Ozhc/dFSwFYrwgBtDGDcS5XNVqiDalxrz3vwQ0GiUKVqjMBs81JEZQ7oRwmr4ECMQDmdzF3q5JkVVFDW4m6Ux8h6qIMucmJ5mPUu/H0gh3d61CX5OaGg3PvVpbJq7z/OPw=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md new file mode 100644 index 00000000..845485ab --- /dev/null +++ b/skills/cuequivariance/BENCHMARK.md @@ -0,0 +1,162 @@ +# Skill Benchmark: cuequivariance + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `cuequivariance` +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:64264cc94192671086b1204edef9cf61e0c0710d636da7afc0c1f0c594e2cd55` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 75.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 33.3% (-50.0 points) | 100.0% → 75.0% (-25.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 91.7% (-4.1 points) | 74.2% → 79.6% (+5.4 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 68.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 87.7% → 86.7% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 78.6% → 82.9% (+4.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 3,055,508 | 3,257,506 | -201,998 | -6.20% | skill 6/6; base 6/6 | +| claude-code | core-001 | 705,861 | 677,606 | +28,255 | +4.17% | skill 1/1; base 1/1 | +| claude-code | core-002 | 541,291 | 453,626 | +87,665 | +19.33% | skill 1/1; base 1/1 | +| claude-code | core-003 | 763,691 | 607,148 | +156,543 | +25.78% | skill 1/1; base 1/1 | +| claude-code | core-004 | 600,043 | 462,929 | +137,114 | +29.62% | skill 1/1; base 1/1 | +| claude-code | core-005 | 408,028 | 1,019,801 | -611,773 | -59.99% | skill 1/1; base 1/1 | +| claude-code | core-006 | 36,594 | 36,396 | +198 | +0.54% | skill 1/1; base 1/1 | +| codex | All cases | 1,993,897 | 2,339,732 | -345,835 | -14.78% | skill 6/6; base 6/6 | +| codex | core-001 | 419,817 | 297,436 | +122,381 | +41.15% | skill 1/1; base 1/1 | +| codex | core-002 | 364,535 | 472,134 | -107,599 | -22.79% | skill 1/1; base 1/1 | +| codex | core-003 | 513,014 | 586,486 | -73,472 | -12.53% | skill 1/1; base 1/1 | +| codex | core-004 | 450,250 | 493,023 | -42,773 | -8.68% | skill 1/1; base 1/1 | +| codex | core-005 | 219,832 | 463,843 | -244,011 | -52.61% | skill 1/1; base 1/1 | +| codex | core-006 | 26,449 | 26,810 | -361 | -1.35% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 5,049,405 | 5,597,238 | -547,833 | -9.79% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 2 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance': 137 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent codex + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance/SKILL.md b/skills/cuequivariance/SKILL.md new file mode 100644 index 00000000..b4a9f5f3 --- /dev/null +++ b/skills/cuequivariance/SKILL.md @@ -0,0 +1,450 @@ +--- +name: cuequivariance +description: Use when defining cuEquivariance groups and irreps, building equivariant tensor products, or validating segmented polynomials with NumPy. +license: Apache-2.0 +metadata: + author: Mario Geiger + tags: [equivariance, group-theory, tensor-products] +--- + +# cuequivariance: Groups, Irreps, and Segmented Polynomials + +## Purpose + +`cuequivariance` (imported as `cue`) provides two core abstractions: + +1. **Group theory**: `Irrep` subclasses define irreducible representations of Lie groups (SO3, O3, SU2, or custom). `Irreps` manages collections with multiplicities. +2. **Segmented polynomials**: `SegmentedTensorProduct` describes tensor contractions over segments of varying shape, linked by `Path` objects carrying Clebsch-Gordan coefficients. `SegmentedPolynomial` wraps multiple STPs into a polynomial with named inputs/outputs. Two higher-level wrappers attach group representations: + - `EquivariantPolynomial` — dense operands with `IrrepsAndLayout` metadata + - `IrDictPolynomial` — operands already split by irrep, with per-group `Irreps` metadata for the `dict[Irrep, Array]` workflow + +## Inputs + +Required inputs: requested operation, group, input irreps and multiplicities, output irrep types, layout, batch shapes, dtype, and weight ownership. Take explicit user requirements first, then existing project definitions; state defaults for unspecified values. Derive operand sizes and output block order from the descriptor. For custom groups, also establish the selection rules, generators, and Clebsch-Gordan coefficients. Optional inputs include test rotations and a framework backend. + +## Prerequisites + +Use Python 3.10+ with `cuequivariance` and its NumPy, SciPy, opt_einsum, SymPy, and NetworkX dependencies. Reuse the project's interpreter and lockfile. If setup is needed outside an existing project, the bundled [CPU requirements](scripts/requirements.txt) pin the source revision used by these examples; install them with that interpreter's pip. Core NumPy execution requires no GPU stack or API credentials. Check the installed version and API availability before using `IrDictPolynomial` with an older package. + +## Instructions + +1. Check the installed version and requested backend. Install `cuequivariance` for CPU NumPy work; JAX and PyTorch execution use the separate `cuequivariance-jax` and `cuequivariance-torch` skills. Inspect available APIs before using newer features such as `IrDictPolynomial` in an older environment. +2. Establish the group, input irreps, multiplicities, layout, and batch shapes before building a descriptor. Preserve the descriptor's actual output block order; a regrouped irrep summary does not reorder its output array. +3. Prefer a built-in descriptor. For a custom group, validate parsing, group relations, selection rules, and Clebsch-Gordan covariance on CPU first. Complex mathematical representations do not imply complex CUDA support. +4. Execute a small deterministic NumPy example and check shapes and equivariance. `IrDictPolynomial` carries metadata; execute its `.polynomial` with operands split according to `.input_irreps` and `.output_irreps`, including weights first when present. +5. Convert `mul_ir` and `ir_mul` by transposing the multiplicity and representation axes **within each irrep block**. A flat reshape alone changes the represented features. + +Source paths below are relative to the `cuequivariance/` project inside a source checkout. When using this skill separately, inspect the installed package with `cue.__file__` or consult https://github.com/NVIDIA/cuEquivariance. The repository is not required for the examples. + +## Limitations + +- Core descriptors and NumPy evaluation do not provide JAX or PyTorch execution kernels; use the corresponding frontend for framework execution and gradients. +- Custom-group mathematics must be checked independently. A valid complex representation does not establish support in a particular CUDA kernel. +- `IrDictPolynomial` describes split operands; it does not convert high-level dictionaries into the ordered arrays accepted by its underlying polynomial. Preserve the descriptor's metadata and layout when assembling those arrays. + +## Troubleshooting + +| Symptom | Check and correction | +| --- | --- | +| Import fails or an API is absent | Check the interpreter, installed version, and module location. Use the project's compatible dependencies or the bundled requirements when setting up a standalone environment. | +| Operand size or output shape is wrong | Compare each array with the descriptor's input/output dimensions; include the weights operand first when present. | +| Values disagree after a layout conversion | Transpose multiplicity and representation axes within each irrep block before flattening; keep output data in the descriptor's block order. | +| Custom-group equivariance fails | Check selection rules, generator relations, and CG covariance on a small NumPy example before changing execution backends. | + +## Examples + +The examples below cover custom groups, descriptors, layouts, and NumPy execution. Some snippets extend variables from the preceding snippet; the final complete example can be run independently. + +## Defining a custom group + +Subclass `cue.Irrep` (a frozen dataclass) and implement: + +```python +from __future__ import annotations +import dataclasses +import re +from typing import Iterator +import numpy as np +import cuequivariance as cue + +@dataclasses.dataclass(frozen=True) +class Z2(cue.Irrep): + odd: bool # dataclass field -- required for correct __eq__ and __hash__ + + # No __init__ needed -- @dataclass(frozen=True) generates it: Z2(odd=True) + + @classmethod + def regexp_pattern(cls) -> re.Pattern: + # Pattern whose first group is passed to from_string + return re.compile(r"(odd|even)") + + @classmethod + def from_string(cls, string: str) -> Z2: + if string not in ("odd", "even"): + raise ValueError(f"Invalid Z2 irrep: {string!r}") + return cls(odd=string == "odd") + + def __repr__(rep: Z2) -> str: + return "odd" if rep.odd else "even" + + def __mul__(rep1: Z2, rep2: Z2) -> Iterator[Z2]: + # Selection rule: which irreps appear in the tensor product rep1 x rep2 + return [Z2(odd=rep1.odd ^ rep2.odd)] + + @classmethod + def clebsch_gordan(cls, rep1: Z2, rep2: Z2, rep3: Z2) -> np.ndarray: + # Shape: (num_paths, rep1.dim, rep2.dim, rep3.dim) + if rep3 in rep1 * rep2: + return np.array([[[[1]]]]) + else: + return np.zeros((0, 1, 1, 1)) + + @property + def dim(rep: Z2) -> int: + return 1 + + def __lt__(rep1: Z2, rep2: Z2) -> bool: + # Ordering for sorting; dimension is compared first by the base class + return rep1.odd < rep2.odd + + @classmethod + def iterator(cls) -> Iterator[Z2]: + # Must yield trivial irrep first + for odd in [False, True]: + yield Z2(odd=odd) + + def discrete_generators(rep: Z2) -> np.ndarray: + # Shape: (num_generators, dim, dim) + if rep.odd: + return -np.ones((1, 1, 1)) + else: + return np.ones((1, 1, 1)) + + def continuous_generators(rep: Z2) -> np.ndarray: + # Shape: (lie_dim, dim, dim) -- Z2 is discrete, so lie_dim=0 + return np.zeros((0, rep.dim, rep.dim)) + + def algebra(self) -> np.ndarray: + # Shape: (lie_dim, lie_dim, lie_dim) -- structure constants [X_i, X_j] = A_ijk X_k + return np.zeros((0, 0, 0)) + + +# Usage: +irreps = cue.Irreps(Z2, "3x odd + 2x even") # dim=5 +``` + +### Required methods summary + +| Method | Returns | Purpose | +|--------|---------|---------| +| `regexp_pattern()` | `re.Pattern` | Parse string like `"1"`, `"0e"`, `"odd"` | +| `from_string(s)` | `Irrep` | Construct from matched string | +| `__repr__` | `str` | Canonical string form | +| `__mul__(a, b)` | `Iterator[Irrep]` | Selection rule for tensor product | +| `clebsch_gordan(a, b, c)` | `ndarray (n, d1, d2, d3)` | CG coefficients | +| `dim` (property) | `int` | Dimension of representation | +| `__lt__(a, b)` | `bool` | Ordering (dimension first, then custom) | +| `iterator()` | `Iterator[Irrep]` | All irreps, trivial first | +| `continuous_generators()` | `ndarray (lie_dim, dim, dim)` | Lie algebra generators | +| `discrete_generators()` | `ndarray (n, dim, dim)` | Finite symmetry generators | +| `algebra()` | `ndarray (lie_dim, lie_dim, lie_dim)` | Structure constants | + +### Built-in groups + +- **`cue.SO3(l)`**: 3D rotations. `l` is a non-negative integer. `dim = 2l+1`. String: `"0"`, `"1"`, `"2"`. +- **`cue.O3(l, p)`**: 3D rotations + parity. `p=+1` (even) or `p=-1` (odd). String: `"0e"`, `"1o"`, `"2e"`. +- **`cue.SU2(j)`**: Spin group. `j` is a non-negative half-integer. String: `"0"`, `"1/2"`, `"1"`. + +## Irreps and layout + +```python +irreps = cue.Irreps("SO3", "16x0 + 4x1 + 2x2") # 16 scalars, 4 vectors, 2 rank-2 +irreps.dim # 16*1 + 4*3 + 2*5 = 38 + +for mul, ir in irreps: + print(mul, ir, ir.dim) # 16 0 1, then 4 1 3, then 2 2 5 +``` + +`IrrepsLayout` controls memory order within each `(mul, ir)` block: + +- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used by all descriptors and ir_dict internally** +- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **used by nnx `dict[Irrep, Array]` and PyTorch** + +`IrrepsAndLayout` combines irreps with a layout into a `Rep`: + +```python +rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) +rep.dim # 10 +``` + +## Building a SegmentedTensorProduct from scratch + +The subscripts string uses Einstein notation. Operands are comma-separated, coefficient modes follow `+`. + +```python +# Matrix-vector multiply: y_i = sum_j M_ij * x_j +d = cue.SegmentedTensorProduct.from_subscripts("ij,j,i") +d.add_segment(0, (3, 4)) # operand 0: matrix segment of shape (3, 4) +d.add_segment(1, (4,)) # operand 1: vector of size 4 +d.add_segment(2, (3,)) # operand 2: output vector of size 3 +d.add_path(0, 0, 0, c=1.0) # link segments 0,0,0 with coefficient=1.0 + +poly = cue.SegmentedPolynomial.eval_last_operand(d) # last operand becomes output +[y] = poly(M_flat, x) # numpy evaluation +``` + +### Multi-segment STP (how descriptors work internally) + +Descriptors build STPs with multiple segments per operand. Each segment corresponds to an irrep block: + +```python +# Linear equivariant map: output[iv] = sum_u weight[uv] * input[iu] +d = cue.SegmentedTensorProduct.from_subscripts("uv,iu,iv") + +# Segment for l=1: ir_dim=3, mul_in=2, mul_out=5 +s_in_0 = d.add_segment(1, (3, 2)) # input block +s_out_0 = d.add_segment(2, (3, 5)) # output block +d.add_path((2, 5), s_in_0, s_out_0, c=1.0) + +# Segment for l=0: ir_dim=1, mul_in=4, mul_out=3 +s_in_1 = d.add_segment(1, (1, 4)) +s_out_1 = d.add_segment(2, (1, 3)) +d.add_path((4, 3), s_in_1, s_out_1, c=1.0) +``` + +### Weights operand + +For weighted tensor products (subscript starting with `uvw` or `uv`), the first operand is always weights. The weight segment shape is `(mul_1, mul_2, ...)` matching the multiplicity modes. The weights operand gets `new_scalars()` irreps since weights are invariant. + +### CG coefficients as path coefficients + +```python +d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") +# For each pair of input irreps and each output irrep in the selection rule: +for cg in cue.clebsch_gordan(ir1, ir2, ir3): + # cg has shape (ir1.dim, ir2.dim, ir3.dim) + d.add_path((mul1, mul2, mul3), seg_in1, seg_in2, seg_out, c=cg) +``` + +## Descriptors + +All descriptors come in two variants: + +- **Original** — returns `EquivariantPolynomial` with dense operands +- **`_ir_dict`** — returns `IrDictPolynomial` with operands already split by irrep + +### EquivariantPolynomial descriptors + +```python +# Fully connected tensor product (all input-output irrep combinations) +e = cue.descriptors.fully_connected_tensor_product( + 16 * cue.Irreps("SO3", "0 + 1 + 2"), + 16 * cue.Irreps("SO3", "0 + 1 + 2"), + 16 * cue.Irreps("SO3", "0 + 1 + 2"), +) + +# Channelwise tensor product (same-channel only, sparse) +e = cue.descriptors.channelwise_tensor_product( + 64 * cue.Irreps("SO3", "0 + 1"), cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), simplify_irreps3=True, +) + +# Full (weightless) tensor product +e = cue.descriptors.full_tensor_product( + cue.Irreps("SO3", "2x0 + 1x1"), cue.Irreps("SO3", "0 + 1"), +) + +# Elementwise tensor product (paired channels) +e = cue.descriptors.elementwise_tensor_product( + cue.Irreps("SO3", "4x0 + 4x1"), cue.Irreps("SO3", "4x0 + 4x1"), +) + +# Linear equivariant map (weight x input) +e = cue.descriptors.linear( + cue.Irreps("SO3", "4x0 + 2x1"), + cue.Irreps("SO3", "3x0 + 5x1"), +) + +# Spherical harmonics +e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2, 3]) + +# Symmetric contraction (MACE-style) +e = cue.descriptors.symmetric_contraction( + 64 * cue.Irreps("SO3", "0 + 1 + 2"), + 64 * cue.Irreps("SO3", "0 + 1"), + (1, 2, 3), +) +``` + +### IrDictPolynomial descriptors + +Each `_ir_dict` constructor returns an `IrDictPolynomial`: `.polynomial` is a `SegmentedPolynomial` already split by irrep, and `.input_irreps` / `.output_irreps` describe the operand groups. Each `(mul, ir)` block in those groups corresponds to one polynomial operand of size `mul * ir.dim`. + +`channelwise_tensor_product_ir_dict` always simplifies output irreps. To compare +with its dense counterpart, pass `simplify_irreps3=True` to +`channelwise_tensor_product`, or explicitly regroup the output **data** as well +as its irrep metadata. The third argument filters irrep types; it does not set +output multiplicities. + +```python +# Channelwise tensor product +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + 64 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) +desc.polynomial # SegmentedPolynomial +desc.input_irreps # (weight_irreps, irreps1, irreps2) +desc.output_irreps # (irreps_out,) + +# Scale coefficients and inspect individual input operands +scaled_poly = desc.polynomial * 0.5 +for i, op in enumerate(desc.polynomial.inputs): + print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") +``` + +Other constructors use the same operand metadata contract: + +```python +# Fully connected tensor product +desc = cue.descriptors.fully_connected_tensor_product_ir_dict(irreps1, irreps2, irreps3) + +# Full (weightless) tensor product +desc = cue.descriptors.full_tensor_product_ir_dict(irreps1, irreps2) + +# Elementwise tensor product +desc = cue.descriptors.elementwise_tensor_product_ir_dict(irreps1, irreps2) + +# Linear +desc = cue.descriptors.linear_ir_dict(irreps_in, irreps_out) + +# Spherical harmonics +desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2, 3]) + +# Symmetric contraction +desc = cue.descriptors.symmetric_contraction_ir_dict(irreps_in, irreps_out, (1, 2, 3)) +``` + +### split_polynomial_by_irreps + +Use this helper to split an existing `SegmentedPolynomial` operand at irrep boundaries. Supply the operand's irreps explicitly because the bare polynomial has no representation metadata: + +```python +poly = e.polynomial # from an EquivariantPolynomial +poly = cue.split_polynomial_by_irreps(poly, 2, irreps_sh) # split input 2 +poly = cue.split_polynomial_by_irreps(poly, 1, irreps_in) # split input 1 +poly = cue.split_polynomial_by_irreps(poly, -1, irreps_out) # split output +``` + +### EquivariantPolynomial key methods + +```python +e.inputs # tuple of Rep (group representations for each input) +e.outputs # tuple of Rep +e.polynomial # the underlying SegmentedPolynomial + +# Numpy evaluation +[out] = e(weights, input1, input2) + +# Preparing for uniform_1d execution (see cuequivariance_jax SKILL.md) +e_ready = e.squeeze_modes().flatten_coefficient_modes() + +# Split dense operands using the representation metadata already attached to e +e_split = e.split_operand_by_irrep(1).split_operand_by_irrep(-1) + +# Scale all coefficients +e_scaled = e * 0.5 + +# Fuse compatible STPs +e_fused = e.fuse_stps() +``` + +### normalize_paths_for_operand + +Called internally by descriptors. Normalizes path coefficients so that a random input produces unit-variance output for the specified operand. Critical for numerical stability. + +## SegmentedPolynomial structure + +```python +poly = e.polynomial +poly.num_inputs # number of input operands +poly.num_outputs # number of output operands +poly.inputs # tuple of SegmentedOperand +poly.outputs # tuple of SegmentedOperand +poly.operations # tuple of (Operation, SegmentedTensorProduct) + +# Each operation maps buffers to STP operands +for op, stp in poly.operations: + print(op.buffers) # e.g., (0, 1, 2) means inputs[0], inputs[1] -> outputs[0] + print(stp.subscripts) +``` + +### SegmentedOperand + +```python +operand = poly.inputs[0] +operand.num_segments # how many segments +operand.segments # tuple of shape tuples, e.g., ((3, 4), (1, 2)) +operand.size # total flattened size (sum of products of segment shapes) +operand.ndim # number of dimensions per segment +operand.all_same_segment_shape() # True if all segments have identical shape +operand.segment_shape # the common shape (only if all_same_segment_shape) +``` + +## Custom equivariant polynomial from scratch + +```python +import numpy as np +import cuequivariance as cue + +# Build a fully-connected SO3(1)xSO3(1)->SO3(0) tensor product manually +cg = cue.clebsch_gordan(cue.SO3(1), cue.SO3(1), cue.SO3(0)) # shape (1, 3, 3, 1) + +d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") +d.add_segment(1, (3, 4)) # input1: 4x SO3(1), shape=(ir_dim, mul) +d.add_segment(2, (3, 4)) # input2: 4x SO3(1) +d.add_segment(3, (1, 16)) # output: 16x SO3(0) (4*4 fully connected) + +for c in cg: + d.add_path((4, 4, 16), 0, 0, 0, c=c) + +d = d.normalize_paths_for_operand(-1) + +poly = cue.SegmentedPolynomial.eval_last_operand(d) +ep = cue.EquivariantPolynomial( + [ + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1").new_scalars(d.operands[0].size), cue.ir_mul), + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), + ], + [cue.IrrepsAndLayout(cue.Irreps("SO3", "16x0"), cue.ir_mul)], + poly, +) + +# Numpy evaluation +w = np.random.randn(ep.inputs[0].dim) +x = np.random.randn(ep.inputs[1].dim) +y = np.random.randn(ep.inputs[2].dim) +[out] = ep(w, x, y) +``` + +## Key file locations + +| Component | Path | +|-----------|------| +| `Irrep` base class | `cuequivariance/group_theory/representations/irrep.py` | +| `Rep` base class | `cuequivariance/group_theory/representations/rep.py` | +| `SO3` | `cuequivariance/group_theory/representations/irrep_so3.py` | +| `O3` | `cuequivariance/group_theory/representations/irrep_o3.py` | +| `SU2` | `cuequivariance/group_theory/representations/irrep_su2.py` | +| `Irreps` | `cuequivariance/group_theory/irreps_array/irreps.py` | +| `IrrepsLayout` | `cuequivariance/group_theory/irreps_array/irreps_layout.py` | +| `IrrepsAndLayout` | `cuequivariance/group_theory/irreps_array/irreps_and_layout.py` | +| `SegmentedTensorProduct` | `cuequivariance/segmented_polynomials/segmented_tensor_product.py` | +| `SegmentedPolynomial` | `cuequivariance/segmented_polynomials/segmented_polynomial.py` | +| `EquivariantPolynomial` | `cuequivariance/group_theory/equivariant_polynomial.py` | +| `IrDictPolynomial` | `cuequivariance/group_theory/ir_dict_polynomial.py` | +| Descriptors | `cuequivariance/group_theory/descriptors/` | +| Tensor product descriptors | `cuequivariance/group_theory/descriptors/irreps_tp.py` | +| `spherical_harmonics` | `cuequivariance/group_theory/descriptors/spherical_harmonics_.py` | +| `symmetric_contraction` | `cuequivariance/group_theory/descriptors/symmetric_contractions.py` | diff --git a/skills/cuequivariance/evals/config.yml b/skills/cuequivariance/evals/config.yml new file mode 100644 index 00000000..7c9468f9 --- /dev/null +++ b/skills/cuequivariance/evals/config.yml @@ -0,0 +1,10 @@ +# Dependency versions below are exercised by the CPU contract checks. +# Build the task environment with scripts/requirements.txt; do not assume +# the shared Harbor base image contains these scientific packages. +schema_version: 1 +harbor: + base_image_mode: rebuild +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance/evals/evals.json b/skills/cuequivariance/evals/evals.json new file mode 100644 index 00000000..977efc84 --- /dev/null +++ b/skills/cuequivariance/evals/evals.json @@ -0,0 +1,77 @@ +{ + "skill_name": "cuequivariance", + "evals": [ + { + "expected_skill": "cuequivariance", + "id": "core-001", + "prompt": "Implement a frozen, hashable cue.Irrep subclass for the three complex one-dimensional irreps of Z3, labelled k=0,1,2. Use the generator exp(2j*pi*k/3), reject invalid labels and strings, and yield the trivial irrep first. Write and run a standalone CPU NumPy script that checks string round trips, the group relation, all tensor-product selection rules, and Clebsch-Gordan covariance. Do not install a GPU framework.", + "expected_output": "Runnable Z3 implementation and executed assertions for all three irreps and all 27 CG triples, with a clear boundary between complex mathematical representations and GPU execution support.", + "assertions": [ + "The immutable dataclass stores and validates k in {0,1,2}; equal irreps are usable as dictionary keys and valid strings round-trip.", + "The generator has shape (1,1,1), its cube is identity, and the continuous-generator and algebra arrays have shapes (0,1,1) and (0,0,0).", + "Tensor products select k3=(k1+k2)%3; allowed CG arrays have shape (1,1,1,1) and disallowed arrays (0,1,1,1).", + "The agent executes the script and numerically checks generator covariance for every allowed coupling, rather than only displaying untested code.", + "It does not claim that complex Z3 representations imply support in CUDA kernels." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-002", + "prompt": "Using only cuequivariance and NumPy on CPU, build the full, unweighted SO3 tensor product of '2x0 + 1x1 + 1x2' and '1x0 + 1x1'. Give the aggregated output irreps and dimension, but preserve the descriptor's actual output order when executing it. Write and run a deterministic example with two batched inputs and verify equivariance under a nonzero rotation.", + "expected_output": "A full_tensor_product descriptor with two inputs, an aggregated output of 3x0 + 5x1 + 3x2 + 1x3 (dimension 40), and a passing numerical equivariance check.", + "assertions": [ + "The descriptor is full_tensor_product with no trainable weight operand; both input arrays and the random seed are defined.", + "The aggregated output is 3x0 + 5x1 + 3x2 + 1x3 with dimension 40.", + "The code uses the descriptor's actual representations and layouts for input/output rotation, without interpreting a regrouped summary as the stored output order.", + "For row-vector batches it applies representation matrices with the appropriate transpose and executes a nontrivial equivariance comparison with a reported tolerance." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-003", + "prompt": "I have SO3 feature blocks with irreps '2x0 + 2x1 + 2x2' and edge blocks '0 + 1 + 2 + 3', each stored as (..., multiplicity, ir.dim). Use channelwise_tensor_product_ir_dict, keeping output irrep types l=0,1,2. On CPU with NumPy, write and run a check against the equivalent dense descriptor using the same nonconstant inputs and weights. Explain operand ordering, layout conversion, and what the third descriptor argument controls.", + "expected_output": "A valid IrDictPolynomial execution through its polynomial and a numerical match against the dense channelwise tensor product after consistent block ordering and layouts.", + "assertions": [ + "It treats IrDictPolynomial as metadata and calls its .polynomial with arrays split according to input_irreps and output_irreps, with the weight group first.", + "Each high-level (...,mul,ir.dim) block is transposed to (...,ir.dim,mul) before flattening for ir_mul; nonconstant data makes a reshape-only bug detectable.", + "It respects each descriptor's actual output ordering when comparing results; any regrouping also reorders the data.", + "It explains that the third channelwise argument filters output irrep types and does not prescribe output multiplicities.", + "The NumPy reference comparison is actually run and passes; no JAX or CUDA installation is needed." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-004", + "prompt": "My NumPy features for SO3 '2x0 + 3x1' are flattened in mul_ir order, but a cue.descriptors.linear descriptor uses ir_mul. Write and run a correct per-block conversion in both directions. Use distinct values in every channel, demonstrate why reshape alone is wrong, and verify the linear operation under a nonzero SO3 rotation.", + "expected_output": "Executed, invertible blockwise layout conversions and an equivariance check on correctly converted data.", + "assertions": [ + "The conversion slices each irrep block and swaps only multiplicity and representation axes, preserving batch axes and block order.", + "It tests exact round trips on nonconstant data and demonstrates a difference from reshape-only conversion for the vector block.", + "The linear descriptor is evaluated with matching weights and layouts, and rotations use row-vector conventions consistently.", + "A nontrivial numerical equivariance assertion is executed and passes." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-005", + "prompt": "I need a compatibility path for an older CPU-only deployment where cuequivariance imports but may not expose IrDictPolynomial or channelwise_tensor_product_ir_dict. That deployment cannot upgrade or install JAX or PyTorch. Show version and capability checks and a dense NumPy fallback for a channelwise tensor product. The sandbox may have a newer version; do not assume its available APIs exist in the older deployment and do not change any dependencies.", + "expected_output": "Capability checks and a dense NumPy fallback, without requiring the missing API or changing dependencies.", + "assertions": [ + "It checks or shows checks for cue.__version__ and hasattr/getattr for the missing APIs, rather than assuming this skill's newest API is available.", + "It proposes cue.descriptors.channelwise_tensor_product and NumPy execution if present, making any unavailable fallback explicit.", + "It explains how to convert per-irrep blocks to the dense descriptor's layout and preserve operand order.", + "It does not install or upgrade packages, switch frameworks, or pretend the unavailable IrDictPolynomial call succeeded." + ] + }, + { + "expected_skill": null, + "id": "core-006", + "prompt": "Explain how to choose a PostgreSQL index for SELECT * FROM events WHERE tenant_id = 42 AND created_at >= now() - interval '1 day' ORDER BY created_at DESC LIMIT 100. I only need database advice.", + "expected_output": "An answer about PostgreSQL indexing without using cuEquivariance.", + "assertions": [ + "It answers the database indexing question directly.", + "It does not import, install, or invoke cuEquivariance or force the question into group theory or tensor products." + ] + } + ] +} diff --git a/skills/cuequivariance/scripts/requirements.txt b/skills/cuequivariance/scripts/requirements.txt new file mode 100644 index 00000000..3a1a557b --- /dev/null +++ b/skills/cuequivariance/scripts/requirements.txt @@ -0,0 +1,3 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Pin the source revision because main can contain APIs newer than PyPI. +cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md new file mode 100644 index 00000000..b1789a3a --- /dev/null +++ b/skills/cuequivariance/skill-card.md @@ -0,0 +1,111 @@ +## Description:
+Use when defining cuEquivariance groups and irreps, building equivariant tensor products, or validating segmented polynomials with NumPy.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers building geometric (equivariant) neural networks use this skill to define custom groups and irreps, build segmented tensor products with Clebsch-Gordan coefficients, use built-in descriptors, and validate equivariant polynomials with NumPy on CPU.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [No]
+**Credential Type(s):** [None]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [Bundled CPU requirements](scripts/requirements.txt)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+**Overall verdict:** ✅ PASS — Recommended for publication + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 75.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 33.3% (-50.0 points) | 100.0% → 75.0% (-25.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 91.7% (-4.1 points) | 74.2% → 79.6% (+5.4 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 68.1% — baseline ran, but no comparable score was available; uplift unavailable | + +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 87.7% → 86.7% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 78.6% → 82.9% (+4.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Skill Version(s):
+605793b (source: git SHA, committed 2026-11-03)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig new file mode 100644 index 00000000..695c19ac --- /dev/null +++ b/skills/cuequivariance/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNjZmYjkzM2NjNDgxZjBjNDRkODJlZGEyZDQ3OGM4ZWVjYzhjMjg1Y2UzNDhhNDQyMjMxMjE5YzVkMDQwZjhmOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ5MjgwNWQ1ZjFkNjYyMGI1M2JiZWY3NzE2NGQ3NmVlMmU1NWI3NDAyMjI5MGQwNDk5Njg1OTY0Y2Y4MTM4NmEiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjczNjRjYzdkZTNmMDExMDZkNzFjNDZjNDBkMzcyYjFjZGFjOTUxMmJlMWQ0M2ZlMDkxMGI5M2NhMTNiYjUyMWYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNGY1YjU2NTYyZmUzYTY1NDAzYzE3YWNhNmEzYjYzN2Y0ZDNjNmRmNzczN2FhYTdjZGFkNWE5NTAyOTQyNTUzIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMBAu9wTJNRJ7WiH56Rwv6LrXwAOkJiir9Do+keqlD6iQ2QrM4C5Qgb4Fg5tZr1VxGwIwLenEHwhR3dqB/BeGwkSgXOYDaExvesDVp/qhU2WPHzeJQ4TtE1CmcoW59Pdu0gZ6","keyid":""}]}} \ No newline at end of file