From 5e144b44a5a89dc66b372fe6351b88cfaaacfb28 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Fri, 16 Oct 2026 10:03:39 -0700 Subject: [PATCH 01/19] Sign skill Signed-off-by: Ohad Mosafi --- .github/scripts/check_skill_packaging.py | 115 +++++ .github/tests/test_core_skill_evals.py | 155 ++++++ .github/tests/test_jax_skill_evals.py | 124 +++++ .github/tests/test_torch_skill_evals.py | 118 +++++ .github/workflows/publish.yml | 2 + .github/workflows/request-nvskills-ci.yml | 26 ++ .github/workflows/tests.yml | 20 + README.md | 15 +- cuequivariance/cuequivariance/SKILL.md | 418 +---------------- .../cuequivariance_jax/SKILL.md | 411 +--------------- .../cuequivariance_torch/SKILL.md | 401 +--------------- skills/README.md | 128 +++++ skills/cuequivariance-jax/SKILL.md | 435 +++++++++++++++++ skills/cuequivariance-jax/evals/config.yml | 10 + skills/cuequivariance-jax/evals/evals.json | 76 +++ .../scripts/requirements.txt | 6 + skills/cuequivariance-torch/SKILL.md | 418 +++++++++++++++++ skills/cuequivariance-torch/evals/config.yml | 10 + skills/cuequivariance-torch/evals/evals.json | 74 +++ .../scripts/requirements.txt | 6 + skills/cuequivariance/SKILL.md | 442 ++++++++++++++++++ skills/cuequivariance/evals/config.yml | 10 + skills/cuequivariance/evals/evals.json | 77 +++ .../cuequivariance/scripts/requirements.txt | 3 + 24 files changed, 2272 insertions(+), 1228 deletions(-) create mode 100644 .github/scripts/check_skill_packaging.py create mode 100644 .github/tests/test_core_skill_evals.py create mode 100644 .github/tests/test_jax_skill_evals.py create mode 100644 .github/tests/test_torch_skill_evals.py create mode 100644 .github/workflows/request-nvskills-ci.yml mode change 100644 => 120000 cuequivariance/cuequivariance/SKILL.md mode change 100644 => 120000 cuequivariance_jax/cuequivariance_jax/SKILL.md mode change 100644 => 120000 cuequivariance_torch/cuequivariance_torch/SKILL.md create mode 100644 skills/README.md create mode 100644 skills/cuequivariance-jax/SKILL.md create mode 100644 skills/cuequivariance-jax/evals/config.yml create mode 100644 skills/cuequivariance-jax/evals/evals.json create mode 100644 skills/cuequivariance-jax/scripts/requirements.txt create mode 100644 skills/cuequivariance-torch/SKILL.md create mode 100644 skills/cuequivariance-torch/evals/config.yml create mode 100644 skills/cuequivariance-torch/evals/evals.json create mode 100644 skills/cuequivariance-torch/scripts/requirements.txt create mode 100644 skills/cuequivariance/SKILL.md create mode 100644 skills/cuequivariance/evals/config.yml create mode 100644 skills/cuequivariance/evals/evals.json create mode 100644 skills/cuequivariance/scripts/requirements.txt diff --git a/.github/scripts/check_skill_packaging.py b/.github/scripts/check_skill_packaging.py new file mode 100644 index 00000000..d84fb97f --- /dev/null +++ b/.github/scripts/check_skill_packaging.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Check skill resources and CLI through the same build path as publish.yml.""" + +import argparse +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PACKAGES = ("cuequivariance", "cuequivariance_jax", "cuequivariance_torch") + + +def check_cli(package, import_root, expected, cwd): + env = {**os.environ, "PYTHONPATH": str(import_root)} + # Verify import provenance too: an editable install must not hide a missing + # resource or module in the wheel being checked. + subprocess.run( + [ + sys.executable, + "-c", + "import importlib, pathlib, sys; " + "m = importlib.import_module(sys.argv[1]); " + "assert pathlib.Path(m.__file__).is_relative_to(sys.argv[2]), m.__file__", + package, + str(import_root), + ], + cwd=cwd, + env=env, + check=True, + ) + actual = subprocess.check_output( + [sys.executable, "-m", package, "skill"], cwd=cwd, env=env + ) + assert actual == expected + b"\n", ( + f"{package}: CLI output differs from canonical skill" + ) + + +def build(project, kind, destination): + subprocess.run( + [ + sys.executable, + "-m", + "hatchling", + "build", + "-t", + kind, + "-d", + str(destination), + ], + cwd=project, + check=True, + ) + + +def check_package(package, temp): + canonical = ROOT / "skills" / package.replace("_", "-") / "SKILL.md" + expected = canonical.read_bytes() + assert not canonical.is_symlink(), f"{canonical}: nv-carps needs a real file" + assert (ROOT / package / package / "SKILL.md").read_bytes() == expected + check_cli(package, ROOT / package, expected, temp) + + # Publishing already dereferences LICENSE, README, VERSION and SKILL links. + # Copy with dereferencing to exercise that behavior without modifying the + # developer's checkout. Direct sdists from the linked tree are not releases. + project = temp / "source" / package + shutil.copytree( + ROOT / package, + project, + symlinks=False, + ignore=shutil.ignore_patterns( + "__pycache__", ".pytest_cache", "dist", "*.egg-info" + ), + ) + wheel_dir, sdist_dir = temp / "wheel", temp / "sdist" + build(project, "wheel", wheel_dir) + build(project, "sdist", sdist_dir) + archive = next(sdist_dir.glob("*.tar.gz")) + with tarfile.open(archive) as source: + members = [ + m for m in source.getmembers() if m.name.endswith(f"/{package}/SKILL.md") + ] + assert len(members) == 1 and members[0].isfile(), ( + "sdist must contain a regular SKILL.md" + ) + assert source.extractfile(members[0]).read() == expected + source.extractall(temp / "unpacked", filter="data") + rebuilt = temp / "rebuilt" + build(next((temp / "unpacked").iterdir()), "wheel", rebuilt) + for label, directory in (("wheel", wheel_dir), ("rebuilt", rebuilt)): + wheel = next(directory.glob("*.whl")) + import_root = temp / f"import-{label}" + with zipfile.ZipFile(wheel) as contents: + assert contents.read(f"{package}/SKILL.md") == expected + contents.extractall(import_root) + check_cli(package, import_root, expected, temp) + print(f"{package}: source CLI, wheel, sdist, and rebuilt wheel passed") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package", choices=PACKAGES) + args = parser.parse_args() + for package in (args.package,) if args.package else PACKAGES: + with tempfile.TemporaryDirectory(prefix=f"{package}-skill-packaging-") as temp: + check_package(package, Path(temp)) + + +if __name__ == "__main__": + main() diff --git a/.github/tests/test_core_skill_evals.py b/.github/tests/test_core_skill_evals.py new file mode 100644 index 00000000..66670b25 --- /dev/null +++ b/.github/tests/test_core_skill_evals.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical contracts behind the core skill evals; not agent benchmark scores.""" + +import dataclasses +import re + +import numpy as np +import pytest + +import cuequivariance as cue + + +@dataclasses.dataclass(frozen=True) +class Z3(cue.Irrep): + k: int + + def __post_init__(self): + if type(self.k) is not int or self.k not in (0, 1, 2): + raise ValueError("Z3 labels are integers 0, 1, 2") + + @classmethod + def regexp_pattern(cls): + return re.compile(r"([012])") + + @classmethod + def from_string(cls, string): + if string not in ("0", "1", "2"): + raise ValueError(string) + return cls(int(string)) + + def __repr__(self): + return str(self.k) + + def __lt__(self, other): + return self.k < other.k + + def __mul__(self, other): + return [Z3((self.k + other.k) % 3)] + + @classmethod + def iterator(cls): + return iter([cls(0), cls(1), cls(2)]) + + @classmethod + def clebsch_gordan(cls, a, b, c): + return np.ones((1, 1, 1, 1)) if c in a * b else np.zeros((0, 1, 1, 1)) + + @property + def dim(self): + return 1 + + def discrete_generators(self): + return np.array([[[np.exp(2j * np.pi * self.k / 3)]]]) + + def continuous_generators(self): + return np.zeros((0, 1, 1)) + + def algebra(self): + return np.zeros((0, 0, 0)) + + +def test_z3_group_and_cg_covariance(): + irreps = list(Z3.iterator()) + assert irreps[0] == Z3(0) + assert len(set(irreps)) == 3 + for invalid in (-1, 3, 1.5, True): + with pytest.raises(ValueError): + Z3(invalid) + for invalid in ("-1", "3", "01", "bad"): + with pytest.raises(ValueError): + Z3.from_string(invalid) + for a in irreps: + assert Z3.from_string(str(a)) == a + np.testing.assert_allclose(a.discrete_generators() ** 3, 1, atol=1e-14) + assert a.continuous_generators().shape == (0, 1, 1) + assert a.algebra().shape == (0, 0, 0) + for b in irreps: + assert a * b == [Z3((a.k + b.k) % 3)] + for c in irreps: + cg = cue.clebsch_gordan(a, b, c) + allowed = (a.k + b.k) % 3 == c.k + assert cg.shape == (int(allowed), 1, 1, 1) + if allowed: + np.testing.assert_allclose( + a.discrete_generators() * b.discrete_generators() * cg, + cg * c.discrete_generators(), + atol=1e-14, + ) + + +def test_full_tensor_product_dimension_and_equivariance(): + e = cue.descriptors.full_tensor_product( + cue.Irreps("SO3", "2x0 + 1 + 2"), cue.Irreps("SO3", "0 + 1") + ) + assert len(e.inputs) == 2 + assert e.outputs[0].irreps.regroup() == cue.Irreps("SO3", "3x0+5x1+3x2+3") + assert e.outputs[0].dim == 40 + rng = np.random.default_rng(12) + x = [rng.normal(size=(4, rep.dim)) for rep in e.inputs] + rotation = np.array([0.31, -0.19, 0.43]) + matrices = [rep.exp_map(rotation, []) for rep in (*e.inputs, *e.outputs)] + np.testing.assert_allclose( + e(*(v @ d.T for v, d in zip(x, matrices)))[0], + e(*x)[0] @ matrices[-1].T, + atol=1e-12, + ) + + +def test_ir_dict_operand_contract_and_dense_reference(): + a = cue.Irreps("SO3", "2x0+2x1+2x2") + b = cue.Irreps("SO3", "0+1+2+3") + keep = [cue.SO3(degree) for degree in range(3)] + split = cue.descriptors.channelwise_tensor_product_ir_dict(a, b, keep) + # The ir_dict descriptor simplifies outputs automatically. Request the + # same representation ordering in the dense reference as well. + dense = cue.descriptors.channelwise_tensor_product( + a, b, keep, simplify_irreps3=True + ) + rng = np.random.default_rng(14) + blocks = [ + [rng.normal(size=(3, mul, ir.dim)) for mul, ir in group] + for group in split.input_irreps + ] + operands = [[v.swapaxes(-1, -2).reshape(3, -1) for v in group] for group in blocks] + actual = split.polynomial(*(v for group in operands for v in group)) + expected = dense(*(np.concatenate(group, axis=-1) for group in operands))[0] + assert split.output_irreps[0] == dense.outputs[0].irreps + np.testing.assert_allclose(np.concatenate(actual, axis=-1), expected, atol=1e-12) + + +def test_layout_conversion_and_linear_equivariance(): + irreps = cue.Irreps("SO3", "2x0+3x1") + x = np.arange(2 * irreps.dim, dtype=float).reshape(2, -1) / 7 + + def convert(data, source_layout): + pieces, offset = [], 0 + for mul, ir in irreps: + shape = (mul, ir.dim) if source_layout == cue.mul_ir else (ir.dim, mul) + block = data[..., offset : offset + mul * ir.dim] + pieces.append( + block.reshape(*data.shape[:-1], *shape) + .swapaxes(-1, -2) + .reshape(*data.shape[:-1], -1) + ) + offset += mul * ir.dim + return np.concatenate(pieces, axis=-1) + + flat = convert(x, cue.mul_ir) + np.testing.assert_array_equal(convert(flat, cue.ir_mul), x) + assert not np.array_equal(flat, x) + e = cue.descriptors.linear(irreps, irreps) + w = np.random.default_rng(9).normal(size=e.inputs[0].dim) + rotation = np.array([0.2, 0.3, -0.4]) + d = e.inputs[1].exp_map(rotation, []) + np.testing.assert_allclose(e(w, flat @ d.T)[0], e(w, flat)[0] @ d.T, atol=1e-12) diff --git a/.github/tests/test_jax_skill_evals.py b/.github/tests/test_jax_skill_evals.py new file mode 100644 index 00000000..b53c8f1c --- /dev/null +++ b/.github/tests/test_jax_skill_evals.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU API and numerical contracts for the JAX skill evaluation prompts.""" + +import cuequivariance_jax.ir_dict as ir_dict +import cuequivariance_jax.nnx as cue_nnx +import jax +import jax.numpy as jnp +import numpy as np +from flax import nnx + +import cuequivariance as cue +import cuequivariance_jax as cuex + +jax.config.update("jax_enable_x64", True) + + +def test_broadcast_jit_and_finite_difference(): + e = cue.descriptors.channelwise_tensor_product( + cue.Irreps("SO3", "2x0+2x1"), + cue.Irreps("SO3", "0+1"), + [cue.SO3(0), cue.SO3(1)], + ) + rng = np.random.default_rng(21) + w = rng.normal(size=e.inputs[0].dim) + x = rng.normal(size=(2, 1, e.inputs[1].dim)) + y = rng.normal(size=(1, 3, e.inputs[2].dim)) + + def run(weights): + return cuex.segmented_polynomial( + e.polynomial, + [weights, jnp.asarray(x), jnp.asarray(y)], + [jax.ShapeDtypeStruct((2, 3, e.outputs[0].dim), jnp.float64)], + method="naive", + )[0] + + expected = e(w, x, y)[0] + np.testing.assert_allclose(run(jnp.asarray(w)), expected, atol=1e-12) + np.testing.assert_allclose(jax.jit(run)(jnp.asarray(w)), expected, atol=1e-12) + grad = jax.grad(lambda v: jnp.sum(run(v) ** 2))(jnp.asarray(w)) + eps = 1e-5 + wp, wm = w.copy(), w.copy() + wp[0] += eps + wm[0] -= eps + finite_diff = (np.sum(e(wp, x, y)[0] ** 2) - np.sum(e(wm, x, y)[0] ** 2)) / ( + 2 * eps + ) + assert abs(grad[0]) > 1e-5 + np.testing.assert_allclose(grad[0], finite_diff, rtol=1e-7, atol=1e-7) + + +def test_dict_layout_conversion_is_not_a_flat_reshape(): + irreps = cue.Irreps("SO3", "2x0+3x1") + flat = jnp.arange(2 * irreps.dim).reshape(2, -1) + blocks = ir_dict.flat_to_dict(irreps, flat, layout="ir_mul") + ir_dict.assert_mul_ir_dict(irreps, blocks) + np.testing.assert_array_equal( + blocks[cue.SO3(1)], np.asarray(flat[:, 2:]).reshape(2, 3, 3).swapaxes(-1, -2) + ) + mul_ir = ir_dict.dict_to_flat(irreps, blocks) + assert not np.array_equal(mul_ir, flat) + restored = jnp.concatenate( + [blocks[ir].swapaxes(-1, -2).reshape(2, -1) for _, ir in irreps], axis=-1 + ) + np.testing.assert_array_equal(restored, flat) + + +def test_nnx_linear_normalization_missing_irrep_and_equivariance(): + a, b = cue.Irreps("SO3", "2x0+3x1"), cue.Irreps("SO3", "4x0+2x1+2") + layer = cue_nnx.IrrepsLinear(a, b, dtype=jnp.float64, rngs=nnx.Rngs(22)) + x = { + ir: jnp.asarray(np.random.default_rng(23 + ir.l).normal(size=(2, mul, ir.dim))) + for mul, ir in a + } + y = layer(x) + ir_dict.assert_mul_ir_dict(b, y) + np.testing.assert_array_equal(y[cue.SO3(2)], 0) + for mul, ir in a: + expected = np.einsum( + "uv,bui->bvi", np.asarray(layer.w[str(ir)][...]), x[ir] + ) / np.sqrt(mul) + np.testing.assert_allclose(y[ir], expected, atol=1e-12) + rotation = np.array([0.2, -0.3, 0.4]) + rotated = layer({ir: v @ ir.exp_map(rotation, []).T for ir, v in x.items()}) + for ir in y: + np.testing.assert_allclose( + rotated[ir], y[ir] @ ir.exp_map(rotation, []).T, atol=1e-12 + ) + + +def test_ir_dict_naive_matches_numpy(): + e = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("SO3", "2x0+2x1+2x2"), + cue.Irreps("SO3", "0+1+2+3"), + [cue.SO3(degree) for degree in range(3)], + ) + rng = np.random.default_rng(24) + inputs = [ + rng.normal(size=(2, mul, ir.dim)).swapaxes(-1, -2).reshape(2, -1) + for group in e.input_irreps + for mul, ir in group + ] + result = cuex.segmented_polynomial( + e.polynomial, + list(map(jnp.asarray, inputs)), + [ + jax.ShapeDtypeStruct((2, op.size), jnp.float64) + for op in e.polynomial.outputs + ], + method="naive", + ) + expected = e.polynomial(*inputs) + for actual, reference in zip(result, expected): + np.testing.assert_allclose(actual, reference, atol=1e-12) + + +def test_nnx_spherical_harmonics_cpu(): + layer = cue_nnx.SphericalHarmonics(max_degree=2) + output = layer(jnp.array([[1.0, 2.0, 3.0], [-0.4, 0.8, 0.2]])) + assert {ir.l: value.shape for ir, value in output.items()} == { + 0: (2, 1, 1), + 1: (2, 1, 3), + 2: (2, 1, 5), + } + assert all(np.isfinite(value).all() for value in output.values()) diff --git a/.github/tests/test_torch_skill_evals.py b/.github/tests/test_torch_skill_evals.py new file mode 100644 index 00000000..acb80289 --- /dev/null +++ b/.github/tests/test_torch_skill_evals.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU reference and autograd contracts for the PyTorch skill evals.""" + +import numpy as np +import torch + +import cuequivariance as cue +import cuequivariance_torch as cuet + + +def test_spherical_harmonics_numpy_and_gradcheck(): + torch.manual_seed(30) + x = torch.randn(2, 3, dtype=torch.float64, requires_grad=True) + module = cuet.SphericalHarmonics([0, 1, 2], normalize=False, method="naive") + e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + y = module(x) + assert y.shape == (2, 9) + np.testing.assert_allclose(y.detach().numpy(), e(x.detach().numpy())[0], atol=1e-12) + assert torch.autograd.gradcheck(module, (x,)) + + +def transpose_blocks(data, irreps, source_layout): + pieces, offset = [], 0 + for mul, ir in irreps: + shape = (mul, ir.dim) if source_layout == cue.mul_ir else (ir.dim, mul) + block = data[..., offset : offset + mul * ir.dim] + pieces.append( + block.reshape(*data.shape[:-1], *shape) + .transpose(-1, -2) + .reshape(*data.shape[:-1], -1) + ) + offset += mul * ir.dim + return torch.cat(pieces, dim=-1) + + +def test_linear_layouts_rotation_and_gradients(): + torch.manual_seed(31) + a, b = cue.Irreps("SO3", "2x0+3x1"), cue.Irreps("SO3", "3x0+2x1") + kwargs = dict(internal_weights=False, method="naive", dtype=torch.float64) + mul = cuet.Linear(a, b, layout=cue.mul_ir, **kwargs) + ir = cuet.Linear(a, b, layout=cue.ir_mul, **kwargs) + x = torch.randn(2, a.dim, dtype=torch.float64, requires_grad=True) + w = torch.randn(1, mul.weight_numel, dtype=torch.float64, requires_grad=True) + y = mul(x, w) + y_ir = ir(transpose_blocks(x, a, cue.mul_ir), w) + torch.testing.assert_close(y, transpose_blocks(y_ir, b, cue.ir_mul)) + rotation = np.array([0.3, -0.1, 0.4]) + da, db = [ + torch.from_numpy(cue.IrrepsAndLayout(rep, cue.mul_ir).exp_map(rotation, [])) + for rep in (a, b) + ] + torch.testing.assert_close(mul(x @ da.T, w), y @ db.T) + for grad in torch.autograd.grad(y.square().sum(), (x, w)): + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + + +def test_channelwise_external_unshared_weights(): + torch.manual_seed(32) + a, b = cue.Irreps("SO3", "2x0+2x1"), cue.Irreps("SO3", "0+1") + keep = [cue.SO3(0), cue.SO3(1)] + module = cuet.ChannelWiseTensorProduct( + a, + b, + keep, + shared_weights=False, + internal_weights=False, + layout=cue.ir_mul, + method="naive", + dtype=torch.float64, + ) + e = cue.descriptors.channelwise_tensor_product(a, b, keep) + x, y, w = [ + torch.randn(3, size, dtype=torch.float64, requires_grad=True) + for size in (a.dim, b.dim, module.weight_numel) + ] + actual = module(x, y, w) + expected = e(*(v.detach().numpy() for v in (w, x, y)))[0] + np.testing.assert_allclose(actual.detach().numpy(), expected, atol=1e-12) + for grad in torch.autograd.grad(actual.square().sum(), (x, y, w)): + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + + +def test_gather_scatter_repeated_receivers_and_gradients(): + torch.manual_seed(33) + e = cue.descriptors.linear(cue.Irreps("SO3", "2x0"), cue.Irreps("SO3", "3x0")) + module = cuet.SegmentedPolynomial(e.polynomial, method="naive") + x = torch.randn(4, 2, dtype=torch.float64, requires_grad=True) + w = torch.randn(1, e.inputs[0].dim, dtype=torch.float64, requires_grad=True) + senders = torch.tensor([0, 1, 0, 3, 1]) + receivers = torch.tensor([2, 0, 2, 1, 2]) + shape = torch.empty(4, 1, dtype=torch.float64) + [actual] = module( + [w, x], + input_indices={1: senders}, + output_indices={0: receivers}, + output_shapes={0: shape}, + ) + [edges] = module([w, x[senders]]) + expected = torch.zeros(4, 3, dtype=torch.float64).index_add_(0, receivers, edges) + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(actual[3], torch.zeros(3, dtype=torch.float64)) + actual_grads = torch.autograd.grad(actual.square().sum(), (x, w)) + expected_grads = torch.autograd.grad(expected.square().sum(), (x, w)) + for actual_grad, expected_grad in zip(actual_grads, expected_grads): + torch.testing.assert_close(actual_grad, expected_grad) + + +def test_fully_connected_tensor_product_cpu(): + torch.manual_seed(34) + irreps = cue.Irreps("SO3", "0+1") + module = cuet.FullyConnectedTensorProduct( + irreps, irreps, irreps, method="naive", dtype=torch.float64 + ) + x = torch.randn(2, irreps.dim, dtype=torch.float64, requires_grad=True) + result = module(x, x) + assert result.shape == x.shape + result.square().sum().backward() + assert torch.isfinite(x.grad).all() and x.grad.abs().sum() > 0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2c371767..36f17de8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,6 +33,8 @@ jobs: - name: Replace symlinks with actual files run: | + # Includes package SKILL.md links into the canonical skills/ tree. + # Keep this before building: sdists must carry regular resource files. find . -type l -exec sh -c 'cp --remove-destination "$(readlink -f "$1")" "$1"' _ {} \; - name: Build & Publish cuequivariance diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 00000000..4092d220 --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,26 @@ +name: Request NVSkills CI + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + +jobs: + request: + if: > + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + statuses: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@main + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 920e7fbd..0bff3678 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,12 @@ jobs: run: | pytest --doctest-modules -x -m "not slow" cuequivariance + - name: Check core skill evaluations and packaging + run: | + pytest -q .github/tests/test_core_skill_evals.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance + - name: Downgrade numpy if: matrix.python-version == '3.10' run: | @@ -62,6 +68,14 @@ jobs: run: | XLA_PYTHON_CLIENT_PREALLOCATE=false pytest --doctest-modules -x -m "not slow" cuequivariance_jax + - name: Check JAX skill evaluations and packaging + env: + JAX_PLATFORMS: cpu + run: | + pytest -q .github/tests/test_jax_skill_evals.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance_jax + cuequivariance-jax-no-flax: runs-on: ubuntu-latest @@ -105,3 +119,9 @@ jobs: - name: Test with pytest run: | pytest --doctest-modules -x -m "not slow" cuequivariance_torch + + - name: Check PyTorch skill evaluations and packaging + run: | + pytest -q .github/tests/test_torch_skill_evals.py + uv pip install hatchling + python .github/scripts/check_skill_packaging.py --package cuequivariance_torch diff --git a/README.md b/README.md index d374df55..c2b132aa 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,20 @@ pip install cuequivariance-ops-jax-cu12 # or -cu13 pip install cuequivariance-ops-torch-cu12 # or -cu13 ``` +## Agent skills + +The repository provides three independent agent skills: + +- [cuequivariance](skills/cuequivariance/SKILL.md): group theory, descriptors, and NumPy execution. +- [cuequivariance-jax](skills/cuequivariance-jax/SKILL.md): JAX execution and Flax NNX layers. +- [cuequivariance-torch](skills/cuequivariance-torch/SKILL.md): PyTorch operations and layers. + +Installed Python packages also expose these instructions through +`python -m cuequivariance skill`, `python -m cuequivariance_jax skill`, and +`python -m cuequivariance_torch skill`. +See [skill maintenance and validation](skills/README.md) for source layout, +evaluation coverage, packaging checks, and NVSkills CI onboarding. + ## License All files hosted in this repository are subject to the Apache 2.0 license. @@ -28,4 +42,3 @@ All files hosted in this repository are subject to the Apache 2.0 license. ## Disclaimer cuEquivariance is in a Beta state. Beta products may not be fully functional, may contain errors or design flaws, and may be changed at any time without notice. We appreciate your feedback to improve and iterate on our Beta products. - diff --git a/cuequivariance/cuequivariance/SKILL.md b/cuequivariance/cuequivariance/SKILL.md deleted file mode 100644 index f14cbd8b..00000000 --- a/cuequivariance/cuequivariance/SKILL.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -name: cuequivariance -description: Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics). Use when working with cuequivariance group theory, irreps, or segmented polynomials. ---- - -# cuequivariance: Groups, Irreps, and Segmented Polynomials - -## Overview - -`cuequivariance` (imported as `cue`) provides two core abstractions: - -1. **Group theory**: `Irrep` subclasses define irreducible representations of Lie groups (SO3, O3, SU2, or custom). `Irreps` manages collections with multiplicities. -2. **Segmented polynomials**: `SegmentedTensorProduct` describes tensor contractions over segments of varying shape, linked by `Path` objects carrying Clebsch-Gordan coefficients. `SegmentedPolynomial` wraps multiple STPs into a polynomial with named inputs/outputs. Two higher-level wrappers attach group representations: - - `EquivariantPolynomial` — dense operands with `IrrepsAndLayout` metadata - - `IrDictPolynomial` — operands already split by irrep, with per-group `Irreps` metadata for the `dict[Irrep, Array]` workflow - -## Defining a custom group - -Subclass `cue.Irrep` (a frozen dataclass) and implement: - -```python -from __future__ import annotations -import dataclasses -import re -from typing import Iterator -import numpy as np -import cuequivariance as cue - -@dataclasses.dataclass(frozen=True) -class Z2(cue.Irrep): - odd: bool # dataclass field -- required for correct __eq__ and __hash__ - - # No __init__ needed -- @dataclass(frozen=True) generates it: Z2(odd=True) - - @classmethod - def regexp_pattern(cls) -> re.Pattern: - # Pattern whose first group is passed to from_string - return re.compile(r"(odd|even)") - - @classmethod - def from_string(cls, string: str) -> Z2: - return cls(odd=string == "odd") - - def __repr__(rep: Z2) -> str: - return "odd" if rep.odd else "even" - - def __mul__(rep1: Z2, rep2: Z2) -> Iterator[Z2]: - # Selection rule: which irreps appear in the tensor product rep1 x rep2 - return [Z2(odd=rep1.odd ^ rep2.odd)] - - @classmethod - def clebsch_gordan(cls, rep1: Z2, rep2: Z2, rep3: Z2) -> np.ndarray: - # Shape: (num_paths, rep1.dim, rep2.dim, rep3.dim) - if rep3 in rep1 * rep2: - return np.array([[[[1]]]]) - else: - return np.zeros((0, 1, 1, 1)) - - @property - def dim(rep: Z2) -> int: - return 1 - - def __lt__(rep1: Z2, rep2: Z2) -> bool: - # Ordering for sorting; dimension is compared first by the base class - return rep1.odd < rep2.odd - - @classmethod - def iterator(cls) -> Iterator[Z2]: - # Must yield trivial irrep first - for odd in [False, True]: - yield Z2(odd=odd) - - def discrete_generators(rep: Z2) -> np.ndarray: - # Shape: (num_generators, dim, dim) - if rep.odd: - return -np.ones((1, 1, 1)) - else: - return np.ones((1, 1, 1)) - - def continuous_generators(rep: Z2) -> np.ndarray: - # Shape: (lie_dim, dim, dim) -- Z2 is discrete, so lie_dim=0 - return np.zeros((0, rep.dim, rep.dim)) - - def algebra(self) -> np.ndarray: - # Shape: (lie_dim, lie_dim, lie_dim) -- structure constants [X_i, X_j] = A_ijk X_k - return np.zeros((0, 0, 0)) - - -# Usage: -irreps = cue.Irreps(Z2, "3x odd + 2x even") # dim=5 -``` - -### Required methods summary - -| Method | Returns | Purpose | -|--------|---------|---------| -| `regexp_pattern()` | `re.Pattern` | Parse string like `"1"`, `"0e"`, `"odd"` | -| `from_string(s)` | `Irrep` | Construct from matched string | -| `__repr__` | `str` | Canonical string form | -| `__mul__(a, b)` | `Iterator[Irrep]` | Selection rule for tensor product | -| `clebsch_gordan(a, b, c)` | `ndarray (n, d1, d2, d3)` | CG coefficients | -| `dim` (property) | `int` | Dimension of representation | -| `__lt__(a, b)` | `bool` | Ordering (dimension first, then custom) | -| `iterator()` | `Iterator[Irrep]` | All irreps, trivial first | -| `continuous_generators()` | `ndarray (lie_dim, dim, dim)` | Lie algebra generators | -| `discrete_generators()` | `ndarray (n, dim, dim)` | Finite symmetry generators | -| `algebra()` | `ndarray (lie_dim, lie_dim, lie_dim)` | Structure constants | - -### Built-in groups - -- **`cue.SO3(l)`**: 3D rotations. `l` is a non-negative integer. `dim = 2l+1`. String: `"0"`, `"1"`, `"2"`. -- **`cue.O3(l, p)`**: 3D rotations + parity. `p=+1` (even) or `p=-1` (odd). String: `"0e"`, `"1o"`, `"2e"`. -- **`cue.SU2(j)`**: Spin group. `j` is a non-negative half-integer. String: `"0"`, `"1/2"`, `"1"`. - -## Irreps and layout - -```python -irreps = cue.Irreps("SO3", "16x0 + 4x1 + 2x2") # 16 scalars, 4 vectors, 2 rank-2 -irreps.dim # 16*1 + 4*3 + 2*5 = 38 - -for mul, ir in irreps: - print(mul, ir, ir.dim) # 16 0 1, then 4 1 3, then 2 2 5 -``` - -`IrrepsLayout` controls memory order within each `(mul, ir)` block: - -- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used by all descriptors and ir_dict internally** -- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **used by nnx `dict[Irrep, Array]` and PyTorch** - -`IrrepsAndLayout` combines irreps with a layout into a `Rep`: - -```python -rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) -rep.dim # 10 -``` - -## Building a SegmentedTensorProduct from scratch - -The subscripts string uses Einstein notation. Operands are comma-separated, coefficient modes follow `+`. - -```python -# Matrix-vector multiply: y_i = sum_j M_ij * x_j -d = cue.SegmentedTensorProduct.from_subscripts("ij,j,i") -d.add_segment(0, (3, 4)) # operand 0: matrix segment of shape (3, 4) -d.add_segment(1, (4,)) # operand 1: vector of size 4 -d.add_segment(2, (3,)) # operand 2: output vector of size 3 -d.add_path(0, 0, 0, c=1.0) # link segments 0,0,0 with coefficient=1.0 - -poly = cue.SegmentedPolynomial.eval_last_operand(d) # last operand becomes output -[y] = poly(M_flat, x) # numpy evaluation -``` - -### Multi-segment STP (how descriptors work internally) - -Descriptors build STPs with multiple segments per operand. Each segment corresponds to an irrep block: - -```python -# Linear equivariant map: output[iv] = sum_u weight[uv] * input[iu] -d = cue.SegmentedTensorProduct.from_subscripts("uv,iu,iv") - -# Segment for l=1: ir_dim=3, mul_in=2, mul_out=5 -s_in_0 = d.add_segment(1, (3, 2)) # input block -s_out_0 = d.add_segment(2, (3, 5)) # output block -d.add_path((2, 5), s_in_0, s_out_0, c=1.0) - -# Segment for l=0: ir_dim=1, mul_in=4, mul_out=3 -s_in_1 = d.add_segment(1, (1, 4)) -s_out_1 = d.add_segment(2, (1, 3)) -d.add_path((4, 3), s_in_1, s_out_1, c=1.0) -``` - -### Weights operand - -For weighted tensor products (subscript starting with `uvw` or `uv`), the first operand is always weights. The weight segment shape is `(mul_1, mul_2, ...)` matching the multiplicity modes. The weights operand gets `new_scalars()` irreps since weights are invariant. - -### CG coefficients as path coefficients - -```python -d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") -# For each pair of input irreps and each output irrep in the selection rule: -for cg in cue.clebsch_gordan(ir1, ir2, ir3): - # cg has shape (ir1.dim, ir2.dim, ir3.dim) - d.add_path((mul1, mul2, mul3), seg_in1, seg_in2, seg_out, c=cg) -``` - -## Descriptors - -All descriptors come in two variants: - -- **Original** — returns `EquivariantPolynomial` with dense operands -- **`_ir_dict`** — returns `IrDictPolynomial` with operands already split by irrep - -### EquivariantPolynomial descriptors - -```python -# Fully connected tensor product (all input-output irrep combinations) -e = cue.descriptors.fully_connected_tensor_product( - 16 * cue.Irreps("SO3", "0 + 1 + 2"), - 16 * cue.Irreps("SO3", "0 + 1 + 2"), - 16 * cue.Irreps("SO3", "0 + 1 + 2"), -) - -# Channelwise tensor product (same-channel only, sparse) -e = cue.descriptors.channelwise_tensor_product( - 64 * cue.Irreps("SO3", "0 + 1"), cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), simplify_irreps3=True, -) - -# Full (weightless) tensor product -e = cue.descriptors.full_tensor_product( - cue.Irreps("SO3", "2x0 + 1x1"), cue.Irreps("SO3", "0 + 1"), -) - -# Elementwise tensor product (paired channels) -e = cue.descriptors.elementwise_tensor_product( - cue.Irreps("SO3", "4x0 + 4x1"), cue.Irreps("SO3", "4x0 + 4x1"), -) - -# Linear equivariant map (weight x input) -e = cue.descriptors.linear( - cue.Irreps("SO3", "4x0 + 2x1"), - cue.Irreps("SO3", "3x0 + 5x1"), -) - -# Spherical harmonics -e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2, 3]) - -# Symmetric contraction (MACE-style) -e = cue.descriptors.symmetric_contraction( - 64 * cue.Irreps("SO3", "0 + 1 + 2"), - 64 * cue.Irreps("SO3", "0 + 1"), - (1, 2, 3), -) -``` - -### IrDictPolynomial descriptors - -Each `_ir_dict` variant returns an `IrDictPolynomial` whose polynomial is already split by irrep. The `input_irreps` and `output_irreps` tuples describe the operand groups. - -```python -# Channelwise tensor product -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 64 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -# desc.polynomial — SegmentedPolynomial, already split by irrep -# desc.input_irreps — (weight_irreps, irreps1, irreps2) -# desc.output_irreps — (irreps_out,) - -# Fully connected tensor product -desc = cue.descriptors.fully_connected_tensor_product_ir_dict(irreps1, irreps2, irreps3) - -# Full (weightless) tensor product -desc = cue.descriptors.full_tensor_product_ir_dict(irreps1, irreps2) - -# Elementwise tensor product -desc = cue.descriptors.elementwise_tensor_product_ir_dict(irreps1, irreps2) - -# Linear -desc = cue.descriptors.linear_ir_dict(irreps_in, irreps_out) - -# Spherical harmonics -desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2, 3]) - -# Symmetric contraction -desc = cue.descriptors.symmetric_contraction_ir_dict(irreps_in, irreps_out, (1, 2, 3)) -``` - -### IrDictPolynomial - -`IrDictPolynomial` pairs a `SegmentedPolynomial` (already split by irrep) with the `Irreps` that describe each operand group. - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) - -desc.polynomial # SegmentedPolynomial — each operand is one (mul, ir) block -desc.input_irreps # (weight_irreps, irreps1, irreps2) -desc.output_irreps # (irreps_out,) - -# Scale coefficients -scaled_poly = desc.polynomial * 0.5 - -# Access individual operand info -for i, op in enumerate(desc.polynomial.inputs): - print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") -``` - -Contract: for each `(mul, ir)` block in `input_irreps` / `output_irreps`, the corresponding polynomial operand has size `mul * ir.dim`. - -### split_polynomial_by_irreps - -The low-level function underlying `_ir_dict` descriptors. Splits one polynomial operand at irrep boundaries: - -```python -poly = e.polynomial # from an EquivariantPolynomial -poly = cue.split_polynomial_by_irreps(poly, 2, irreps_sh) # split input 2 -poly = cue.split_polynomial_by_irreps(poly, 1, irreps_in) # split input 1 -poly = cue.split_polynomial_by_irreps(poly, -1, irreps_out) # split output -``` - -### EquivariantPolynomial key methods - -```python -e.inputs # tuple of Rep (group representations for each input) -e.outputs # tuple of Rep -e.polynomial # the underlying SegmentedPolynomial - -# Numpy evaluation -[out] = e(weights, input1, input2) - -# Preparing for uniform_1d execution (see cuequivariance_jax SKILL.md) -e_ready = e.squeeze_modes().flatten_coefficient_modes() - -# Split an operand into per-irrep pieces (for ir_dict interface) -e_split = e.split_operand_by_irrep(1).split_operand_by_irrep(-1) - -# Scale all coefficients -e_scaled = e * 0.5 - -# Fuse compatible STPs -e_fused = e.fuse_stps() -``` - -### normalize_paths_for_operand - -Called internally by descriptors. Normalizes path coefficients so that a random input produces unit-variance output for the specified operand. Critical for numerical stability. - -## SegmentedPolynomial structure - -```python -poly = e.polynomial -poly.num_inputs # number of input operands -poly.num_outputs # number of output operands -poly.inputs # tuple of SegmentedOperand -poly.outputs # tuple of SegmentedOperand -poly.operations # tuple of (Operation, SegmentedTensorProduct) - -# Each operation maps buffers to STP operands -for op, stp in poly.operations: - print(op.buffers) # e.g., (0, 1, 2) means inputs[0], inputs[1] -> outputs[0] - print(stp.subscripts) -``` - -### SegmentedOperand - -```python -operand = poly.inputs[0] -operand.num_segments # how many segments -operand.segments # tuple of shape tuples, e.g., ((3, 4), (1, 2)) -operand.size # total flattened size (sum of products of segment shapes) -operand.ndim # number of dimensions per segment -operand.all_same_segment_shape() # True if all segments have identical shape -operand.segment_shape # the common shape (only if all_same_segment_shape) -``` - -## Custom equivariant polynomial from scratch - -```python -import numpy as np -import cuequivariance as cue - -# Build a fully-connected SO3(1)xSO3(1)->SO3(0) tensor product manually -cg = cue.clebsch_gordan(cue.SO3(1), cue.SO3(1), cue.SO3(0)) # shape (1, 3, 3, 1) - -d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") -d.add_segment(1, (3, 4)) # input1: 4x SO3(1), shape=(ir_dim, mul) -d.add_segment(2, (3, 4)) # input2: 4x SO3(1) -d.add_segment(3, (1, 16)) # output: 16x SO3(0) (4*4 fully connected) - -for c in cg: - d.add_path((4, 4, 16), 0, 0, 0, c=c) - -d = d.normalize_paths_for_operand(-1) - -poly = cue.SegmentedPolynomial.eval_last_operand(d) -ep = cue.EquivariantPolynomial( - [ - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1").new_scalars(d.operands[0].size), cue.ir_mul), - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), - cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), - ], - [cue.IrrepsAndLayout(cue.Irreps("SO3", "16x0"), cue.ir_mul)], - poly, -) - -# Numpy evaluation -w = np.random.randn(ep.inputs[0].dim) -x = np.random.randn(ep.inputs[1].dim) -y = np.random.randn(ep.inputs[2].dim) -[out] = ep(w, x, y) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `Irrep` base class | `cuequivariance/group_theory/representations/irrep.py` | -| `Rep` base class | `cuequivariance/group_theory/representations/rep.py` | -| `SO3` | `cuequivariance/group_theory/representations/irrep_so3.py` | -| `O3` | `cuequivariance/group_theory/representations/irrep_o3.py` | -| `SU2` | `cuequivariance/group_theory/representations/irrep_su2.py` | -| `Irreps` | `cuequivariance/group_theory/irreps_array/irreps.py` | -| `IrrepsLayout` | `cuequivariance/group_theory/irreps_array/irreps_layout.py` | -| `IrrepsAndLayout` | `cuequivariance/group_theory/irreps_array/irreps_and_layout.py` | -| `SegmentedTensorProduct` | `cuequivariance/segmented_polynomials/segmented_tensor_product.py` | -| `SegmentedPolynomial` | `cuequivariance/segmented_polynomials/segmented_polynomial.py` | -| `EquivariantPolynomial` | `cuequivariance/group_theory/equivariant_polynomial.py` | -| `IrDictPolynomial` | `cuequivariance/group_theory/ir_dict_polynomial.py` | -| Descriptors | `cuequivariance/group_theory/descriptors/` | -| Tensor product descriptors | `cuequivariance/group_theory/descriptors/irreps_tp.py` | -| `spherical_harmonics` | `cuequivariance/group_theory/descriptors/spherical_harmonics_.py` | -| `symmetric_contraction` | `cuequivariance/group_theory/descriptors/symmetric_contractions.py` | diff --git a/cuequivariance/cuequivariance/SKILL.md b/cuequivariance/cuequivariance/SKILL.md new file mode 120000 index 00000000..e412fd72 --- /dev/null +++ b/cuequivariance/cuequivariance/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance/SKILL.md \ No newline at end of file diff --git a/cuequivariance_jax/cuequivariance_jax/SKILL.md b/cuequivariance_jax/cuequivariance_jax/SKILL.md deleted file mode 100644 index 1da88a63..00000000 --- a/cuequivariance_jax/cuequivariance_jax/SKILL.md +++ /dev/null @@ -1,410 +0,0 @@ ---- -name: cuequivariance-jax -description: Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear). Use when writing JAX code with cuequivariance. ---- - -# cuequivariance_jax: Executing Equivariant Polynomials in JAX - -## Overview - -`cuequivariance_jax` (imported as `cuex`) executes `cuequivariance` polynomials on GPU via JAX. It provides: - -1. **Core primitive**: `cuex.segmented_polynomial()` — JAX primitive with full AD/vmap/JIT support -2. **Two data representations** (both built on `segmented_polynomial`): - - `cuex.equivariant_polynomial()` + `RepArray` — the original interface, a single contiguous array with representation metadata - - `cuex.ir_dict` module — `dict[Irrep, Array]` interface, uses `IrDictPolynomial` descriptors, works naturally with `jax.tree` -3. **NNX layers**: `cuex.nnx` module — Flax NNX `Module` wrappers using `dict[Irrep, Array]` - -## Execution methods - -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure JAX | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"indexed_linear"` | CUDA kernel | GPU, linear operations with `cuex.Repeats` indexing | - -## Core primitive: segmented_polynomial - -```python -import jax -import jax.numpy as jnp -import cuequivariance as cue -import cuequivariance_jax as cuex - -# Build a descriptor -e = cue.descriptors.channelwise_tensor_product( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -batch = 64 -w = jnp.ones((poly.inputs[0].size,)) # weights (shared across batch) -x = jax.random.normal(key, (batch, poly.inputs[1].size)) # batched input 1 -y = jax.random.normal(key, (batch, poly.inputs[2].size)) # batched input 2 - -# Execute with naive method -[out] = cuex.segmented_polynomial( - poly, - [w, x, y], # inputs - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], # output spec - method="naive", -) - -# Execute with uniform_1d (GPU, requires uniform segments) -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) -``` - -### Multiple batch axes with broadcasting - -Inputs can have any number of batch axes (everything before the last axis). Standard NumPy broadcasting applies: each batch axis is either size-1 or a common size. Inputs with fewer batch dimensions are implicitly prepended with size-1 axes: - -```python -# Fewer batch dims: weights with no batch axis broadcast across all -w = jnp.ones((poly.inputs[0].size,)) # 0 batch axes -> broadcasts -x = jnp.ones((5, 10, poly.inputs[1].size)) -y = jnp.ones((5, 10, poly.inputs[2].size)) - -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((5, 10, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) -``` - -### Indexing (gather/scatter) - -Index arrays provide gather (for inputs) and scatter (for outputs). One index per operand (inputs + outputs), `None` means no indexing: - -```python -i = jax.random.randint(key, (100, 50), 0, 10) # gather b along axis 0 -j1 = jax.random.randint(key, (100, 50), 0, 11) # scatter output axis 0 -j2 = jax.random.randint(key, (100, 1), 0, 12) # scatter output axis 1 - -[out] = cuex.segmented_polynomial( - poly, [a, b, c], - [jax.ShapeDtypeStruct((11, 12, poly.outputs[0].size), jnp.float32)], - indices=[None, np.s_[i, :], None, np.s_[j1, j2]], - method="uniform_1d", -) -``` - -### Gradients - -Fully differentiable — supports `jax.grad`, `jax.jacobian`, `jax.jvp`, `jax.vmap`: - -```python -def loss(w, x, y): - [out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="naive", - ) - return jnp.sum(out ** 2) - -grad_w = jax.grad(loss, 0)(w, x, y) -``` - -## ir_dict interface - -Uses `dict[Irrep, Array]` where each value has shape `(..., multiplicity, irrep_dim)`. This is the standard representation for NNX layers and works naturally with `jax.tree` operations. - -### Getting an ir_dict-ready polynomial - -Use `_ir_dict` descriptor variants, which return `IrDictPolynomial` with the polynomial already split by irrep: - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) - -poly = desc.polynomial # SegmentedPolynomial, already split by irrep -weight_irreps, irreps1, irreps2 = desc.input_irreps -(irreps_out,) = desc.output_irreps # tuple unpacking to get the single output group -``` - -Each polynomial operand corresponds to exactly one `(mul, ir)` block. The `input_irreps` and `output_irreps` tuples describe how operands group into logical operand groups (weights, node features, spherical harmonics, output). - -### Executing with segmented_polynomial_uniform_1d - -```python -from einops import rearrange - -num_edges, num_nodes = 100, 30 - -# Weights: reshape to (batch, num_segments, segment_size) -w_flat = jax.random.normal(key, (num_edges, poly.inputs[0].size)) -w = rearrange(w_flat, "e (s m) -> e s m", s=poly.inputs[0].num_segments) - -# Node features: dict[Irrep, Array] reshaped to (nodes, ir.dim, mul) for ir_mul layout -node_feats = { - cue.SO3(0): jnp.ones((num_nodes, 32, 1)), # 32x scalar - cue.SO3(1): jnp.ones((num_nodes, 32, 3)), # 32x vector -} -x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - -# Spherical harmonics: (edges, ir.dim) — no multiplicity dimension -sph = { - cue.SO3(0): jnp.ones((num_edges, 1)), - cue.SO3(1): jnp.ones((num_edges, 3)), -} - -# Build output template -senders = jax.random.randint(key, (num_edges,), 0, num_nodes) -receivers = jax.random.randint(key, (num_edges,), 0, num_nodes) -out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(irreps_out, poly.outputs) -} - -# Execute with gather (senders) and scatter (receivers) -y = cuex.ir_dict.segmented_polynomial_uniform_1d( - poly, - [w, x1, sph], - out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", -) -# y is dict[Irrep, Array] with accumulated results at receiver nodes -``` - -### ir_dict utility functions - -```python -# Validate dict matches irreps -cuex.ir_dict.assert_mul_ir_dict(irreps, x) # asserts shape (..., mul, ir.dim) - -# Convert flat array <-> dict -d = cuex.ir_dict.flat_to_dict(irreps, flat_array) # layout="mul_ir" default -d = cuex.ir_dict.flat_to_dict(irreps, flat_array, layout="ir_mul") -flat = cuex.ir_dict.dict_to_flat(irreps, d) - -# Arithmetic -z = cuex.ir_dict.irreps_add(x, y) -z = cuex.ir_dict.irreps_zeros_like(x) - -# Create template dict -template = cuex.ir_dict.mul_ir_dict(irreps, jax.ShapeDtypeStruct(shape, dtype)) -``` - -## RepArray interface: equivariant_polynomial - -The original interface. Wraps `segmented_polynomial` with `RepArray` — a single contiguous array with representation metadata: - -```python -e = cue.descriptors.fully_connected_tensor_product( - 4 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - 4 * cue.Irreps("SO3", "0 + 1"), -) - -inputs = [ - cuex.randn(jax.random.key(i), rep, (batch,), jnp.float32) - for i, rep in enumerate(e.inputs) -] - -# Returns a RepArray with representation metadata -out = cuex.equivariant_polynomial(e, inputs, method="naive") -out.array # the raw jax.Array -out.reps # dict mapping axes to Rep objects -``` - -## NNX layers - -### IrrepsLinear - -Equivariant linear layer using `dict[Irrep, Array]`: - -```python -from flax import nnx - -linear = cuex.nnx.IrrepsLinear( - irreps_in=cue.Irreps(cue.SO3, "4x0 + 2x1").regroup(), # must be regrouped - irreps_out=cue.Irreps(cue.SO3, "3x0 + 5x1").regroup(), - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) - -# Input/output: dict[Irrep, Array] with shape (batch, mul, ir.dim) -x = { - cue.SO3(0): jnp.ones((batch, 4, 1)), - cue.SO3(1): jnp.ones((batch, 2, 3)), -} -y = linear(x) -# y[cue.SO3(0)].shape == (batch, 3, 1) -# y[cue.SO3(1)].shape == (batch, 5, 3) -``` - -Implementation uses `jnp.einsum("uv,...ui->...vi", w, x[ir])` per irrep with `1/sqrt(mul_in)` normalization. - -### SphericalHarmonics - -Uses `spherical_harmonics_ir_dict` internally for the `dict[Irrep, Array]` output: - -```python -sh = cuex.nnx.SphericalHarmonics(max_degree=3, eps=0.0) - -vectors = jax.random.normal(key, (batch, 3)) # 3D vectors -y = sh(vectors) -# y[cue.O3(0, 1)].shape == (batch, 1, 1) # L=0 -# y[cue.O3(1, -1)].shape == (batch, 1, 3) # L=1 -# y[cue.O3(2, 1)].shape == (batch, 1, 5) # L=2 -# y[cue.O3(3, -1)].shape == (batch, 1, 7) # L=3 -``` - -### IrrepsNormalize - -```python -norm = cuex.nnx.IrrepsNormalize(eps=1e-6, scale=1.0, skip_scalars=True) -y = norm(x) # normalizes non-scalar irreps by RMS over ir.dim, averaged over mul -``` - -### MLP (scalar only) - -```python -mlp = cuex.nnx.MLP( - layer_sizes=[64, 128, 64], - activation=jax.nn.silu, - output_activation=False, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) -y = mlp(x_scalar) # standard dense MLP with 1/sqrt(fan_in) normalization -``` - -### IrrepsIndexedLinear - -For species-indexed linear layers (different weights per atom type): - -```python -indexed_linear = cuex.nnx.IrrepsIndexedLinear( - irreps_in=cue.Irreps(cue.O3, "8x0e").regroup(), - irreps_out=cue.Irreps(cue.O3, "16x0e").regroup(), - num_indices=50, # number of species - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) - -# num_index_counts: how many atoms of each species -species_counts = jnp.array([3, 4, 3, ...]) # sum = batch_size -y = indexed_linear(x, species_counts) -``` - -Uses `method="indexed_linear"` internally with `cuex.Repeats`. - -## Preparing polynomials for uniform_1d - -The `uniform_1d` CUDA kernel requires: -1. All segments within each operand have **the same shape** -2. A **single mode** in the subscripts (after preprocessing) - -### From EquivariantPolynomial to uniform_1d-ready - -For `equivariant_polynomial()` (RepArray interface): - -```python -e = cue.descriptors.channelwise_tensor_product(...) -e = e.squeeze_modes().flatten_coefficient_modes() -out = cuex.equivariant_polynomial(e, inputs, method="uniform_1d") -``` - -For `ir_dict` (dict[Irrep, Array] interface), use `_ir_dict` descriptors directly: - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out -) -poly = desc.polynomial -# Each operand has a single irrep type -> maps naturally to dict[Irrep, Array] -``` - -### Why splitting by irrep matters - -Without splitting, a dense operand like `32x0+32x1` requires all irreps packed into a single contiguous buffer. After splitting, each irrep gets its own separate buffer passed to the CUDA kernel via FFI. The buffers no longer need to be contiguous with each other. - -This is especially useful when the polynomial is preceded or followed by per-irrep linear layers (like `IrrepsLinear`). With split operands, no transpose or copy is needed between the linear layers and the polynomial — the `dict[Irrep, Array]` flows directly through the pipeline. - -## Complete GNN message-passing example - -This pattern is used in NequIP, MACE, and similar equivariant GNN models: - -```python -class MessagePassing(nnx.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out, epsilon, *, name, dtype, rngs): - self.name = name - desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out - ) - (self.irreps_out,) = desc.output_irreps - self.poly = desc.polynomial * epsilon - self.weight_numel = self.poly.inputs[0].size - - def __call__(self, weights, node_feats, sph, senders, receivers, num_nodes): - # weights: (num_edges, weight_numel) - w = rearrange(weights, "e (s m) -> e s m", s=self.poly.inputs[0].num_segments) - # node_feats: dict[Irrep, Array] with (nodes, mul, ir.dim) - x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - # sph: dict[Irrep, Array] with (edges, 1, ir.dim) or (edges, ir.dim) - x2 = jax.tree.map(lambda v: rearrange(v, "e 1 i -> e i"), sph) - - out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(self.irreps_out, self.poly.outputs) - } - - y = cuex.ir_dict.segmented_polynomial_uniform_1d( - self.poly, [w, x1, x2], out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", - ) - # Rearrange output back to (nodes, mul, ir.dim) for downstream layers - return { - ir: rearrange(v, "n (i s) m -> n (s m) i", i=ir.dim) - for ir, v in y.items() - } -``` - -## RepArray - -Representation-aware JAX array: - -```python -rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) -x = cuex.RepArray(rep, jnp.ones((batch, rep.dim))) -x = cuex.randn(jax.random.key(0), rep, (batch,), jnp.float32) - -x.array # raw jax.Array -x.reps # {axis: Rep} -x.irreps # Irreps (if last axis is IrrepsAndLayout) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `segmented_polynomial` primitive | `cuequivariance_jax/segmented_polynomials/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_naive.py` | -| `indexed_linear` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_indexed_linear.py` | -| `equivariant_polynomial` | `cuequivariance_jax/equivariant_polynomial.py` | -| `ir_dict` module | `cuequivariance_jax/ir_dict.py` | -| `nnx` module | `cuequivariance_jax/nnx.py` | -| `RepArray` | `cuequivariance_jax/rep_array/rep_array_.py` | -| `Repeats` / utilities | `cuequivariance_jax/segmented_polynomials/utils.py` | -| NequIP example | `cuequivariance_jax/examples/nequip_nnx.py` | -| MACE example | `cuequivariance_jax/examples/mace_nnx.py` | diff --git a/cuequivariance_jax/cuequivariance_jax/SKILL.md b/cuequivariance_jax/cuequivariance_jax/SKILL.md new file mode 120000 index 00000000..c1736cef --- /dev/null +++ b/cuequivariance_jax/cuequivariance_jax/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance-jax/SKILL.md \ No newline at end of file diff --git a/cuequivariance_torch/cuequivariance_torch/SKILL.md b/cuequivariance_torch/cuequivariance_torch/SKILL.md deleted file mode 100644 index 1f547491..00000000 --- a/cuequivariance_torch/cuequivariance_torch/SKILL.md +++ /dev/null @@ -1,400 +0,0 @@ ---- -name: cuequivariance-torch -description: Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv). Use when writing PyTorch code with cuequivariance. ---- - -# cuequivariance_torch: Executing Equivariant Polynomials in PyTorch - -## Overview - -`cuequivariance_torch` (imported as `cuet`) executes `cuequivariance` polynomials on GPU via PyTorch. It provides: - -1. **Core primitive**: `cuet.SegmentedPolynomial` — `torch.nn.Module` with multiple CUDA backends -2. **High-level operations** (`torch.nn.Module`): `ChannelWiseTensorProduct`, `FullyConnectedTensorProduct`, `Linear`, `SymmetricContraction`, `SphericalHarmonics`, `Rotation`, `Inversion` -3. **Layers**: `cuet.layers.BatchNorm`, `cuet.layers.FullyConnectedTensorProductConv` (message passing) -4. **Utilities**: `triangle_attention`, `triangle_multiplicative_update`, `attention_pair_bias` (AlphaFold3-style) -5. **Export support**: `onnx_custom_translation_table()`, `register_tensorrt_plugins()` - -## Execution methods - -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure PyTorch (einsum) | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"fused_tp"` | CUDA kernel | GPU, 3- or 4-operand contractions, float32/float64 | -| `"indexed_linear"` | CUDA kernel | GPU, linear with indexed weights, sorted indices | - -## Core primitive: SegmentedPolynomial - -```python -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet - -# Build a descriptor -e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) -poly = e.polynomial - -# Create the module -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -# Forward pass -x = torch.randn(batch, 3, device="cuda") -[output] = sp([x]) -# output.shape == (batch, 9) -- 1 + 3 + 5 -``` - -### Inputs, indexing, and scatter - -```python -e = cue.descriptors.channelwise_tensor_product( - 16 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -w = torch.randn(1, poly.inputs[0].size, device="cuda") # shared weights -x1 = torch.randn(batch, poly.inputs[1].size, device="cuda") # batched input 1 -x2 = torch.randn(batch, poly.inputs[2].size, device="cuda") # batched input 2 - -# Basic forward -[out] = sp([w, x1, x2]) - -# With input gathering (e.g., gather x1 by node index) -senders = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp([w, x1, x2], input_indices={1: senders}) - -# With output scattering (accumulate into target nodes) -receivers = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp( - [w, x1, x2], - input_indices={1: senders}, - output_indices={0: receivers}, - output_shapes={0: torch.empty(num_nodes, 1, device="cuda")}, -) -``` - -### Math dtype control - -```python -# Compute in float32 regardless of input dtype -sp = cuet.SegmentedPolynomial(poly, method="fused_tp", math_dtype=torch.float32) - -# For fused_tp, math_dtype must be float32 or float64 -# For naive, any torch.dtype works -# For uniform_1d, float32 or float64 (auto-selects float32 if input is e.g. float16) -``` - -## High-level operations - -All operations are `torch.nn.Module` subclasses. They wrap `SegmentedPolynomial` and handle layout transposition automatically. - -### Memory layout - -`IrrepsLayout` controls memory order within each `(mul, ir)` block: - -- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **default, compatible with e3nn** -- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used internally by descriptors** - -Operations accept `layout` (applies to all), or per-operand `layout_in1`, `layout_in2`, `layout_out`. - -### ChannelWiseTensorProduct - -Channel-wise tensor product: pairs channels of `x1` with channels of `x2`. - -```python -# With internal weights (default: shared_weights=True, internal_weights=True) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), # irreps_in1 - cue.Irreps("SO3", "0 + 1"), # irreps_in2 - layout=cue.mul_ir, - device="cuda", - dtype=torch.float32, -) -# tp.weight_numel -- number of weight parameters -# tp.irreps_out -- output irreps (auto-computed) - -x1 = torch.randn(batch, tp.irreps_in1.dim, device="cuda") -x2 = torch.randn(batch, tp.irreps_in2.dim, device="cuda") - -out = tp(x1, x2) # uses internal weight parameter -# out.shape == (batch, tp.irreps_out.dim) - -# With external weights (shared_weights=False) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), - cue.Irreps("SO3", "0 + 1"), - layout=cue.mul_ir, - shared_weights=False, - device="cuda", -) -w = torch.randn(batch, tp.weight_numel, device="cuda") -out = tp(x1, x2, weight=w) - -# With gather/scatter for graph neural networks -out = tp(x1, x2, weight=w, indices_1=senders, indices_out=receivers, size_out=num_nodes) -``` - -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. - -### FullyConnectedTensorProduct - -All input irrep pairs contribute to all output irreps (dense contraction). - -```python -tp = cuet.FullyConnectedTensorProduct( - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_in1 - cue.Irreps("O3", "0e + 1o"), # irreps_in2 - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) - -out = tp(x1, x2) # uses internal weights -# or: out = tp(x1, x2, weight=w) # external weights -``` - -Default method: `"fused_tp"`. - -### Linear - -Equivariant linear layer (weight-only, no second input). - -```python -linear = cuet.Linear( - cue.Irreps("SO3", "4x0 + 2x1"), # irreps_in - cue.Irreps("SO3", "3x0 + 5x1"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) - -out = linear(x) - -# Species-indexed weights (different weights per atom type) -linear = cuet.Linear( - irreps_in, irreps_out, - weight_classes=50, # 50 different weight sets - internal_weights=True, - device="cuda", -) -out = linear(x, weight_indices=species_indices) # species_indices: (batch,) int tensor -``` - -Default method: `"naive"`. Use `method="fused_tp"` for CUDA acceleration. - -### SymmetricContraction - -MACE-style symmetric contraction with element-indexed weights. - -```python -sc = cuet.SymmetricContraction( - cue.Irreps("O3", "32x0e + 32x1o"), # irreps_in (uniform mul required) - cue.Irreps("O3", "32x0e"), # irreps_out (uniform mul required) - contraction_degree=3, - num_elements=95, # number of chemical elements - layout=cue.ir_mul, - dtype=torch.float32, - device="cuda", -) - -# indices: (batch,) int tensor selecting which element weights to use -out = sc(x, indices) -# out.shape == (batch, irreps_out.dim) -``` - -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. - -### SphericalHarmonics - -```python -sh = cuet.SphericalHarmonics( - ls=[0, 1, 2, 3], - normalize=True, - device="cuda", -) - -vectors = torch.randn(batch, 3, device="cuda") -out = sh(vectors) -# out.shape == (batch, 1 + 3 + 5 + 7) -- sum of 2l+1 -``` - -Default method: `"uniform_1d"`. - -### Rotation and Inversion - -```python -# Rotation (SO3 or O3 irreps) -rot = cuet.Rotation( - cue.Irreps("SO3", "4x0 + 2x1 + 1x2"), - layout=cue.ir_mul, - device="cuda", -) - -# Euler angles (YXY convention) -gamma = torch.tensor([0.1], device="cuda") -beta = torch.tensor([0.2], device="cuda") -alpha = torch.tensor([0.3], device="cuda") -out = rot(gamma, beta, alpha, x) - -# Helper: encode angle for rotation -encoded = cuet.encode_rotation_angle(angle, ell=3) # cos/sin encoding - -# Helper: 3D vector to Euler angles -beta, alpha = cuet.vector_to_euler_angles(vector) - -# Inversion (O3 irreps only) -inv = cuet.Inversion( - cue.Irreps("O3", "4x0e + 2x1o"), - layout=cue.ir_mul, - device="cuda", -) -out = inv(x) -``` - -## Layers - -### BatchNorm - -Batch normalization for equivariant representations (adapted from e3nn). - -```python -bn = cuet.layers.BatchNorm( - cue.Irreps("O3", "4x0e + 4x1o"), - layout=cue.mul_ir, - eps=1e-5, - momentum=0.1, - affine=True, -) - -out = bn(x) # x.shape == (batch, irreps.dim) -``` - -### FullyConnectedTensorProductConv - -Message passing layer for equivariant GNNs (DiffDock-style). - -```python -conv = cuet.layers.FullyConnectedTensorProductConv( - in_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - sh_irreps=cue.Irreps("O3", "0e + 1o"), - out_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - mlp_channels=[16, 32, 32], - mlp_activation=torch.nn.ReLU(), - batch_norm=True, - layout=cue.ir_mul, -) - -# graph = ((src, dst), (num_src_nodes, num_dst_nodes)) -graph = ((src, dst), (num_src_nodes, num_dst_nodes)) - -out = conv(src_features, edge_sh, edge_emb, graph, reduce="mean") -# out.shape == (num_dst_nodes, out_irreps.dim) - -# Optional: separate scalar features for efficient first-layer GEMM -out = conv(src_features, edge_sh, edge_emb, graph, - src_scalars=src_scalars, dst_scalars=dst_scalars) -``` - -## Triangle operations (AlphaFold2-style) - -Require `cuequivariance_ops_torch`. - -```python -# Triangle attention with pair bias -kv_lengths = cuet.mask_to_kv_lengths(prefix_mask) -out = cuet.triangle_attention(q, k, v, bias, scale=scale, kv_lengths=kv_lengths) -# Arbitrary dense masks are supported through the fallback path: -out = cuet.triangle_attention(q, k, v, bias, mask=holey_mask, scale=scale) -# q, k, v: (B, N, H, Q/K, D), bias: (B, 1, H, Q, K) - -# Triangle multiplicative update -out = cuet.triangle_multiplicative_update( - x, # (B, I, J, C) - mask=mask, # (B, I, J) - precision=cuet.TriMulPrecision.DEFAULT, -) - -# Attention with pair bias (strict OpenFold3/Boltz contract) -out, _ = cuet.attention_pair_bias( - single_repr, pair_repr, mask, num_heads, - w_ln_a, b_ln_a, - w_proj_q, b_proj_q, w_proj_k, w_proj_v, - w_proj_g, w_proj_o, w_proj_z, - w_ln_z=w_ln_z, b_ln_z=b_ln_z, -) -# Optional generalized projection params are keyword-only: -# b_proj_k, b_proj_v, w_ln_q, b_ln_q, w_ln_k, b_ln_k, b_proj_g (default None), -# and norm_kind (default "layer_norm"). -``` - -## ONNX and TensorRT export - -```python -# ONNX export -table = cuet.onnx_custom_translation_table() -onnx_program = torch.onnx.export(model, inputs, custom_translation_table=table) - -# TensorRT plugin registration -cuet.register_tensorrt_plugins() -``` - -## Complete GNN example - -```python -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet - -class SimpleGNN(torch.nn.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out): - super().__init__() - self.tp = cuet.ChannelWiseTensorProduct( - irreps_in, irreps_sh, layout=cue.mul_ir, - shared_weights=False, device="cuda", - ) - self.linear = cuet.Linear( - self.tp.irreps_out, irreps_out, - layout=cue.mul_ir, internal_weights=True, device="cuda", - ) - self.sh = cuet.SphericalHarmonics( - ls=[ir.l for _, ir in irreps_sh], normalize=True, device="cuda", - ) - - def forward(self, node_feats, edge_vec, edge_index, num_nodes): - src, dst = edge_index - edge_sh = self.sh(edge_vec) - w = torch.randn(1, self.tp.weight_numel, device=node_feats.device) - - # Message: tensor product on edges with scatter to destination nodes - messages = self.tp( - node_feats, edge_sh, weight=w, - indices_1=src, indices_2=None, - indices_out=dst, size_out=num_nodes, - ) - return self.linear(messages) -``` - -## Key file locations - -| Component | Path | -|-----------|------| -| `SegmentedPolynomial` | `cuequivariance_torch/primitives/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_torch/primitives/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_torch/primitives/segmented_polynomial_naive.py` | -| `fused_tp` backend | `cuequivariance_torch/primitives/segmented_polynomial_fused_tp.py` | -| `indexed_linear` backend | `cuequivariance_torch/primitives/segmented_polynomial_indexed_linear.py` | -| `ChannelWiseTensorProduct` | `cuequivariance_torch/operations/tp_channel_wise.py` | -| `FullyConnectedTensorProduct` | `cuequivariance_torch/operations/tp_fully_connected.py` | -| `Linear` | `cuequivariance_torch/operations/linear.py` | -| `SymmetricContraction` | `cuequivariance_torch/operations/symmetric_contraction.py` | -| `SphericalHarmonics` | `cuequivariance_torch/operations/spherical_harmonics.py` | -| `Rotation` / `Inversion` | `cuequivariance_torch/operations/rotation.py` | -| `BatchNorm` | `cuequivariance_torch/layers/batchnorm.py` | -| `FullyConnectedTensorProductConv` | `cuequivariance_torch/layers/tp_conv_fully_connected.py` | -| Triangle operations | `cuequivariance_torch/primitives/triangle.py` | -| Layout transposition | `cuequivariance_torch/primitives/transpose.py` | diff --git a/cuequivariance_torch/cuequivariance_torch/SKILL.md b/cuequivariance_torch/cuequivariance_torch/SKILL.md new file mode 120000 index 00000000..e2451973 --- /dev/null +++ b/cuequivariance_torch/cuequivariance_torch/SKILL.md @@ -0,0 +1 @@ +../../skills/cuequivariance-torch/SKILL.md \ No newline at end of file diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 00000000..382486b1 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,128 @@ +# Agent skill maintenance + +Each immediate child directory containing a `SKILL.md` is one independent skill. +NVSkills CI and nv-carps discover these paths: + +```text +skills/ + cuequivariance/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt + cuequivariance-jax/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt + cuequivariance-torch/ + SKILL.md + evals/{evals.json,config.yml} + scripts/requirements.txt +``` + +Edit the regular files here. The Python package resources at +`//SKILL.md` are relative symlinks back to these files, keeping +`python -m skill` compatible without maintaining duplicate instructions. +Do not replace the canonical files with symlinks, or nest one skill inside another. +Each skill can be distributed independently; its source-location tables explain +how to find code in a checkout or an installed package. + +## Evaluation coverage + +Each dataset contains five positive cases and one unrelated negative case. +The core cases were adapted from the BioNeMo agent toolkit's cuEquivariance +evaluation ideas; they use this repository's APIs and contain no references to +BioNeMo-only skill resources. The JAX and PyTorch cases exercise their respective +frontends rather than reusing the core skill's expected routing. + +| Skill | Positive cases | +| --- | --- | +| Core | Complex Z3 irreps and CG covariance; full SO3 tensor product; split/dense channelwise agreement; layout conversion and rotation; older-API fallback without installing packages | +| JAX | Broadcasting, JIT and finite-difference gradients; dictionary layout conversion; NNX linear normalization and rotation; split polynomial execution; CPU spherical harmonics and backend choice | +| PyTorch | Spherical harmonics and gradcheck; linear layouts and rotation; external per-example weights; gather/scatter with repeated receivers and gradients; CPU backend fallback | + +`evals/config.yml` requests an isolated skill workspace and a rebuilt Harbor +environment. SkillEvaluator installs `scripts/requirements.txt` when generating +that environment. Those requirements pin cuEquivariance to the immutable source +revision whose APIs were checked, and pin the JAX/Flax or CPU PyTorch versions. +They do not require sibling skills or a repository checkout inside the sandbox. +When updating the API baseline, update the pins and run the numerical checks +before changing the assertions. The CUDA backend constraints are checked as +guidance; these datasets do not benchmark GPU performance or validate CUDA kernels. + +The tests under `.github/tests/test_*_skill_evals.py` verify the mathematical and +API expectations behind the prompts. They are maintenance checks, not live agent +scores. To run them against a development checkout with the relevant frontend +dependencies installed: + +```bash +pytest -q .github/tests/test_core_skill_evals.py +JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py +pytest -q .github/tests/test_torch_skill_evals.py +``` + +To verify the pinned evaluation environment in a separate Python 3.12 environment: + +```bash +python -m pip install -r skills/cuequivariance/scripts/requirements.txt \ + -r skills/cuequivariance-jax/scripts/requirements.txt \ + -r skills/cuequivariance-torch/scripts/requirements.txt pytest +JAX_PLATFORMS=cpu pytest -q .github/tests +``` + +The request workflow delegates live evaluation to NVSkills CI. With the same +SkillEvaluator version as the service (1.5.6 when this layout was introduced), +the dataset/config preflight can also be run locally: + +```bash +skillevaluator tier3 validate skills/cuequivariance +skillevaluator tier3 validate skills/cuequivariance-jax +skillevaluator tier3 validate skills/cuequivariance-torch +``` + +## Packaging + +The release workflow materializes symlinks **before** running Hatchling, including +the existing LICENSE, README, VERSION and attribution links. Keep this step: +building an sdist directly from the linked development tree can leave links whose +targets are outside that distribution. Release wheels and sdists must carry a +regular `SKILL.md` resource. + +The ordinary CPU test workflow runs the packaging regression check for each +package. To check all three locally with their runtime dependencies and Hatchling +installed: + +```bash +python .github/scripts/check_skill_packaging.py +``` + +The script builds from a temporary materialized copy, checks the source CLI, +inspects wheel and sdist resources, rebuilds a wheel from each sdist, and verifies +the CLI imports and prints the resource from each wheel. It leaves the checkout's +symlinks intact. Use `--package cuequivariance_jax` (or either other Python package +name) to check one package. + +## Service onboarding and publication + +The source workflow is `.github/workflows/request-nvskills-ci.yml`, using the +public reusable workflow in `NVIDIA/skills`. Activation also requires: + +1. An enabled `NVIDIA/cuEquivariance` entry in + `NVIDIA/nvskills-ci/config/onboarded-repositories.json`, with the `external` + profile and `main` as an allowed PR base. +2. The NVSkills GitHub App installed with access to this repository, and the + repository secret `NVSKILLS_CI_DISPATCH_TOKEN` provisioned by the CI owners. + Protect the request workflow with the repository's normal maintainer review. +3. The request workflow on the default branch so `issue_comment` events run. + A maintainer can then comment `/nvskills-ci` on a same-repository PR that + changes these skills. Fork PRs require a separate central policy opt-in. +4. The nv-carps standalone-skill catalog fix: Tier 2 should use the `skills/` + catalog rather than traversing unrelated Python package symlinks. Team/plugin + catalogs keep their existing handling. Cross-tree duplicate comparison is + outside this standalone catalog's scope. +5. Successful service validation and genuine generated signatures, skill cards, + and benchmarks. Only then merge the three-path registration in + `NVIDIA/skills/components.d/cuequivariance.yml` and let catalog sync publish it. + +The component registry must point at each of the three individual skill +directories, with matching catalog directory names. Do not manufacture generated +validation artifacts to satisfy catalog onboarding checks. diff --git a/skills/cuequivariance-jax/SKILL.md b/skills/cuequivariance-jax/SKILL.md new file mode 100644 index 00000000..a5ebaa88 --- /dev/null +++ b/skills/cuequivariance-jax/SKILL.md @@ -0,0 +1,435 @@ +--- +name: cuequivariance-jax +description: Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear). Use when writing JAX code with cuequivariance. +license: Apache-2.0 +metadata: + author: Mario Geiger +--- + +# cuequivariance_jax: Executing Equivariant Polynomials in JAX + +## Overview + +`cuequivariance_jax` (imported as `cuex`) executes `cuequivariance` polynomials on CPU or GPU via JAX. It provides: + +1. **Core primitive**: `cuex.segmented_polynomial()` — JAX primitive with full AD/vmap/JIT support +2. **Two data representations** (both built on `segmented_polynomial`): + - `cuex.equivariant_polynomial()` + `RepArray` — the original interface, a single contiguous array with representation metadata + - `cuex.ir_dict` module — `dict[Irrep, Array]` interface, uses `IrDictPolynomial` descriptors, works naturally with `jax.tree` +3. **NNX layers**: `cuex.nnx` module — Flax NNX `Module` wrappers using `dict[Irrep, Array]` + +## Instructions + +1. Check `jax.devices()`, installed versions, and whether Flax NNX is needed. Install `cuequivariance-jax` with JAX; add Flax for NNX layers. Follow the installed release's compatibility requirements. +2. Start with `method="naive"` for a portable CPU reference. Select a CUDA method only when a compatible GPU, kernel package, dtype, and descriptor are available; do not assume default methods run on CPU. +3. Choose dense `RepArray` or `ir_dict` explicitly. High-level irrep dictionaries store `(..., multiplicity, ir.dim)` blocks; low-level CUDA segmented arrays have a different contract. Match the dense layout in both conversion directions. +4. Derive operand sizes and output specs from the descriptor. Split PRNG keys, check broadcasting, compare to the core NumPy reference, and verify `jax.jit` and gradients before optimizing. +5. For custom group definitions and CG coefficients, use the `cuequivariance` skill. For PyTorch modules, use `cuequivariance-torch`. + +Source paths below are relative to the `cuequivariance_jax/` project inside a source checkout. For a separately installed skill, locate code with `cuex.__file__` or https://github.com/NVIDIA/cuEquivariance. + +## Examples + +Examples build on the imports and descriptors shown earlier. CUDA examples are marked by their method or device; use the naive method for CPU checks. + +## Execution methods + +| Method | Backend | Requirements | +|--------|---------|-------------| +| `"naive"` | Pure JAX | Always works, any platform | +| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | +| `"indexed_linear"` | CUDA kernel | GPU, linear operations with `cuex.Repeats` indexing | + +## Core primitive: segmented_polynomial + +```python +import jax +import jax.numpy as jnp +import numpy as np +import cuequivariance as cue +import cuequivariance_jax as cuex + +# Build a descriptor +e = cue.descriptors.channelwise_tensor_product( + 32 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) +poly = e.polynomial + +batch = 64 +key_x, key_y = jax.random.split(jax.random.key(0)) +key = jax.random.key(1) # independent key for subsequent indexing examples +w = jnp.ones((poly.inputs[0].size,)) # weights (shared across batch) +x = jax.random.normal(key_x, (batch, poly.inputs[1].size)) # batched input 1 +y = jax.random.normal(key_y, (batch, poly.inputs[2].size)) # batched input 2 + +# Execute with naive method +[out] = cuex.segmented_polynomial( + poly, + [w, x, y], # inputs + [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], # output spec + method="naive", +) + +# Execute with uniform_1d (GPU, requires uniform segments) +[out] = cuex.segmented_polynomial( + poly, [w, x, y], + [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], + method="uniform_1d", +) +``` + +### Multiple batch axes with broadcasting + +Inputs can have any number of batch axes (everything before the last axis). Standard NumPy broadcasting applies: each batch axis is either size-1 or a common size. Inputs with fewer batch dimensions are implicitly prepended with size-1 axes: + +```python +# Fewer batch dims: weights with no batch axis broadcast across all +w = jnp.ones((poly.inputs[0].size,)) # 0 batch axes -> broadcasts +x = jnp.ones((5, 10, poly.inputs[1].size)) +y = jnp.ones((5, 10, poly.inputs[2].size)) + +[out] = cuex.segmented_polynomial( + poly, [w, x, y], + [jax.ShapeDtypeStruct((5, 10, poly.outputs[0].size), jnp.float32)], + method="uniform_1d", +) +``` + +### Indexing (gather/scatter) + +Index arrays provide gather (for inputs) and scatter (for outputs). One index per operand (inputs + outputs), `None` means no indexing: + +```python +i = jax.random.randint(key, (100, 50), 0, 10) # gather b along axis 0 +j1 = jax.random.randint(key, (100, 50), 0, 11) # scatter output axis 0 +j2 = jax.random.randint(key, (100, 1), 0, 12) # scatter output axis 1 + +[out] = cuex.segmented_polynomial( + poly, [a, b, c], + [jax.ShapeDtypeStruct((11, 12, poly.outputs[0].size), jnp.float32)], + indices=[None, np.s_[i, :], None, np.s_[j1, j2]], + method="uniform_1d", +) +``` + +### Gradients + +Fully differentiable — supports `jax.grad`, `jax.jacobian`, `jax.jvp`, `jax.vmap`: + +```python +def loss(w, x, y): + [out] = cuex.segmented_polynomial( + poly, [w, x, y], + [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], + method="naive", + ) + return jnp.sum(out ** 2) + +grad_w = jax.grad(loss, 0)(w, x, y) +``` + +## ir_dict interface + +Uses `dict[Irrep, Array]` where each value has shape `(..., multiplicity, irrep_dim)`. This is the standard representation for NNX layers and works naturally with `jax.tree` operations. + +### Getting an ir_dict-ready polynomial + +Use `_ir_dict` descriptor variants, which return `IrDictPolynomial` with the polynomial already split by irrep: + +```python +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + 32 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) + +poly = desc.polynomial # SegmentedPolynomial, already split by irrep +weight_irreps, irreps1, irreps2 = desc.input_irreps +(irreps_out,) = desc.output_irreps # tuple unpacking to get the single output group +``` + +Each polynomial operand corresponds to exactly one `(mul, ir)` block. The `input_irreps` and `output_irreps` tuples describe how operands group into logical operand groups (weights, node features, spherical harmonics, output). + +### Executing with segmented_polynomial_uniform_1d + +```python +from einops import rearrange + +num_edges, num_nodes = 100, 30 + +# Weights: reshape to (batch, num_segments, segment_size) +w_flat = jax.random.normal(key, (num_edges, poly.inputs[0].size)) +w = rearrange(w_flat, "e (s m) -> e s m", s=poly.inputs[0].num_segments) + +# Node features: dict[Irrep, Array] reshaped to (nodes, ir.dim, mul) for ir_mul layout +node_feats = { + cue.SO3(0): jnp.ones((num_nodes, 32, 1)), # 32x scalar + cue.SO3(1): jnp.ones((num_nodes, 32, 3)), # 32x vector +} +x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) + +# Spherical harmonics: (edges, ir.dim) — no multiplicity dimension +sph = { + cue.SO3(0): jnp.ones((num_edges, 1)), + cue.SO3(1): jnp.ones((num_edges, 3)), +} + +# Build output template +senders = jax.random.randint(key, (num_edges,), 0, num_nodes) +receivers = jax.random.randint(key, (num_edges,), 0, num_nodes) +out_template = { + ir: jax.ShapeDtypeStruct( + (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype + ) + for (_, ir), desc in zip(irreps_out, poly.outputs) +} + +# Execute with gather (senders) and scatter (receivers) +y = cuex.ir_dict.segmented_polynomial_uniform_1d( + poly, + [w, x1, sph], + out_template, + input_indices=[None, senders, None], + output_indices=receivers, + name="tensor_product", +) +# y is dict[Irrep, Array] with accumulated results at receiver nodes +``` + +### ir_dict utility functions + +```python +# Validate dict matches irreps +cuex.ir_dict.assert_mul_ir_dict(irreps, x) # asserts shape (..., mul, ir.dim) + +# Convert flat array <-> dict +d = cuex.ir_dict.flat_to_dict(irreps, flat_array) # layout="mul_ir" default +d = cuex.ir_dict.flat_to_dict(irreps, flat_array, layout="ir_mul") +flat = cuex.ir_dict.dict_to_flat(irreps, d) + +# Arithmetic +z = cuex.ir_dict.irreps_add(x, y) +z = cuex.ir_dict.irreps_zeros_like(x) + +# Create template dict +template = cuex.ir_dict.mul_ir_dict(irreps, jax.ShapeDtypeStruct(shape, dtype)) +``` + +`dict_to_flat` always emits `mul_ir`; it has no `layout` argument. To emit +`ir_mul`, swap the last two axes of each dictionary block, flatten each block, +then concatenate in `irreps` order. Regroup duplicate irreps before storing them +as dictionary keys, so a later block cannot overwrite an earlier one. + +## RepArray interface: equivariant_polynomial + +The original interface. Wraps `segmented_polynomial` with `RepArray` — a single contiguous array with representation metadata: + +```python +e = cue.descriptors.fully_connected_tensor_product( + 4 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + 4 * cue.Irreps("SO3", "0 + 1"), +) + +inputs = [ + cuex.randn(jax.random.key(i), rep, (batch,), jnp.float32) + for i, rep in enumerate(e.inputs) +] + +# Returns a RepArray with representation metadata +out = cuex.equivariant_polynomial(e, inputs, method="naive") +out.array # the raw jax.Array +out.reps # dict mapping axes to Rep objects +``` + +## NNX layers + +### IrrepsLinear + +Equivariant linear layer using `dict[Irrep, Array]`: + +```python +from flax import nnx + +linear = cuex.nnx.IrrepsLinear( + irreps_in=cue.Irreps(cue.SO3, "4x0 + 2x1").regroup(), # must be regrouped + irreps_out=cue.Irreps(cue.SO3, "3x0 + 5x1").regroup(), + scale=1.0, + dtype=jnp.float32, + rngs=nnx.Rngs(0), +) + +# Input/output: dict[Irrep, Array] with shape (batch, mul, ir.dim) +x = { + cue.SO3(0): jnp.ones((batch, 4, 1)), + cue.SO3(1): jnp.ones((batch, 2, 3)), +} +y = linear(x) +# y[cue.SO3(0)].shape == (batch, 3, 1) +# y[cue.SO3(1)].shape == (batch, 5, 3) +``` + +Implementation uses `jnp.einsum("uv,...ui->...vi", w, x[ir])` per irrep with `1/sqrt(mul_in)` normalization. + +### SphericalHarmonics + +Uses `spherical_harmonics_ir_dict` internally for the `dict[Irrep, Array]` output: + +```python +sh = cuex.nnx.SphericalHarmonics(max_degree=3, eps=0.0) + +vectors = jax.random.normal(key, (batch, 3)) # 3D vectors +y = sh(vectors) +# y[cue.O3(0, 1)].shape == (batch, 1, 1) # L=0 +# y[cue.O3(1, -1)].shape == (batch, 1, 3) # L=1 +# y[cue.O3(2, 1)].shape == (batch, 1, 5) # L=2 +# y[cue.O3(3, -1)].shape == (batch, 1, 7) # L=3 +``` + +### IrrepsNormalize + +```python +norm = cuex.nnx.IrrepsNormalize(eps=1e-6, scale=1.0, skip_scalars=True) +y = norm(x) # normalizes non-scalar irreps by RMS over ir.dim, averaged over mul +``` + +### MLP (scalar only) + +```python +mlp = cuex.nnx.MLP( + layer_sizes=[64, 128, 64], + activation=jax.nn.silu, + output_activation=False, + dtype=jnp.float32, + rngs=nnx.Rngs(0), +) +y = mlp(x_scalar) # standard dense MLP with 1/sqrt(fan_in) normalization +``` + +### IrrepsIndexedLinear + +For species-indexed linear layers (different weights per atom type): + +```python +indexed_linear = cuex.nnx.IrrepsIndexedLinear( + irreps_in=cue.Irreps(cue.O3, "8x0e").regroup(), + irreps_out=cue.Irreps(cue.O3, "16x0e").regroup(), + num_indices=50, # number of species + scale=1.0, + dtype=jnp.float32, + rngs=nnx.Rngs(0), +) + +# num_index_counts: how many atoms of each species +species_counts = jnp.array([3, 4, 3, ...]) # sum = batch_size +y = indexed_linear(x, species_counts) +``` + +Uses `method="indexed_linear"` internally with `cuex.Repeats`. + +## Preparing polynomials for uniform_1d + +The `uniform_1d` CUDA kernel requires: +1. All segments within each operand have **the same shape** +2. A **single mode** in the subscripts (after preprocessing) + +### From EquivariantPolynomial to uniform_1d-ready + +For `equivariant_polynomial()` (RepArray interface): + +```python +e = cue.descriptors.channelwise_tensor_product(...) +e = e.squeeze_modes().flatten_coefficient_modes() +out = cuex.equivariant_polynomial(e, inputs, method="uniform_1d") +``` + +For `ir_dict` (dict[Irrep, Array] interface), use `_ir_dict` descriptors directly: + +```python +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + irreps_in, irreps_sh, irreps_out +) +poly = desc.polynomial +# Each operand has a single irrep type -> maps naturally to dict[Irrep, Array] +``` + +### Why splitting by irrep matters + +Without splitting, a dense operand like `32x0+32x1` requires all irreps packed into a single contiguous buffer. After splitting, each irrep gets its own separate buffer passed to the CUDA kernel via FFI. The buffers no longer need to be contiguous with each other. + +This is especially useful when the polynomial is preceded or followed by per-irrep linear layers (like `IrrepsLinear`). With split operands, no transpose or copy is needed between the linear layers and the polynomial — the `dict[Irrep, Array]` flows directly through the pipeline. + +## Complete GNN message-passing example + +This pattern is used in NequIP, MACE, and similar equivariant GNN models: + +```python +class MessagePassing(nnx.Module): + def __init__(self, irreps_in, irreps_sh, irreps_out, epsilon, *, name, dtype, rngs): + self.name = name + desc = cue.descriptors.channelwise_tensor_product_ir_dict( + irreps_in, irreps_sh, irreps_out + ) + (self.irreps_out,) = desc.output_irreps + self.poly = desc.polynomial * epsilon + self.weight_numel = self.poly.inputs[0].size + + def __call__(self, weights, node_feats, sph, senders, receivers, num_nodes): + # weights: (num_edges, weight_numel) + w = rearrange(weights, "e (s m) -> e s m", s=self.poly.inputs[0].num_segments) + # node_feats: dict[Irrep, Array] with (nodes, mul, ir.dim) + x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) + # sph: dict[Irrep, Array] with (edges, 1, ir.dim) or (edges, ir.dim) + x2 = jax.tree.map(lambda v: rearrange(v, "e 1 i -> e i"), sph) + + out_template = { + ir: jax.ShapeDtypeStruct( + (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype + ) + for (_, ir), desc in zip(self.irreps_out, self.poly.outputs) + } + + y = cuex.ir_dict.segmented_polynomial_uniform_1d( + self.poly, [w, x1, x2], out_template, + input_indices=[None, senders, None], + output_indices=receivers, + name="tensor_product", + ) + # Rearrange output back to (nodes, mul, ir.dim) for downstream layers + return { + ir: rearrange(v, "n (i s) m -> n (s m) i", i=ir.dim) + for ir, v in y.items() + } +``` + +## RepArray + +Representation-aware JAX array: + +```python +rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) +x = cuex.RepArray(rep, jnp.ones((batch, rep.dim))) +x = cuex.randn(jax.random.key(0), rep, (batch,), jnp.float32) + +x.array # raw jax.Array +x.reps # {axis: Rep} +x.irreps # Irreps (if last axis is IrrepsAndLayout) +``` + +## Key file locations + +| Component | Path | +|-----------|------| +| `segmented_polynomial` primitive | `cuequivariance_jax/segmented_polynomials/segmented_polynomial.py` | +| `uniform_1d` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_uniform_1d.py` | +| `naive` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_naive.py` | +| `indexed_linear` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_indexed_linear.py` | +| `equivariant_polynomial` | `cuequivariance_jax/equivariant_polynomial.py` | +| `ir_dict` module | `cuequivariance_jax/ir_dict.py` | +| `nnx` module | `cuequivariance_jax/nnx.py` | +| `RepArray` | `cuequivariance_jax/rep_array/rep_array_.py` | +| `Repeats` / utilities | `cuequivariance_jax/segmented_polynomials/utils.py` | +| NequIP example | `cuequivariance_jax/examples/nequip_nnx.py` | +| MACE example | `cuequivariance_jax/examples/mace_nnx.py` | diff --git a/skills/cuequivariance-jax/evals/config.yml b/skills/cuequivariance-jax/evals/config.yml new file mode 100644 index 00000000..7c9468f9 --- /dev/null +++ b/skills/cuequivariance-jax/evals/config.yml @@ -0,0 +1,10 @@ +# Dependency versions below are exercised by the CPU contract checks. +# Build the task environment with scripts/requirements.txt; do not assume +# the shared Harbor base image contains these scientific packages. +schema_version: 1 +harbor: + base_image_mode: rebuild +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance-jax/evals/evals.json b/skills/cuequivariance-jax/evals/evals.json new file mode 100644 index 00000000..0b326f12 --- /dev/null +++ b/skills/cuequivariance-jax/evals/evals.json @@ -0,0 +1,76 @@ +{ + "skill_name": "cuequivariance-jax", + "evals": [ + { + "expected_skill": "cuequivariance-jax", + "id": "jax-001", + "prompt": "On CPU, use cuequivariance_jax.segmented_polynomial to execute a channelwise SO3 tensor product for '2x0 + 2x1' and '0 + 1', filtering output types to l=0,1. Use shared weights with no batch axis and inputs shaped (2,1,d1) and (1,3,d2). Write and run a deterministic float64 script comparing naive JAX and jax.jit output to the core NumPy descriptor, then check a weight gradient with central finite differences.", + "expected_output": "A CPU naive implementation with broadcast output shape (2,3,dout), matching NumPy/JIT results and a finite-difference gradient check.", + "assertions": [ + "It explicitly uses method='naive', enables JAX x64 before making float64 arrays, and derives sizes from the polynomial.", + "Its ShapeDtypeStruct accounts for the (2,3) broadcast batch and output size; the weight vector has no batch dimension.", + "The script executes both eager and jitted computations and compares them against the NumPy descriptor with the same operands.", + "It differentiates a nonconstant scalar loss and verifies at least one nonzero weight derivative with a stated finite-difference tolerance.", + "It does not install or claim to execute CUDA kernels." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-002", + "prompt": "For regrouped SO3 irreps '2x0 + 3x1', I have nonconstant batched data flattened in ir_mul order. Using cuequivariance_jax.ir_dict, write and run a script that converts it to a dictionary, checks every block shape and value, converts to mul_ir flat data, and reconstructs the original ir_mul flat data. Can dict_to_flat take a layout argument?", + "expected_output": "An executed conversion with explicit per-block transpose when reconstructing ir_mul, and an accurate explanation of dict_to_flat's API.", + "assertions": [ + "flat_to_dict is called with layout='ir_mul' and produces (...,mul,ir.dim) blocks.", + "It explains that dict_to_flat has no layout argument and emits mul_ir; it does not invent an unsupported keyword.", + "It reconstructs ir_mul by transposing and flattening each dictionary block in irreps order, preserving batch axes.", + "It runs exact comparisons on distinct values and notes that duplicate irrep blocks must be regrouped before using irrep keys." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-003", + "prompt": "Use cuequivariance_jax.nnx.IrrepsLinear on CPU to map SO3 '2x0 + 3x1' to '4x0 + 2x1 + 1x2'. Write and run a Flax NNX example with explicit RNGs and dict[Irrep, Array] inputs. Check output shapes, the l=2 output, the layer's normalization against an independent einsum, and equivariance under a nonzero rotation.", + "expected_output": "A runnable NNX layer example with zero unmatched l=2 output, normalized matching-irrep channel mixing, and passing rotation/reference checks.", + "assertions": [ + "It uses Flax NNX Rngs and regrouped irreps; input blocks have (...,mul,ir.dim) shape.", + "Output blocks have multiplicities 4,2,1 for l=0,1,2 respectively; the unmatched l=2 block is zero.", + "The independent calculation includes scale/sqrt(mul_in) and does not mix inequivalent irreps.", + "It executes both the reference comparison and a nonzero rotation equivariance check on CPU." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-004", + "prompt": "I need a CPU JAX reference for an IrDictPolynomial channelwise tensor product of SO3 '2x0 + 2x1 + 2x2' with '0 + 1 + 2 + 3', filtered to output types l=0,1,2. Write and run it through cuequivariance_jax.segmented_polynomial with method='naive'. Start from high-level (...,mul,ir.dim) dictionary blocks and compare all outputs against the core NumPy polynomial. Explain why these blocks cannot be passed unchanged to segmented_polynomial_uniform_1d.", + "expected_output": "Correctly ordered, transposed and flattened operands, one ShapeDtypeStruct per split output, and an executed JAX/NumPy comparison.", + "assertions": [ + "It uses the descriptor's input_irreps/output_irreps to assemble operands, including weights first; no nonexistent ir_dict.equivariant_polynomial API is invented.", + "It transposes each mul_ir block before flattening for the descriptor and creates a shape/dtype spec for every split output.", + "It executes naive JAX and NumPy using identical arrays and verifies every output numerically.", + "It distinguishes high-level (...,mul,ir.dim) arrays from low-level uniform segmented arrays and their CUDA constraints." + ] + }, + { + "expected_skill": "cuequivariance-jax", + "id": "jax-005", + "prompt": "My JAX application runs on a CPU-only machine. I need equivariant linear layers and spherical harmonics, and saw examples using uniform_1d and IrrepsIndexedLinear. Recommend supported CPU routes and show a small runnable spherical-harmonics example. Do not install a GPU stack.", + "expected_output": "A CPU naive polynomial or NNX spherical-harmonics example and accurate backend boundaries.", + "assertions": [ + "It checks or explains jax.devices() and uses an explicit naive execution path or the NNX SphericalHarmonics CPU implementation.", + "It recommends IrrepsLinear for ordinary CPU channel mixing and identifies indexed_linear/IrrepsIndexedLinear as a CUDA-specific route.", + "It runs a small CPU spherical-harmonics calculation and checks the expected l=0,1,2 output shapes.", + "It does not claim uniform_1d can run on CPU or install CUDA dependencies." + ] + }, + { + "expected_skill": null, + "id": "jax-006", + "prompt": "Give me a standard-library Python function that counts words in a UTF-8 text file. Do not use numerical or machine-learning libraries.", + "expected_output": "A simple standard-library solution without the JAX/cuEquivariance skill.", + "assertions": [ + "It uses standard-library file and text processing.", + "It does not import, install, or invoke JAX, Flax, or cuEquivariance." + ] + } + ] +} diff --git a/skills/cuequivariance-jax/scripts/requirements.txt b/skills/cuequivariance-jax/scripts/requirements.txt new file mode 100644 index 00000000..e50097dc --- /dev/null +++ b/skills/cuequivariance-jax/scripts/requirements.txt @@ -0,0 +1,6 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Pin the source revision because main can contain APIs newer than PyPI. +cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance +cuequivariance-jax @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance_jax +jax==0.11.1 +flax==0.12.10 diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md new file mode 100644 index 00000000..dfff410f --- /dev/null +++ b/skills/cuequivariance-torch/SKILL.md @@ -0,0 +1,418 @@ +--- +name: cuequivariance-torch +description: Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv). Use when writing PyTorch code with cuequivariance. +license: Apache-2.0 +metadata: + author: Mario Geiger +--- + +# cuequivariance_torch: Executing Equivariant Polynomials in PyTorch + +## Overview + +`cuequivariance_torch` (imported as `cuet`) executes `cuequivariance` polynomials on CPU or GPU via PyTorch. It provides: + +1. **Core primitive**: `cuet.SegmentedPolynomial` — `torch.nn.Module` with multiple CUDA backends +2. **High-level operations** (`torch.nn.Module`): `ChannelWiseTensorProduct`, `FullyConnectedTensorProduct`, `Linear`, `SymmetricContraction`, `SphericalHarmonics`, `Rotation`, `Inversion` +3. **Layers**: `cuet.layers.BatchNorm`, `cuet.layers.FullyConnectedTensorProductConv` (message passing) +4. **Utilities**: `triangle_attention`, `triangle_multiplicative_update`, `attention_pair_bias` (AlphaFold3-style) +5. **Export support**: `onnx_custom_translation_table()`, `register_tensorrt_plugins()` + +## Instructions + +1. Check installed versions, `torch.cuda.is_available()`, and the target dtype. Install `cuequivariance-torch` and a suitable PyTorch build; CUDA kernels are optional for the CPU examples. +2. Use `method="naive"` explicitly for a CPU reference, including high-level operations whose defaults select CUDA. Check device, dtype, and segment constraints before selecting a fused method. +3. Derive sizes from the descriptor or module. Specify the feature layout, shared versus per-example weights, and whether weights are internal parameters or supplied externally. +4. Compare a deterministic forward pass against NumPy or an explicit PyTorch reference. Verify autograd, nontrivial rotations, and repeated-destination scatter sums where applicable before optimizing. +5. Use the core `cuequivariance` skill for group definitions and descriptors, or `cuequivariance-jax` for JAX. Do not substitute frameworks or install GPU dependencies for a CPU-only request. + +Source paths below are relative to the `cuequivariance_torch/` project inside a source checkout. For a separately installed skill, inspect `cuet.__file__` or https://github.com/NVIDIA/cuEquivariance. + +## Examples + +Examples build on preceding imports and descriptors. Set `method="naive"` and remove `device="cuda"` for CPU execution; CUDA-specific indexing methods still require their documented backend. + +## Execution methods + +| Method | Backend | Requirements | +|--------|---------|-------------| +| `"naive"` | Pure PyTorch (einsum) | Always works, any platform | +| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | +| `"fused_tp"` | CUDA kernel | GPU, 3- or 4-operand contractions, float32/float64 | +| `"indexed_linear"` | CUDA kernel | GPU, linear with indexed weights, sorted indices | + +## Core primitive: SegmentedPolynomial + +```python +import torch +import cuequivariance as cue +import cuequivariance_torch as cuet + +# Build a descriptor +e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) +poly = e.polynomial + +# Create the module +sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") + +# Forward pass +batch = 8 +x = torch.randn(batch, 3, device="cuda") +[output] = sp([x]) +# output.shape == (batch, 9) -- 1 + 3 + 5 +``` + +### Inputs, indexing, and scatter + +```python +e = cue.descriptors.channelwise_tensor_product( + 16 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) +poly = e.polynomial + +sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") + +w = torch.randn(1, poly.inputs[0].size, device="cuda") # shared weights +x1 = torch.randn(batch, poly.inputs[1].size, device="cuda") # batched input 1 +x2 = torch.randn(batch, poly.inputs[2].size, device="cuda") # batched input 2 + +# Basic forward +[out] = sp([w, x1, x2]) + +# With input gathering (e.g., gather x1 by node index) +senders = torch.randint(0, num_nodes, (num_edges,), device="cuda") +[out] = sp([w, x1, x2], input_indices={1: senders}) + +# With output scattering (accumulate into target nodes) +receivers = torch.randint(0, num_nodes, (num_edges,), device="cuda") +[out] = sp( + [w, x1, x2], + input_indices={1: senders}, + output_indices={0: receivers}, + output_shapes={0: torch.empty(num_nodes, 1, device="cuda")}, +) +``` + +### Math dtype control + +```python +# Compute in float32 regardless of input dtype +sp = cuet.SegmentedPolynomial(poly, method="fused_tp", math_dtype=torch.float32) + +# For fused_tp, math_dtype must be float32 or float64 +# For naive, any torch.dtype works +# For uniform_1d, float32 or float64 (auto-selects float32 if input is e.g. float16) +``` + +## High-level operations + +All operations are `torch.nn.Module` subclasses. They wrap `SegmentedPolynomial` and handle layout transposition automatically. + +### Memory layout + +`IrrepsLayout` controls memory order within each `(mul, ir)` block: + +- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **default, compatible with e3nn** +- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used internally by descriptors** + +Operations accept `layout` (applies to all), or per-operand `layout_in1`, `layout_in2`, `layout_out`. + +### ChannelWiseTensorProduct + +Channel-wise tensor product: pairs channels of `x1` with channels of `x2`. + +```python +# With internal weights (default: shared_weights=True, internal_weights=True) +tp = cuet.ChannelWiseTensorProduct( + cue.Irreps("SO3", "32x0 + 32x1"), # irreps_in1 + cue.Irreps("SO3", "0 + 1"), # irreps_in2 + layout=cue.mul_ir, + device="cuda", + dtype=torch.float32, +) +# tp.weight_numel -- number of weight parameters +# tp.irreps_out -- output irreps (auto-computed) + +x1 = torch.randn(batch, tp.irreps_in1.dim, device="cuda") +x2 = torch.randn(batch, tp.irreps_in2.dim, device="cuda") + +out = tp(x1, x2) # uses internal weight parameter +# out.shape == (batch, tp.irreps_out.dim) + +# With external weights (shared_weights=False) +tp = cuet.ChannelWiseTensorProduct( + cue.Irreps("SO3", "32x0 + 32x1"), + cue.Irreps("SO3", "0 + 1"), + layout=cue.mul_ir, + shared_weights=False, + device="cuda", +) +w = torch.randn(batch, tp.weight_numel, device="cuda") +out = tp(x1, x2, weight=w) + +# With gather/scatter for graph neural networks +out = tp(x1, x2, weight=w, indices_1=senders, indices_out=receivers, size_out=num_nodes) +``` + +Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. + +### FullyConnectedTensorProduct + +All input irrep pairs contribute to all output irreps (dense contraction). + +```python +tp = cuet.FullyConnectedTensorProduct( + cue.Irreps("O3", "4x0e + 4x1o"), # irreps_in1 + cue.Irreps("O3", "0e + 1o"), # irreps_in2 + cue.Irreps("O3", "4x0e + 4x1o"), # irreps_out + layout=cue.mul_ir, + internal_weights=True, + device="cuda", +) + +out = tp(x1, x2) # uses internal weights +# or: out = tp(x1, x2, weight=w) # external weights +``` + +Default method: `"fused_tp"`. + +### Linear + +Equivariant linear layer (weight-only, no second input). + +```python +linear = cuet.Linear( + cue.Irreps("SO3", "4x0 + 2x1"), # irreps_in + cue.Irreps("SO3", "3x0 + 5x1"), # irreps_out + layout=cue.mul_ir, + internal_weights=True, + device="cuda", +) + +out = linear(x) + +# Species-indexed weights (different weights per atom type) +linear = cuet.Linear( + irreps_in, irreps_out, + weight_classes=50, # 50 different weight sets + internal_weights=True, + device="cuda", +) +out = linear(x, weight_indices=species_indices) # species_indices: (batch,) int tensor +``` + +Default method: `"naive"`. Use `method="fused_tp"` for CUDA acceleration. + +### SymmetricContraction + +MACE-style symmetric contraction with element-indexed weights. + +```python +sc = cuet.SymmetricContraction( + cue.Irreps("O3", "32x0e + 32x1o"), # irreps_in (uniform mul required) + cue.Irreps("O3", "32x0e"), # irreps_out (uniform mul required) + contraction_degree=3, + num_elements=95, # number of chemical elements + layout=cue.ir_mul, + dtype=torch.float32, + device="cuda", +) + +# indices: (batch,) int tensor selecting which element weights to use +out = sc(x, indices) +# out.shape == (batch, irreps_out.dim) +``` + +Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. + +### SphericalHarmonics + +```python +sh = cuet.SphericalHarmonics( + ls=[0, 1, 2, 3], + normalize=True, + device="cuda", +) + +vectors = torch.randn(batch, 3, device="cuda") +out = sh(vectors) +# out.shape == (batch, 1 + 3 + 5 + 7) -- sum of 2l+1 +``` + +Default method: `"uniform_1d"`. + +### Rotation and Inversion + +```python +# Rotation (SO3 or O3 irreps) +rot = cuet.Rotation( + cue.Irreps("SO3", "4x0 + 2x1 + 1x2"), + layout=cue.ir_mul, + device="cuda", +) + +# Euler angles (YXY convention) +gamma = torch.tensor([0.1], device="cuda") +beta = torch.tensor([0.2], device="cuda") +alpha = torch.tensor([0.3], device="cuda") +out = rot(gamma, beta, alpha, x) + +# Helper: encode angle for rotation +encoded = cuet.encode_rotation_angle(angle, ell=3) # cos/sin encoding + +# Helper: 3D vector to Euler angles +beta, alpha = cuet.vector_to_euler_angles(vector) + +# Inversion (O3 irreps only) +inv = cuet.Inversion( + cue.Irreps("O3", "4x0e + 2x1o"), + layout=cue.ir_mul, + device="cuda", +) +out = inv(x) +``` + +## Layers + +### BatchNorm + +Batch normalization for equivariant representations (adapted from e3nn). + +```python +bn = cuet.layers.BatchNorm( + cue.Irreps("O3", "4x0e + 4x1o"), + layout=cue.mul_ir, + eps=1e-5, + momentum=0.1, + affine=True, +) + +out = bn(x) # x.shape == (batch, irreps.dim) +``` + +### FullyConnectedTensorProductConv + +Message passing layer for equivariant GNNs (DiffDock-style). + +```python +conv = cuet.layers.FullyConnectedTensorProductConv( + in_irreps=cue.Irreps("O3", "4x0e + 4x1o"), + sh_irreps=cue.Irreps("O3", "0e + 1o"), + out_irreps=cue.Irreps("O3", "4x0e + 4x1o"), + mlp_channels=[16, 32, 32], + mlp_activation=torch.nn.ReLU(), + batch_norm=True, + layout=cue.ir_mul, +) + +# graph = ((src, dst), (num_src_nodes, num_dst_nodes)) +graph = ((src, dst), (num_src_nodes, num_dst_nodes)) + +out = conv(src_features, edge_sh, edge_emb, graph, reduce="mean") +# out.shape == (num_dst_nodes, out_irreps.dim) + +# Optional: separate scalar features for efficient first-layer GEMM +out = conv(src_features, edge_sh, edge_emb, graph, + src_scalars=src_scalars, dst_scalars=dst_scalars) +``` + +## Triangle operations (AlphaFold2-style) + +Require `cuequivariance_ops_torch`. + +```python +# Triangle attention with pair bias +kv_lengths = cuet.mask_to_kv_lengths(prefix_mask) +out = cuet.triangle_attention(q, k, v, bias, scale=scale, kv_lengths=kv_lengths) +# Arbitrary dense masks are supported through the fallback path: +out = cuet.triangle_attention(q, k, v, bias, mask=holey_mask, scale=scale) +# q, k, v: (B, N, H, Q/K, D), bias: (B, 1, H, Q, K) + +# Triangle multiplicative update +out = cuet.triangle_multiplicative_update( + x, # (B, I, J, C) + mask=mask, # (B, I, J) + precision=cuet.TriMulPrecision.DEFAULT, +) + +# Attention with pair bias (strict OpenFold3/Boltz contract) +out, _ = cuet.attention_pair_bias( + single_repr, pair_repr, mask, num_heads, + w_ln_a, b_ln_a, + w_proj_q, b_proj_q, w_proj_k, w_proj_v, + w_proj_g, w_proj_o, w_proj_z, + w_ln_z=w_ln_z, b_ln_z=b_ln_z, +) +# Optional generalized projection params are keyword-only: +# b_proj_k, b_proj_v, w_ln_q, b_ln_q, w_ln_k, b_ln_k, b_proj_g (default None), +# and norm_kind (default "layer_norm"). +``` + +## ONNX and TensorRT export + +```python +# ONNX export +table = cuet.onnx_custom_translation_table() +onnx_program = torch.onnx.export(model, inputs, custom_translation_table=table) + +# TensorRT plugin registration +cuet.register_tensorrt_plugins() +``` + +## Complete GNN example + +```python +import torch +import cuequivariance as cue +import cuequivariance_torch as cuet + +class SimpleGNN(torch.nn.Module): + def __init__(self, irreps_in, irreps_sh, irreps_out): + super().__init__() + self.tp = cuet.ChannelWiseTensorProduct( + irreps_in, irreps_sh, layout=cue.mul_ir, + shared_weights=False, device="cuda", + ) + self.linear = cuet.Linear( + self.tp.irreps_out, irreps_out, + layout=cue.mul_ir, internal_weights=True, device="cuda", + ) + self.sh = cuet.SphericalHarmonics( + ls=[ir.l for _, ir in irreps_sh], normalize=True, device="cuda", + ) + + def forward(self, node_feats, edge_vec, edge_index, num_nodes): + src, dst = edge_index + edge_sh = self.sh(edge_vec) + w = torch.randn(1, self.tp.weight_numel, device=node_feats.device) + + # Message: tensor product on edges with scatter to destination nodes + messages = self.tp( + node_feats, edge_sh, weight=w, + indices_1=src, indices_2=None, + indices_out=dst, size_out=num_nodes, + ) + return self.linear(messages) +``` + +## Key file locations + +| Component | Path | +|-----------|------| +| `SegmentedPolynomial` | `cuequivariance_torch/primitives/segmented_polynomial.py` | +| `uniform_1d` backend | `cuequivariance_torch/primitives/segmented_polynomial_uniform_1d.py` | +| `naive` backend | `cuequivariance_torch/primitives/segmented_polynomial_naive.py` | +| `fused_tp` backend | `cuequivariance_torch/primitives/segmented_polynomial_fused_tp.py` | +| `indexed_linear` backend | `cuequivariance_torch/primitives/segmented_polynomial_indexed_linear.py` | +| `ChannelWiseTensorProduct` | `cuequivariance_torch/operations/tp_channel_wise.py` | +| `FullyConnectedTensorProduct` | `cuequivariance_torch/operations/tp_fully_connected.py` | +| `Linear` | `cuequivariance_torch/operations/linear.py` | +| `SymmetricContraction` | `cuequivariance_torch/operations/symmetric_contraction.py` | +| `SphericalHarmonics` | `cuequivariance_torch/operations/spherical_harmonics.py` | +| `Rotation` / `Inversion` | `cuequivariance_torch/operations/rotation.py` | +| `BatchNorm` | `cuequivariance_torch/layers/batchnorm.py` | +| `FullyConnectedTensorProductConv` | `cuequivariance_torch/layers/tp_conv_fully_connected.py` | +| Triangle operations | `cuequivariance_torch/primitives/triangle.py` | +| Layout transposition | `cuequivariance_torch/primitives/transpose.py` | diff --git a/skills/cuequivariance-torch/evals/config.yml b/skills/cuequivariance-torch/evals/config.yml new file mode 100644 index 00000000..7c9468f9 --- /dev/null +++ b/skills/cuequivariance-torch/evals/config.yml @@ -0,0 +1,10 @@ +# Dependency versions below are exercised by the CPU contract checks. +# Build the task environment with scripts/requirements.txt; do not assume +# the shared Harbor base image contains these scientific packages. +schema_version: 1 +harbor: + base_image_mode: rebuild +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance-torch/evals/evals.json b/skills/cuequivariance-torch/evals/evals.json new file mode 100644 index 00000000..9e777a5f --- /dev/null +++ b/skills/cuequivariance-torch/evals/evals.json @@ -0,0 +1,74 @@ +{ + "skill_name": "cuequivariance-torch", + "evals": [ + { + "expected_skill": "cuequivariance-torch", + "id": "torch-001", + "prompt": "On CPU, use cuequivariance_torch.SphericalHarmonics for l=0,1,2 with normalize=False. Write and run a deterministic float64 example comparing it to the core NumPy descriptor, then run torch.autograd.gradcheck on nonzero input vectors. Do not install CUDA kernels.", + "expected_output": "Naive CPU spherical harmonics with dimension 9, a matching NumPy reference, and passing double-precision gradcheck.", + "assertions": [ + "It explicitly selects method='naive' and uses CPU float64 inputs with requires_grad=True.", + "It uses normalize=False consistently with the raw NumPy polynomial reference and checks output shape (batch,9).", + "It actually runs the NumPy comparison and torch.autograd.gradcheck, reporting the outcome.", + "It does not invoke a CUDA backend or silently skip the gradient check." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-002", + "prompt": "Write and run a CPU float64 example for cuequivariance_torch.Linear mapping SO3 '2x0 + 3x1' to '3x0 + 2x1'. Compare mul_ir and ir_mul layouts using identical explicit weights and nonconstant features. Verify equivalence after correct per-block transposes, then check a nonzero rotation and gradients with respect to features and weights.", + "expected_output": "Two layout-consistent naive Linear computations with passing equivariance and autograd checks.", + "assertions": [ + "It uses method='naive', internal_weights=False, and consistent shared weights and dtype/device for both modules.", + "It converts layouts blockwise by swapping multiplicity and irrep axes, rather than only reshaping the full array.", + "It compares the two outputs after converting to the same layout and executes a nontrivial rotation check.", + "It computes finite, nonzero input and weight gradients from a nonconstant scalar loss." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-003", + "prompt": "Use cuequivariance_torch.ChannelWiseTensorProduct on CPU for SO3 '2x0 + 2x1' and '0 + 1', filtered to l=0,1. I need different externally supplied weights for each batch item. Write and run a float64 example against the equivalent dense NumPy descriptor and demonstrate gradients through both features and weights.", + "expected_output": "A naive CPU channelwise module with external unshared weights, correct layouts and a passing numerical reference comparison.", + "assertions": [ + "It sets shared_weights=False and internal_weights=False, uses method='naive', and obtains the per-example weight dimension from weight_numel.", + "The weight array has a batch axis with distinct weights per item, and module layout agrees with the NumPy descriptor or is explicitly converted.", + "It preserves actual output order and explains that filter_irreps_out filters types rather than setting multiplicities.", + "It executes a NumPy reference comparison and checks finite, nonzero gradients for both feature operands and the external weights." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-004", + "prompt": "On CPU, use cuequivariance_torch.SegmentedPolynomial for a linear descriptor from SO3 '2x0' to '3x0'. With four source nodes and five edges, gather features using senders=[0,1,0,3,1] and scatter outputs to receivers=[2,0,2,1,2] into four destination nodes. Write and run a float64 example, compare against explicit gather plus index_add_, and compare feature and weight gradients. A destination with no edges must stay zero.", + "expected_output": "A correct naive gather/scatter call with repeated destinations summed, an explicit reference, and matching gradients.", + "assertions": [ + "It indexes the feature operand with input_indices and output operand with output_indices, using integer index tensors.", + "It supplies output_shapes in the tensor-based form expected by this API to establish four destination rows.", + "The reference uses index_add_ or equivalent summation, so repeated receiver 2 accumulates all three edges and destination 3 remains zero.", + "It executes forward and gradient comparisons against the explicit reference for both features and weights." + ] + }, + { + "expected_skill": "cuequivariance-torch", + "id": "torch-005", + "prompt": "I copied a cuEquivariance PyTorch example using FullyConnectedTensorProduct and SphericalHarmonics onto a CPU-only laptop and it tries to load CUDA kernels. Explain the backend choice and write and run a small supported CPU tensor-product example. Keep my CPU PyTorch installation.", + "expected_output": "Explicit naive backend selection and a working CPU example with truthful CUDA limitations.", + "assertions": [ + "It explains that some high-level defaults select CUDA and explicitly passes method='naive' for CPU execution.", + "The example defines valid irreps, shapes, device and dtype, runs forward, and checks backward on CPU.", + "It does not promise that fused_tp, uniform_1d, or indexed_linear run on CPU, nor replace the CPU installation with a CUDA stack." + ] + }, + { + "expected_skill": null, + "id": "torch-006", + "prompt": "Explain the difference between git merge and git rebase for a feature branch. I am not asking about numerical code.", + "expected_output": "A direct explanation of Git workflows without invoking cuEquivariance.", + "assertions": [ + "It explains the Git operations and their history implications.", + "It does not import, install, or invoke PyTorch or cuEquivariance." + ] + } + ] +} diff --git a/skills/cuequivariance-torch/scripts/requirements.txt b/skills/cuequivariance-torch/scripts/requirements.txt new file mode 100644 index 00000000..c5f8ec68 --- /dev/null +++ b/skills/cuequivariance-torch/scripts/requirements.txt @@ -0,0 +1,6 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Pin the source revision because main can contain APIs newer than PyPI. +cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance +cuequivariance-torch @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance_torch +--extra-index-url=https://download.pytorch.org/whl/cpu +torch==2.14.1+cpu diff --git a/skills/cuequivariance/SKILL.md b/skills/cuequivariance/SKILL.md new file mode 100644 index 00000000..f371e226 --- /dev/null +++ b/skills/cuequivariance/SKILL.md @@ -0,0 +1,442 @@ +--- +name: cuequivariance +description: Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics). Use when working with cuequivariance group theory, irreps, or segmented polynomials. +license: Apache-2.0 +metadata: + author: Mario Geiger +--- + +# cuequivariance: Groups, Irreps, and Segmented Polynomials + +## Overview + +`cuequivariance` (imported as `cue`) provides two core abstractions: + +1. **Group theory**: `Irrep` subclasses define irreducible representations of Lie groups (SO3, O3, SU2, or custom). `Irreps` manages collections with multiplicities. +2. **Segmented polynomials**: `SegmentedTensorProduct` describes tensor contractions over segments of varying shape, linked by `Path` objects carrying Clebsch-Gordan coefficients. `SegmentedPolynomial` wraps multiple STPs into a polynomial with named inputs/outputs. Two higher-level wrappers attach group representations: + - `EquivariantPolynomial` — dense operands with `IrrepsAndLayout` metadata + - `IrDictPolynomial` — operands already split by irrep, with per-group `Irreps` metadata for the `dict[Irrep, Array]` workflow + +## Instructions + +1. Check the installed version and requested backend. Install `cuequivariance` for CPU NumPy work; JAX and PyTorch execution use the separate `cuequivariance-jax` and `cuequivariance-torch` skills. Inspect available APIs before using newer features such as `IrDictPolynomial` in an older environment. +2. Establish the group, input irreps, multiplicities, layout, and batch shapes before building a descriptor. Preserve the descriptor's actual output block order; a regrouped irrep summary does not reorder its output array. +3. Prefer a built-in descriptor. For a custom group, validate parsing, group relations, selection rules, and Clebsch-Gordan covariance on CPU first. Complex mathematical representations do not imply complex CUDA support. +4. Execute a small deterministic NumPy example and check shapes and equivariance. `IrDictPolynomial` carries metadata; execute its `.polynomial` with operands split according to `.input_irreps` and `.output_irreps`, including weights first when present. +5. Convert `mul_ir` and `ir_mul` by transposing the multiplicity and representation axes **within each irrep block**. A flat reshape alone changes the represented features. + +Source paths below are relative to the `cuequivariance/` project inside a source checkout. When using this skill separately, inspect the installed package with `cue.__file__` or consult https://github.com/NVIDIA/cuEquivariance. The repository is not required for the examples. + +## Examples + +The examples below cover custom groups, descriptors, layouts, and NumPy execution. Some snippets extend variables from the preceding snippet; the final complete example can be run independently. + +## Defining a custom group + +Subclass `cue.Irrep` (a frozen dataclass) and implement: + +```python +from __future__ import annotations +import dataclasses +import re +from typing import Iterator +import numpy as np +import cuequivariance as cue + +@dataclasses.dataclass(frozen=True) +class Z2(cue.Irrep): + odd: bool # dataclass field -- required for correct __eq__ and __hash__ + + # No __init__ needed -- @dataclass(frozen=True) generates it: Z2(odd=True) + + @classmethod + def regexp_pattern(cls) -> re.Pattern: + # Pattern whose first group is passed to from_string + return re.compile(r"(odd|even)") + + @classmethod + def from_string(cls, string: str) -> Z2: + if string not in ("odd", "even"): + raise ValueError(f"Invalid Z2 irrep: {string!r}") + return cls(odd=string == "odd") + + def __repr__(rep: Z2) -> str: + return "odd" if rep.odd else "even" + + def __mul__(rep1: Z2, rep2: Z2) -> Iterator[Z2]: + # Selection rule: which irreps appear in the tensor product rep1 x rep2 + return [Z2(odd=rep1.odd ^ rep2.odd)] + + @classmethod + def clebsch_gordan(cls, rep1: Z2, rep2: Z2, rep3: Z2) -> np.ndarray: + # Shape: (num_paths, rep1.dim, rep2.dim, rep3.dim) + if rep3 in rep1 * rep2: + return np.array([[[[1]]]]) + else: + return np.zeros((0, 1, 1, 1)) + + @property + def dim(rep: Z2) -> int: + return 1 + + def __lt__(rep1: Z2, rep2: Z2) -> bool: + # Ordering for sorting; dimension is compared first by the base class + return rep1.odd < rep2.odd + + @classmethod + def iterator(cls) -> Iterator[Z2]: + # Must yield trivial irrep first + for odd in [False, True]: + yield Z2(odd=odd) + + def discrete_generators(rep: Z2) -> np.ndarray: + # Shape: (num_generators, dim, dim) + if rep.odd: + return -np.ones((1, 1, 1)) + else: + return np.ones((1, 1, 1)) + + def continuous_generators(rep: Z2) -> np.ndarray: + # Shape: (lie_dim, dim, dim) -- Z2 is discrete, so lie_dim=0 + return np.zeros((0, rep.dim, rep.dim)) + + def algebra(self) -> np.ndarray: + # Shape: (lie_dim, lie_dim, lie_dim) -- structure constants [X_i, X_j] = A_ijk X_k + return np.zeros((0, 0, 0)) + + +# Usage: +irreps = cue.Irreps(Z2, "3x odd + 2x even") # dim=5 +``` + +### Required methods summary + +| Method | Returns | Purpose | +|--------|---------|---------| +| `regexp_pattern()` | `re.Pattern` | Parse string like `"1"`, `"0e"`, `"odd"` | +| `from_string(s)` | `Irrep` | Construct from matched string | +| `__repr__` | `str` | Canonical string form | +| `__mul__(a, b)` | `Iterator[Irrep]` | Selection rule for tensor product | +| `clebsch_gordan(a, b, c)` | `ndarray (n, d1, d2, d3)` | CG coefficients | +| `dim` (property) | `int` | Dimension of representation | +| `__lt__(a, b)` | `bool` | Ordering (dimension first, then custom) | +| `iterator()` | `Iterator[Irrep]` | All irreps, trivial first | +| `continuous_generators()` | `ndarray (lie_dim, dim, dim)` | Lie algebra generators | +| `discrete_generators()` | `ndarray (n, dim, dim)` | Finite symmetry generators | +| `algebra()` | `ndarray (lie_dim, lie_dim, lie_dim)` | Structure constants | + +### Built-in groups + +- **`cue.SO3(l)`**: 3D rotations. `l` is a non-negative integer. `dim = 2l+1`. String: `"0"`, `"1"`, `"2"`. +- **`cue.O3(l, p)`**: 3D rotations + parity. `p=+1` (even) or `p=-1` (odd). String: `"0e"`, `"1o"`, `"2e"`. +- **`cue.SU2(j)`**: Spin group. `j` is a non-negative half-integer. String: `"0"`, `"1/2"`, `"1"`. + +## Irreps and layout + +```python +irreps = cue.Irreps("SO3", "16x0 + 4x1 + 2x2") # 16 scalars, 4 vectors, 2 rank-2 +irreps.dim # 16*1 + 4*3 + 2*5 = 38 + +for mul, ir in irreps: + print(mul, ir, ir.dim) # 16 0 1, then 4 1 3, then 2 2 5 +``` + +`IrrepsLayout` controls memory order within each `(mul, ir)` block: + +- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used by all descriptors and ir_dict internally** +- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **used by nnx `dict[Irrep, Array]` and PyTorch** + +`IrrepsAndLayout` combines irreps with a layout into a `Rep`: + +```python +rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) +rep.dim # 10 +``` + +## Building a SegmentedTensorProduct from scratch + +The subscripts string uses Einstein notation. Operands are comma-separated, coefficient modes follow `+`. + +```python +# Matrix-vector multiply: y_i = sum_j M_ij * x_j +d = cue.SegmentedTensorProduct.from_subscripts("ij,j,i") +d.add_segment(0, (3, 4)) # operand 0: matrix segment of shape (3, 4) +d.add_segment(1, (4,)) # operand 1: vector of size 4 +d.add_segment(2, (3,)) # operand 2: output vector of size 3 +d.add_path(0, 0, 0, c=1.0) # link segments 0,0,0 with coefficient=1.0 + +poly = cue.SegmentedPolynomial.eval_last_operand(d) # last operand becomes output +[y] = poly(M_flat, x) # numpy evaluation +``` + +### Multi-segment STP (how descriptors work internally) + +Descriptors build STPs with multiple segments per operand. Each segment corresponds to an irrep block: + +```python +# Linear equivariant map: output[iv] = sum_u weight[uv] * input[iu] +d = cue.SegmentedTensorProduct.from_subscripts("uv,iu,iv") + +# Segment for l=1: ir_dim=3, mul_in=2, mul_out=5 +s_in_0 = d.add_segment(1, (3, 2)) # input block +s_out_0 = d.add_segment(2, (3, 5)) # output block +d.add_path((2, 5), s_in_0, s_out_0, c=1.0) + +# Segment for l=0: ir_dim=1, mul_in=4, mul_out=3 +s_in_1 = d.add_segment(1, (1, 4)) +s_out_1 = d.add_segment(2, (1, 3)) +d.add_path((4, 3), s_in_1, s_out_1, c=1.0) +``` + +### Weights operand + +For weighted tensor products (subscript starting with `uvw` or `uv`), the first operand is always weights. The weight segment shape is `(mul_1, mul_2, ...)` matching the multiplicity modes. The weights operand gets `new_scalars()` irreps since weights are invariant. + +### CG coefficients as path coefficients + +```python +d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") +# For each pair of input irreps and each output irrep in the selection rule: +for cg in cue.clebsch_gordan(ir1, ir2, ir3): + # cg has shape (ir1.dim, ir2.dim, ir3.dim) + d.add_path((mul1, mul2, mul3), seg_in1, seg_in2, seg_out, c=cg) +``` + +## Descriptors + +All descriptors come in two variants: + +- **Original** — returns `EquivariantPolynomial` with dense operands +- **`_ir_dict`** — returns `IrDictPolynomial` with operands already split by irrep + +### EquivariantPolynomial descriptors + +```python +# Fully connected tensor product (all input-output irrep combinations) +e = cue.descriptors.fully_connected_tensor_product( + 16 * cue.Irreps("SO3", "0 + 1 + 2"), + 16 * cue.Irreps("SO3", "0 + 1 + 2"), + 16 * cue.Irreps("SO3", "0 + 1 + 2"), +) + +# Channelwise tensor product (same-channel only, sparse) +e = cue.descriptors.channelwise_tensor_product( + 64 * cue.Irreps("SO3", "0 + 1"), cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), simplify_irreps3=True, +) + +# Full (weightless) tensor product +e = cue.descriptors.full_tensor_product( + cue.Irreps("SO3", "2x0 + 1x1"), cue.Irreps("SO3", "0 + 1"), +) + +# Elementwise tensor product (paired channels) +e = cue.descriptors.elementwise_tensor_product( + cue.Irreps("SO3", "4x0 + 4x1"), cue.Irreps("SO3", "4x0 + 4x1"), +) + +# Linear equivariant map (weight x input) +e = cue.descriptors.linear( + cue.Irreps("SO3", "4x0 + 2x1"), + cue.Irreps("SO3", "3x0 + 5x1"), +) + +# Spherical harmonics +e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2, 3]) + +# Symmetric contraction (MACE-style) +e = cue.descriptors.symmetric_contraction( + 64 * cue.Irreps("SO3", "0 + 1 + 2"), + 64 * cue.Irreps("SO3", "0 + 1"), + (1, 2, 3), +) +``` + +### IrDictPolynomial descriptors + +Each `_ir_dict` variant returns an `IrDictPolynomial` whose polynomial is already split by irrep. The `input_irreps` and `output_irreps` tuples describe the operand groups. + +`channelwise_tensor_product_ir_dict` always simplifies output irreps. To compare +with its dense counterpart, pass `simplify_irreps3=True` to +`channelwise_tensor_product`, or explicitly regroup the output **data** as well +as its irrep metadata. The third argument filters irrep types; it does not set +output multiplicities. + +```python +# Channelwise tensor product +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + 64 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) +# desc.polynomial — SegmentedPolynomial, already split by irrep +# desc.input_irreps — (weight_irreps, irreps1, irreps2) +# desc.output_irreps — (irreps_out,) + +# Fully connected tensor product +desc = cue.descriptors.fully_connected_tensor_product_ir_dict(irreps1, irreps2, irreps3) + +# Full (weightless) tensor product +desc = cue.descriptors.full_tensor_product_ir_dict(irreps1, irreps2) + +# Elementwise tensor product +desc = cue.descriptors.elementwise_tensor_product_ir_dict(irreps1, irreps2) + +# Linear +desc = cue.descriptors.linear_ir_dict(irreps_in, irreps_out) + +# Spherical harmonics +desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2, 3]) + +# Symmetric contraction +desc = cue.descriptors.symmetric_contraction_ir_dict(irreps_in, irreps_out, (1, 2, 3)) +``` + +### IrDictPolynomial + +`IrDictPolynomial` pairs a `SegmentedPolynomial` (already split by irrep) with the `Irreps` that describe each operand group. + +```python +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + 32 * cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 1"), +) + +desc.polynomial # SegmentedPolynomial — each operand is one (mul, ir) block +desc.input_irreps # (weight_irreps, irreps1, irreps2) +desc.output_irreps # (irreps_out,) + +# Scale coefficients +scaled_poly = desc.polynomial * 0.5 + +# Access individual operand info +for i, op in enumerate(desc.polynomial.inputs): + print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") +``` + +Contract: for each `(mul, ir)` block in `input_irreps` / `output_irreps`, the corresponding polynomial operand has size `mul * ir.dim`. + +### split_polynomial_by_irreps + +The low-level function underlying `_ir_dict` descriptors. Splits one polynomial operand at irrep boundaries: + +```python +poly = e.polynomial # from an EquivariantPolynomial +poly = cue.split_polynomial_by_irreps(poly, 2, irreps_sh) # split input 2 +poly = cue.split_polynomial_by_irreps(poly, 1, irreps_in) # split input 1 +poly = cue.split_polynomial_by_irreps(poly, -1, irreps_out) # split output +``` + +### EquivariantPolynomial key methods + +```python +e.inputs # tuple of Rep (group representations for each input) +e.outputs # tuple of Rep +e.polynomial # the underlying SegmentedPolynomial + +# Numpy evaluation +[out] = e(weights, input1, input2) + +# Preparing for uniform_1d execution (see cuequivariance_jax SKILL.md) +e_ready = e.squeeze_modes().flatten_coefficient_modes() + +# Split an operand into per-irrep pieces (for ir_dict interface) +e_split = e.split_operand_by_irrep(1).split_operand_by_irrep(-1) + +# Scale all coefficients +e_scaled = e * 0.5 + +# Fuse compatible STPs +e_fused = e.fuse_stps() +``` + +### normalize_paths_for_operand + +Called internally by descriptors. Normalizes path coefficients so that a random input produces unit-variance output for the specified operand. Critical for numerical stability. + +## SegmentedPolynomial structure + +```python +poly = e.polynomial +poly.num_inputs # number of input operands +poly.num_outputs # number of output operands +poly.inputs # tuple of SegmentedOperand +poly.outputs # tuple of SegmentedOperand +poly.operations # tuple of (Operation, SegmentedTensorProduct) + +# Each operation maps buffers to STP operands +for op, stp in poly.operations: + print(op.buffers) # e.g., (0, 1, 2) means inputs[0], inputs[1] -> outputs[0] + print(stp.subscripts) +``` + +### SegmentedOperand + +```python +operand = poly.inputs[0] +operand.num_segments # how many segments +operand.segments # tuple of shape tuples, e.g., ((3, 4), (1, 2)) +operand.size # total flattened size (sum of products of segment shapes) +operand.ndim # number of dimensions per segment +operand.all_same_segment_shape() # True if all segments have identical shape +operand.segment_shape # the common shape (only if all_same_segment_shape) +``` + +## Custom equivariant polynomial from scratch + +```python +import numpy as np +import cuequivariance as cue + +# Build a fully-connected SO3(1)xSO3(1)->SO3(0) tensor product manually +cg = cue.clebsch_gordan(cue.SO3(1), cue.SO3(1), cue.SO3(0)) # shape (1, 3, 3, 1) + +d = cue.SegmentedTensorProduct.from_subscripts("uvw,iu,jv,kw+ijk") +d.add_segment(1, (3, 4)) # input1: 4x SO3(1), shape=(ir_dim, mul) +d.add_segment(2, (3, 4)) # input2: 4x SO3(1) +d.add_segment(3, (1, 16)) # output: 16x SO3(0) (4*4 fully connected) + +for c in cg: + d.add_path((4, 4, 16), 0, 0, 0, c=c) + +d = d.normalize_paths_for_operand(-1) + +poly = cue.SegmentedPolynomial.eval_last_operand(d) +ep = cue.EquivariantPolynomial( + [ + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1").new_scalars(d.operands[0].size), cue.ir_mul), + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), + cue.IrrepsAndLayout(cue.Irreps("SO3", "4x1"), cue.ir_mul), + ], + [cue.IrrepsAndLayout(cue.Irreps("SO3", "16x0"), cue.ir_mul)], + poly, +) + +# Numpy evaluation +w = np.random.randn(ep.inputs[0].dim) +x = np.random.randn(ep.inputs[1].dim) +y = np.random.randn(ep.inputs[2].dim) +[out] = ep(w, x, y) +``` + +## Key file locations + +| Component | Path | +|-----------|------| +| `Irrep` base class | `cuequivariance/group_theory/representations/irrep.py` | +| `Rep` base class | `cuequivariance/group_theory/representations/rep.py` | +| `SO3` | `cuequivariance/group_theory/representations/irrep_so3.py` | +| `O3` | `cuequivariance/group_theory/representations/irrep_o3.py` | +| `SU2` | `cuequivariance/group_theory/representations/irrep_su2.py` | +| `Irreps` | `cuequivariance/group_theory/irreps_array/irreps.py` | +| `IrrepsLayout` | `cuequivariance/group_theory/irreps_array/irreps_layout.py` | +| `IrrepsAndLayout` | `cuequivariance/group_theory/irreps_array/irreps_and_layout.py` | +| `SegmentedTensorProduct` | `cuequivariance/segmented_polynomials/segmented_tensor_product.py` | +| `SegmentedPolynomial` | `cuequivariance/segmented_polynomials/segmented_polynomial.py` | +| `EquivariantPolynomial` | `cuequivariance/group_theory/equivariant_polynomial.py` | +| `IrDictPolynomial` | `cuequivariance/group_theory/ir_dict_polynomial.py` | +| Descriptors | `cuequivariance/group_theory/descriptors/` | +| Tensor product descriptors | `cuequivariance/group_theory/descriptors/irreps_tp.py` | +| `spherical_harmonics` | `cuequivariance/group_theory/descriptors/spherical_harmonics_.py` | +| `symmetric_contraction` | `cuequivariance/group_theory/descriptors/symmetric_contractions.py` | diff --git a/skills/cuequivariance/evals/config.yml b/skills/cuequivariance/evals/config.yml new file mode 100644 index 00000000..7c9468f9 --- /dev/null +++ b/skills/cuequivariance/evals/config.yml @@ -0,0 +1,10 @@ +# Dependency versions below are exercised by the CPU contract checks. +# Build the task environment with scripts/requirements.txt; do not assume +# the shared Harbor base image contains these scientific packages. +schema_version: 1 +harbor: + base_image_mode: rebuild +skill_workspace: + mode: isolated +grading: + mode: aces_default diff --git a/skills/cuequivariance/evals/evals.json b/skills/cuequivariance/evals/evals.json new file mode 100644 index 00000000..977efc84 --- /dev/null +++ b/skills/cuequivariance/evals/evals.json @@ -0,0 +1,77 @@ +{ + "skill_name": "cuequivariance", + "evals": [ + { + "expected_skill": "cuequivariance", + "id": "core-001", + "prompt": "Implement a frozen, hashable cue.Irrep subclass for the three complex one-dimensional irreps of Z3, labelled k=0,1,2. Use the generator exp(2j*pi*k/3), reject invalid labels and strings, and yield the trivial irrep first. Write and run a standalone CPU NumPy script that checks string round trips, the group relation, all tensor-product selection rules, and Clebsch-Gordan covariance. Do not install a GPU framework.", + "expected_output": "Runnable Z3 implementation and executed assertions for all three irreps and all 27 CG triples, with a clear boundary between complex mathematical representations and GPU execution support.", + "assertions": [ + "The immutable dataclass stores and validates k in {0,1,2}; equal irreps are usable as dictionary keys and valid strings round-trip.", + "The generator has shape (1,1,1), its cube is identity, and the continuous-generator and algebra arrays have shapes (0,1,1) and (0,0,0).", + "Tensor products select k3=(k1+k2)%3; allowed CG arrays have shape (1,1,1,1) and disallowed arrays (0,1,1,1).", + "The agent executes the script and numerically checks generator covariance for every allowed coupling, rather than only displaying untested code.", + "It does not claim that complex Z3 representations imply support in CUDA kernels." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-002", + "prompt": "Using only cuequivariance and NumPy on CPU, build the full, unweighted SO3 tensor product of '2x0 + 1x1 + 1x2' and '1x0 + 1x1'. Give the aggregated output irreps and dimension, but preserve the descriptor's actual output order when executing it. Write and run a deterministic example with two batched inputs and verify equivariance under a nonzero rotation.", + "expected_output": "A full_tensor_product descriptor with two inputs, an aggregated output of 3x0 + 5x1 + 3x2 + 1x3 (dimension 40), and a passing numerical equivariance check.", + "assertions": [ + "The descriptor is full_tensor_product with no trainable weight operand; both input arrays and the random seed are defined.", + "The aggregated output is 3x0 + 5x1 + 3x2 + 1x3 with dimension 40.", + "The code uses the descriptor's actual representations and layouts for input/output rotation, without interpreting a regrouped summary as the stored output order.", + "For row-vector batches it applies representation matrices with the appropriate transpose and executes a nontrivial equivariance comparison with a reported tolerance." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-003", + "prompt": "I have SO3 feature blocks with irreps '2x0 + 2x1 + 2x2' and edge blocks '0 + 1 + 2 + 3', each stored as (..., multiplicity, ir.dim). Use channelwise_tensor_product_ir_dict, keeping output irrep types l=0,1,2. On CPU with NumPy, write and run a check against the equivalent dense descriptor using the same nonconstant inputs and weights. Explain operand ordering, layout conversion, and what the third descriptor argument controls.", + "expected_output": "A valid IrDictPolynomial execution through its polynomial and a numerical match against the dense channelwise tensor product after consistent block ordering and layouts.", + "assertions": [ + "It treats IrDictPolynomial as metadata and calls its .polynomial with arrays split according to input_irreps and output_irreps, with the weight group first.", + "Each high-level (...,mul,ir.dim) block is transposed to (...,ir.dim,mul) before flattening for ir_mul; nonconstant data makes a reshape-only bug detectable.", + "It respects each descriptor's actual output ordering when comparing results; any regrouping also reorders the data.", + "It explains that the third channelwise argument filters output irrep types and does not prescribe output multiplicities.", + "The NumPy reference comparison is actually run and passes; no JAX or CUDA installation is needed." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-004", + "prompt": "My NumPy features for SO3 '2x0 + 3x1' are flattened in mul_ir order, but a cue.descriptors.linear descriptor uses ir_mul. Write and run a correct per-block conversion in both directions. Use distinct values in every channel, demonstrate why reshape alone is wrong, and verify the linear operation under a nonzero SO3 rotation.", + "expected_output": "Executed, invertible blockwise layout conversions and an equivariance check on correctly converted data.", + "assertions": [ + "The conversion slices each irrep block and swaps only multiplicity and representation axes, preserving batch axes and block order.", + "It tests exact round trips on nonconstant data and demonstrates a difference from reshape-only conversion for the vector block.", + "The linear descriptor is evaluated with matching weights and layouts, and rotations use row-vector conventions consistently.", + "A nontrivial numerical equivariance assertion is executed and passes." + ] + }, + { + "expected_skill": "cuequivariance", + "id": "core-005", + "prompt": "I need a compatibility path for an older CPU-only deployment where cuequivariance imports but may not expose IrDictPolynomial or channelwise_tensor_product_ir_dict. That deployment cannot upgrade or install JAX or PyTorch. Show version and capability checks and a dense NumPy fallback for a channelwise tensor product. The sandbox may have a newer version; do not assume its available APIs exist in the older deployment and do not change any dependencies.", + "expected_output": "Capability checks and a dense NumPy fallback, without requiring the missing API or changing dependencies.", + "assertions": [ + "It checks or shows checks for cue.__version__ and hasattr/getattr for the missing APIs, rather than assuming this skill's newest API is available.", + "It proposes cue.descriptors.channelwise_tensor_product and NumPy execution if present, making any unavailable fallback explicit.", + "It explains how to convert per-irrep blocks to the dense descriptor's layout and preserve operand order.", + "It does not install or upgrade packages, switch frameworks, or pretend the unavailable IrDictPolynomial call succeeded." + ] + }, + { + "expected_skill": null, + "id": "core-006", + "prompt": "Explain how to choose a PostgreSQL index for SELECT * FROM events WHERE tenant_id = 42 AND created_at >= now() - interval '1 day' ORDER BY created_at DESC LIMIT 100. I only need database advice.", + "expected_output": "An answer about PostgreSQL indexing without using cuEquivariance.", + "assertions": [ + "It answers the database indexing question directly.", + "It does not import, install, or invoke cuEquivariance or force the question into group theory or tensor products." + ] + } + ] +} diff --git a/skills/cuequivariance/scripts/requirements.txt b/skills/cuequivariance/scripts/requirements.txt new file mode 100644 index 00000000..3a1a557b --- /dev/null +++ b/skills/cuequivariance/scripts/requirements.txt @@ -0,0 +1,3 @@ +# CPU evaluation baseline for the APIs documented in this skill. +# Pin the source revision because main can contain APIs newer than PyPI. +cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance From 094d600d4bf172ffccb72a4087640f03f50d98bd Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Fri, 16 Oct 2026 18:43:07 +0000 Subject: [PATCH 02/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 131 ++++++++++++++++++++++ skills/cuequivariance-jax/skill-card.md | 83 ++++++++++++++ skills/cuequivariance-jax/skill.oms.sig | 1 + skills/cuequivariance-torch/BENCHMARK.md | 129 +++++++++++++++++++++ skills/cuequivariance-torch/skill-card.md | 83 ++++++++++++++ skills/cuequivariance-torch/skill.oms.sig | 1 + skills/cuequivariance/BENCHMARK.md | 131 ++++++++++++++++++++++ skills/cuequivariance/skill-card.md | 83 ++++++++++++++ skills/cuequivariance/skill.oms.sig | 1 + 9 files changed, 643 insertions(+) create mode 100644 skills/cuequivariance-jax/BENCHMARK.md create mode 100644 skills/cuequivariance-jax/skill-card.md create mode 100644 skills/cuequivariance-jax/skill.oms.sig create mode 100644 skills/cuequivariance-torch/BENCHMARK.md create mode 100644 skills/cuequivariance-torch/skill-card.md create mode 100644 skills/cuequivariance-torch/skill.oms.sig create mode 100644 skills/cuequivariance/BENCHMARK.md create mode 100644 skills/cuequivariance/skill-card.md create mode 100644 skills/cuequivariance/skill.oms.sig diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md new file mode 100644 index 00000000..01f603b0 --- /dev/null +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -0,0 +1,131 @@ +# Skill Benchmark: cuequivariance-jax + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `cuequivariance-jax` +- Evaluation date: 2026-10-16 +- Evaluator version: `1.5.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:f653a01cf39e3dffe96fbacf75b1b1ee214a79c4fc5b682f49f31e6963d7bb1a` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 81.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 50.0% (+16.7 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 100.0% (+2.1 points) | 90.8% → 87.9% (-2.9 points) | +| Efficiency | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | 63.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 6,770,880 | 5,344,900 | +1,425,980 | +26.68% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 1,092,116 | 1,209,345 | -117,229 | -9.69% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 1,669,333 | 476,301 | +1,193,032 | +250.48% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 745,879 | 887,128 | -141,249 | -15.92% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 2,637,995 | 2,282,069 | +355,926 | +15.60% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 596,440 | 460,588 | +135,852 | +29.50% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 29,117 | 29,469 | -352 | -1.19% | skill 1/1; base 1/1 | +| codex | All cases | 3,927,148 | 2,136,742 | +1,790,406 | +83.79% | skill 6/6; base 6/6 | +| codex | jax-001 | 793,152 | 614,925 | +178,227 | +28.98% | skill 1/1; base 1/1 | +| codex | jax-002 | 304,987 | 323,916 | -18,929 | -5.84% | skill 1/1; base 1/1 | +| codex | jax-003 | 1,028,052 | 595,600 | +432,452 | +72.61% | skill 1/1; base 1/1 | +| codex | jax-004 | 1,727,473 | 556,340 | +1,171,133 | +210.51% | skill 1/1; base 1/1 | +| codex | jax-005 | 60,043 | 32,627 | +27,416 | +84.03% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,441 | 13,334 | +107 | +0.80% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 10,698,028 | 7,481,642 | +3,216,386 | +42.99% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance-jax/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: Description very long (276 chars, recommend 50-150) (`skills/cuequivariance-jax/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance-jax/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance-jax/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No limitations documented (`skills/cuequivariance-jax/SKILL.md`) +- 2 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md new file mode 100644 index 00000000..616fc853 --- /dev/null +++ b/skills/cuequivariance-jax/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear).
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers use this skill when writing JAX code with cuequivariance to execute equivariant polynomials on CPU or GPU, work with dict[Irrep, Array] or RepArray data, and build Flax NNX equivariant layers for geometric neural networks such as NequIP- and MACE-style GNNs.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod and compared against a no-skill baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from the `security` signal.
+- Correctness: Is the answer correct? Scored from the `accuracy` signal.
+- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
+- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` plus 50% `token_efficiency`.
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 81.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 50.0% (+16.7 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 100.0% (+2.1 points) | 90.8% → 87.9% (-2.9 points) | +| Efficiency | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | 63.9% — baseline ran, but no comparable score was available; uplift unavailable | + +## Skill Version(s):
+0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig new file mode 100644 index 00000000..e69cc951 --- /dev/null +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjliMjA0M2U4M2RkZDNlYmQwYzFhNTI0ODlmMmI1Zjc2MWMzZjgzNmJhMDk4MjhmOTVlMDM5MmM0MGRjYjY0NDUiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImQyODY3Y2ZkZjc1ZTA2MzFiOTMwYTZiZjY3Y2NiODQ4ZjJjYzY4MDM4YjZkZjlmMzZhNjNhNWI0MDM4MTAwZDQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiZmE3MWJkYjgzYTlkZGI4MWE4OWQ3ODlhOTk3ZGRiYzdlYTk3NmNlNzY1OTYwYmJjMzVjNTBjZDUyY2EyNTYwMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YjY1ZjFmODVjZGY1ZGQ2NDkyMjJkOGEzNWQ4ZmY4Mzg2N2Y3MWU3NDUzOWQxNGM5Y2JiZWI4MWVlNDQ5OTg1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZTFkN2JkM2YzY2JiOTdjMzljZWY1ZWU3ZGQyODFmMTVkMmE1YWRhMjQ0ZDViNTQzMDc2NjIyNDEyMGY2ZGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiOGQ5Mjg4YzExOTExNDlkODk4ODBhYzhkMGU3OTVmYThkZjFiZmE5MzM0NjYzMzZlZjNjZWM3NTFiMzBkMjE0YSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEp0qcsQ+nfR/fEx1/MlAYk7r+kfMuSbZUzBrUM2aqko7jdkPrVf2bcGftAzm6GmRQIwaII+wQG2ppFMVZEyAyAqfXiCrl6F4V5L/2+CRlze6RWflpSkXyye7y50iGnm7mch","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md new file mode 100644 index 00000000..eb34d33c --- /dev/null +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -0,0 +1,129 @@ +# Skill Benchmark: cuequivariance-torch + +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** + +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. + +## Evaluation Metadata + +- Skill: `cuequivariance-torch` +- Evaluation date: 2026-10-16 +- Evaluator version: `1.5.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:c59a90389aabc4ce9d6756217b3f98021106bda2f5a6df42bb36c78c5a0135db` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Security | 50.0% → 50.0% (±0.0 points) | Not available | +| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | +| Discoverability | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Effectiveness | 90.1% → 100.0% (+9.9 points) | Not available | +| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 7,423,526 | 6,490,723 | +932,803 | +14.37% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 1,379,936 | 1,308,078 | +71,858 | +5.49% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 1,182,955 | 603,673 | +579,282 | +95.96% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 1,161,987 | 928,849 | +233,138 | +25.10% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 1,286,524 | 1,044,323 | +242,201 | +23.19% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 2,382,097 | 2,576,008 | -193,911 | -7.53% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 30,027 | 29,792 | +235 | +0.79% | skill 1/1; base 1/1 | +| codex | All cases | 5,264,185 | 4,068,564 | N/A | N/A | skill 5/6; base 6/6 | +| codex | torch-001 | N/A | 1,321,564 | N/A | N/A | skill 0/1; base 1/1 | +| codex | torch-002 | 1,651,497 | 768,903 | +882,594 | +114.79% | skill 1/1; base 1/1 | +| codex | torch-003 | 1,103,907 | 689,022 | +414,885 | +60.21% | skill 1/1; base 1/1 | +| codex | torch-004 | 520,123 | 425,385 | +94,738 | +22.27% | skill 1/1; base 1/1 | +| codex | torch-005 | 1,974,911 | 850,079 | +1,124,832 | +132.32% | skill 1/1; base 1/1 | +| codex | torch-006 | 13,747 | 13,611 | +136 | +1.00% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 12,687,711 | 10,559,287 | N/A | N/A | skill 11/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 8 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance-torch/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: Description very long (363 chars, recommend 50-150) (`skills/cuequivariance-torch/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance-torch/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No mention of error handling or validation (`skills/cuequivariance-torch/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance-torch/SKILL.md`) +- 3 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md new file mode 100644 index 00000000..b36976e2 --- /dev/null +++ b/skills/cuequivariance-torch/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv).
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and researchers building equivariant geometric neural networks in PyTorch use this skill to write and validate cuequivariance_torch code, including SegmentedPolynomial execution backends, high-level equivariant operations and layers, triangle operations, and ONNX/TensorRT export, on CPU or GPU.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox; with-skill runs compared against no-skill baselines.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use?
+- Correctness: Is the answer correct?
+- Discoverability: Was the right skill loaded when needed?
+- Effectiveness: Did the skill help complete the task?
+- Efficiency: Did it avoid wasted tool calls and token usage?
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Security | 50.0% → 50.0% (±0.0 points) | Not available | +| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | +| Discoverability | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Effectiveness | 90.1% → 100.0% (+9.9 points) | Not available | +| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | + +## Skill Version(s):
+0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig new file mode 100644 index 00000000..b2d22eb3 --- /dev/null +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNDczOTRmMWYzYjczYzRjZTFhMDNmNmZhNDU5YjE2OGZlYmU2ZWY5ZWQyMGNlNzM4MzI0YjI1MjYxZDczZGU5MSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTkyM2ZjMjhjYWEwNjIyZjgxZTVhNjEyNjBhZmY0NDc5ODQzNGQzZGRmMWFlNTliNGU2MzJhNjZiODVlNDhmOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5N2RlNTE5MDRkNGFmYTNlMmM5ZDIxYzE3NWJhYWRhZjYxYmNiYTZkNDVjNDgxYjlmNzkxMGQyNzEzMmM3NmEyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogImE5NjliMzAxMTgzZDBmMDJiNzI4ZGY5YzVlYzY5ZGVkNjEyMDk2YzgyMzQzODVhZjcxYzg4YzBjYmMyMjYzNDAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIxODQyYTI3YzBkYTRjNmU1MjQzMTk0NDZhYmJiZGIyNDE1YzQ3N2MzMDI3MzI5OGU4MDBjMTFiNzZjZjM2NjFmIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEJHGYkDyjFJwQprikuv5nCbQp6maPBQkswheYxv1TthPTlwQoNCpmnEGJQiRn+dPgIwfW03qgsYdOS9bwuhuz+5TC4PklmcUw6vml5jg94MsMmYpdDofxP3KEWu9PTk5T+p","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md new file mode 100644 index 00000000..2549d4fd --- /dev/null +++ b/skills/cuequivariance/BENCHMARK.md @@ -0,0 +1,131 @@ +# Skill Benchmark: cuequivariance + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `cuequivariance` +- Evaluation date: 2026-10-16 +- Evaluator version: `1.5.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:64264cc94192671086b1204edef9cf61e0c0710d636da7afc0c1f0c594e2cd55` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 83.3% (+50.0 points) | 66.7% → 83.3% (+16.6 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.5% → 96.7% (-0.8 points) | 90.0% → 94.2% (+4.2 points) | +| Efficiency | 68.2% — baseline ran, but no comparable score was available; uplift unavailable | 72.5% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 6,178,511 | 6,967,056 | -788,545 | -11.32% | skill 6/6; base 6/6 | +| claude-code | core-001 | 1,116,183 | 1,092,606 | +23,577 | +2.16% | skill 1/1; base 1/1 | +| claude-code | core-002 | 590,901 | 1,036,330 | -445,429 | -42.98% | skill 1/1; base 1/1 | +| claude-code | core-003 | 729,662 | 1,931,071 | -1,201,409 | -62.21% | skill 1/1; base 1/1 | +| claude-code | core-004 | 994,549 | 779,822 | +214,727 | +27.54% | skill 1/1; base 1/1 | +| claude-code | core-005 | 2,716,801 | 2,096,799 | +620,002 | +29.57% | skill 1/1; base 1/1 | +| claude-code | core-006 | 30,415 | 30,428 | -13 | -0.04% | skill 1/1; base 1/1 | +| codex | All cases | 2,256,123 | 2,801,003 | -544,880 | -19.45% | skill 6/6; base 6/6 | +| codex | core-001 | 556,778 | 590,845 | -34,067 | -5.77% | skill 1/1; base 1/1 | +| codex | core-002 | 425,769 | 637,619 | -211,850 | -33.23% | skill 1/1; base 1/1 | +| codex | core-003 | 657,989 | 621,907 | +36,082 | +5.80% | skill 1/1; base 1/1 | +| codex | core-004 | 552,745 | 453,685 | +99,060 | +21.83% | skill 1/1; base 1/1 | +| codex | core-005 | 49,004 | 467,286 | -418,282 | -89.51% | skill 1/1; base 1/1 | +| codex | core-006 | 13,838 | 29,661 | -15,823 | -53.35% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 8,434,634 | 9,768,059 | -1,333,425 | -13.65% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: Description very long (310 chars, recommend 50-150) (`skills/cuequivariance/SKILL.md`) +- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance/SKILL.md`) +- **LOW** QUALITY/quality_reliability: No limitations documented (`skills/cuequivariance/SKILL.md`) +- 2 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md new file mode 100644 index 00000000..a4314d24 --- /dev/null +++ b/skills/cuequivariance/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics).
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and researchers building equivariant geometric neural networks use this skill to define irreps and custom symmetry groups, construct segmented tensor products and equivariant polynomial descriptors with the cuequivariance library, and validate them with CPU NumPy execution.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox; with-skill runs compared against no-skill baselines.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use?
+- Correctness: Is the answer correct?
+- Discoverability: Was the right skill loaded when needed?
+- Effectiveness: Did the skill help complete the task?
+- Efficiency: Did it avoid wasted tool calls and token usage?
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 83.3% (+50.0 points) | 66.7% → 83.3% (+16.6 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.5% → 96.7% (-0.8 points) | 90.0% → 94.2% (+4.2 points) | +| Efficiency | 68.2% — baseline ran, but no comparable score was available; uplift unavailable | 72.5% — baseline ran, but no comparable score was available; uplift unavailable | + +## Skill Version(s):
+0.13.0 (source: VERSION file, git commit 5e144b4 2026-10-16)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig new file mode 100644 index 00000000..826d4502 --- /dev/null +++ b/skills/cuequivariance/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNDcxMjgzNTdiZWYxMmQ0ZmUwYmU1MDEyMjA0ZWJmZjI3ZGJhY2M4ZjFiODM5ZmNhNjBhMDU0OTVlOWExY2VhMiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMWE4Y2E4MTMyNGM5NTZlNjM3NjFiM2UwNGJmOTNhOWJkZGIzMmEyMmYyZWNmNzExODE5ZTE4YjlhOTQzYjQ0NyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYmEyYzg3MjVmNDk5MGVhYmRhNGEwZDIwZDVhYjkxOTRkMmEyZTZmOWE2NzdhOGNiNzFlOTgyNDFkMTdmZjRiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjYxOGFkMWMxYTg5ZjE1ZGIxZGIxMTNiMGI5YmY0NDcxMzMwMTEyZTUzOWIxMGZkOGEyNGE3YzQ0ZDI5YjcwMDkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmNThlYjA5ZTBhN2Q5OGJmZDMxZjUwODMxMGE1YzJmM2Y0ZWYwMTE1NDFkNDY3ZmRlOWZkYjU3ZjdjODM4MDdkIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGPNLwQ70vkIpgNWgKFMlKTGVFJiuY+RCSFNs1y7opxKTfiBKn/6ZdD2kOyfgKxa1AIwBScy//PeN4xrjNEvJbP3kkvyicKp7QYIp0a1ra3lIyrSj314C8uJ/B9WfUYtU3d+","keyid":""}]}} \ No newline at end of file From a126d116fa721a47400e0639e1ee36cccb1b3add Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Fri, 16 Oct 2026 14:23:15 -0700 Subject: [PATCH 03/19] tier 3 fixes Signed-off-by: Ohad Mosafi --- .github/tests/test_torch_skill_examples.py | 58 +++ .github/workflows/tests.yml | 2 +- skills/README.md | 9 +- skills/cuequivariance-torch/SKILL.md | 508 +++++++-------------- 4 files changed, 234 insertions(+), 343 deletions(-) create mode 100644 .github/tests/test_torch_skill_examples.py diff --git a/.github/tests/test_torch_skill_examples.py b/.github/tests/test_torch_skill_examples.py new file mode 100644 index 00000000..2b80c869 --- /dev/null +++ b/.github/tests/test_torch_skill_examples.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run the published skill examples and check the trainable GNN's invariants.""" + +import re +from pathlib import Path + +import numpy as np +import pytest +import torch + +import cuequivariance as cue + + +@pytest.fixture(scope="module") +def skill_namespace(): + skill = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" + namespace = {} + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", skill.read_text(), re.S) + ): + exec(compile(code, f"{skill}:example-{index}", "exec"), namespace) + return namespace + + +def test_documented_gnn_is_trainable_and_equivariant(skill_namespace): + torch.manual_seed(41) + irreps_in = cue.Irreps("SO3", "2x0+2x1") + irreps_out = cue.Irreps("SO3", "3x0+1") + model = skill_namespace["SimpleGNN"](irreps_in, irreps_out).double() + features = torch.randn(5, irreps_in.dim, dtype=torch.float64, requires_grad=True) + vectors = torch.randn(6, 3, dtype=torch.float64, requires_grad=True) + senders = torch.tensor([0, 1, 3, 0, 1, 2]) + receivers = torch.tensor([2, 0, 2, 1, 2, 0]) + output = model(features, vectors, senders, receivers, 4) + assert output.shape == (4, irreps_out.dim) + assert torch.count_nonzero(output[3]) == 0 + torch.testing.assert_close( + output, model(features, vectors, senders, receivers, 4), rtol=0, atol=0 + ) + + rotation = np.array([0.2, -0.3, 0.1]) + din, dout = [ + torch.from_numpy(cue.IrrepsAndLayout(r, cue.mul_ir).exp_map(rotation, [])) + for r in (irreps_in, irreps_out) + ] + dvector = torch.from_numpy(cue.SO3(1).exp_map(rotation, [])) + rotated = model(features @ din.T, vectors @ dvector.T, senders, receivers, 4) + torch.testing.assert_close(rotated, output @ dout.T, rtol=1e-7, atol=1e-9) + + output.square().sum().backward() + for name, tensor in [ + *model.named_parameters(), + ("features", features), + ("vectors", vectors), + ]: + assert tensor.grad is not None, name + assert torch.isfinite(tensor.grad).all(), name + assert tensor.grad.abs().sum() > 0, name diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0bff3678..8aa66551 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -122,6 +122,6 @@ jobs: - name: Check PyTorch skill evaluations and packaging run: | - pytest -q .github/tests/test_torch_skill_evals.py + pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py uv pip install hatchling python .github/scripts/check_skill_packaging.py --package cuequivariance_torch diff --git a/skills/README.md b/skills/README.md index 382486b1..fbc82f7e 100644 --- a/skills/README.md +++ b/skills/README.md @@ -50,14 +50,15 @@ before changing the assertions. The CUDA backend constraints are checked as guidance; these datasets do not benchmark GPU performance or validate CUDA kernels. The tests under `.github/tests/test_*_skill_evals.py` verify the mathematical and -API expectations behind the prompts. They are maintenance checks, not live agent -scores. To run them against a development checkout with the relevant frontend -dependencies installed: +API expectations behind the prompts. `test_torch_skill_examples.py` also executes +the published Torch examples and checks the GNN's determinism, equivariance, and +gradients. These are maintenance checks, not live agent scores. To run them against +a development checkout with the relevant frontend dependencies installed: ```bash pytest -q .github/tests/test_core_skill_evals.py JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py -pytest -q .github/tests/test_torch_skill_evals.py +pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py ``` To verify the pinned evaluation environment in a separate Python 3.12 environment: diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md index dfff410f..ca86c49d 100644 --- a/skills/cuequivariance-torch/SKILL.md +++ b/skills/cuequivariance-torch/SKILL.md @@ -1,418 +1,250 @@ --- name: cuequivariance-torch -description: Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv). Use when writing PyTorch code with cuequivariance. +description: Use when building or debugging cuEquivariance PyTorch tensor products or equivariant GNNs, especially CPU references, layouts, indexing, and gradients. license: Apache-2.0 metadata: author: Mario Geiger + tags: [pytorch, equivariance, tensor-products] --- -# cuequivariance_torch: Executing Equivariant Polynomials in PyTorch +# cuEquivariance in PyTorch -## Overview +## Purpose -`cuequivariance_torch` (imported as `cuet`) executes `cuequivariance` polynomials on CPU or GPU via PyTorch. It provides: +Use `cuequivariance_torch` (`cuet`) to execute `cuequivariance` (`cue`) descriptors with PyTorch autograd. This guide covers CPU reference calculations, CUDA backend selection, and equivariant layers. Use the project's existing framework and environment. -1. **Core primitive**: `cuet.SegmentedPolynomial` — `torch.nn.Module` with multiple CUDA backends -2. **High-level operations** (`torch.nn.Module`): `ChannelWiseTensorProduct`, `FullyConnectedTensorProduct`, `Linear`, `SymmetricContraction`, `SphericalHarmonics`, `Rotation`, `Inversion` -3. **Layers**: `cuet.layers.BatchNorm`, `cuet.layers.FullyConnectedTensorProductConv` (message passing) -4. **Utilities**: `triangle_attention`, `triangle_multiplicative_update`, `attention_pair_bias` (AlphaFold3-style) -5. **Export support**: `onnx_custom_translation_table()`, `register_tensorrt_plugins()` +## Inputs -## Instructions - -1. Check installed versions, `torch.cuda.is_available()`, and the target dtype. Install `cuequivariance-torch` and a suitable PyTorch build; CUDA kernels are optional for the CPU examples. -2. Use `method="naive"` explicitly for a CPU reference, including high-level operations whose defaults select CUDA. Check device, dtype, and segment constraints before selecting a fused method. -3. Derive sizes from the descriptor or module. Specify the feature layout, shared versus per-example weights, and whether weights are internal parameters or supplied externally. -4. Compare a deterministic forward pass against NumPy or an explicit PyTorch reference. Verify autograd, nontrivial rotations, and repeated-destination scatter sums where applicable before optimizing. -5. Use the core `cuequivariance` skill for group definitions and descriptors, or `cuequivariance-jax` for JAX. Do not substitute frameworks or install GPU dependencies for a CPU-only request. - -Source paths below are relative to the `cuequivariance_torch/` project inside a source checkout. For a separately installed skill, inspect `cuet.__file__` or https://github.com/NVIDIA/cuEquivariance. +Required for the computation: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership. Derive these from the request or existing code and state any defaults. Graph operations also require source/destination counts and node/edge indexing. Optional inputs include a preferred backend or performance/export targets. Preserve the user's project constraints and installed API. -## Examples - -Examples build on preceding imports and descriptors. Set `method="naive"` and remove `device="cuda"` for CPU execution; CUDA-specific indexing methods still require their documented backend. +## Instructions -## Execution methods +1. Check the selected interpreter and imports once using the probe below. Reuse a working environment. A missing CUDA backend is usually a method-selection issue, not a reason to reinstall PyTorch. +2. Set `method="naive"` for CPU calculations. Match module parameters, features, weights, and reference computations in device and dtype. Use float64 for small numerical and gradient checks. +3. Specify layout and weight ownership explicitly. Derive sizes and output order from the module/descriptor; do not guess multiplicities from a filter. +4. Build the smallest relevant computation, run it, and report the actual checks and any failure. Compare the requested forward/gradient/equivariance properties before optimizing. Avoid repeatedly inspecting unrelated modules or producing several competing implementations. +5. If an API differs from this guide, inspect that installed object's signature and source once. Adapt to the installed version or explain the required version; do not silently replace cuEquivariance with another library. -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure PyTorch (einsum) | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"fused_tp"` | CUDA kernel | GPU, 3- or 4-operand contractions, float32/float64 | -| `"indexed_linear"` | CUDA kernel | GPU, linear with indexed weights, sorted indices | +## Prerequisites and environment reuse -## Core primitive: SegmentedPolynomial +Run this with the project's Python interpreter (including its existing virtual environment, if any): ```python +import sys import torch import cuequivariance as cue import cuequivariance_torch as cuet -# Build a descriptor -e = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) -poly = e.polynomial - -# Create the module -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -# Forward pass -batch = 8 -x = torch.randn(batch, 3, device="cuda") -[output] = sp([x]) -# output.shape == (batch, 9) -- 1 + 3 + 5 -``` - -### Inputs, indexing, and scatter - -```python -e = cue.descriptors.channelwise_tensor_product( - 16 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -sp = cuet.SegmentedPolynomial(poly, method="uniform_1d") - -w = torch.randn(1, poly.inputs[0].size, device="cuda") # shared weights -x1 = torch.randn(batch, poly.inputs[1].size, device="cuda") # batched input 1 -x2 = torch.randn(batch, poly.inputs[2].size, device="cuda") # batched input 2 - -# Basic forward -[out] = sp([w, x1, x2]) - -# With input gathering (e.g., gather x1 by node index) -senders = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp([w, x1, x2], input_indices={1: senders}) - -# With output scattering (accumulate into target nodes) -receivers = torch.randint(0, num_nodes, (num_edges,), device="cuda") -[out] = sp( - [w, x1, x2], - input_indices={1: senders}, - output_indices={0: receivers}, - output_shapes={0: torch.empty(num_nodes, 1, device="cuda")}, -) -``` - -### Math dtype control - -```python -# Compute in float32 regardless of input dtype -sp = cuet.SegmentedPolynomial(poly, method="fused_tp", math_dtype=torch.float32) - -# For fused_tp, math_dtype must be float32 or float64 -# For naive, any torch.dtype works -# For uniform_1d, float32 or float64 (auto-selects float32 if input is e.g. float16) +print("python:", sys.executable) +print("torch:", torch.__version__, "CUDA build:", torch.version.cuda) +print("CUDA available:", torch.cuda.is_available()) +print("cue:", cue.__file__) +print("cuet:", cuet.__file__) ``` -## High-level operations +If these imports work, continue with this interpreter; CPU execution does not require CUDA kernels. When an import fails, first check whether a different project interpreter already has the dependencies. Use that same interpreter for installation and execution. -All operations are `torch.nn.Module` subclasses. They wrap `SegmentedPolynomial` and handle layout transposition automatically. - -### Memory layout - -`IrrepsLayout` controls memory order within each `(mul, ir)` block: - -- `cue.mul_ir`: data ordered as `(mul, ir.dim)` — **default, compatible with e3nn** -- `cue.ir_mul`: data ordered as `(ir.dim, mul)` — **used internally by descriptors** - -Operations accept `layout` (applies to all), or per-operand `layout_in1`, `layout_in2`, `layout_out`. - -### ChannelWiseTensorProduct - -Channel-wise tensor product: pairs channels of `x1` with channels of `x2`. - -```python -# With internal weights (default: shared_weights=True, internal_weights=True) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), # irreps_in1 - cue.Irreps("SO3", "0 + 1"), # irreps_in2 - layout=cue.mul_ir, - device="cuda", - dtype=torch.float32, -) -# tp.weight_numel -- number of weight parameters -# tp.irreps_out -- output irreps (auto-computed) - -x1 = torch.randn(batch, tp.irreps_in1.dim, device="cuda") -x2 = torch.randn(batch, tp.irreps_in2.dim, device="cuda") - -out = tp(x1, x2) # uses internal weight parameter -# out.shape == (batch, tp.irreps_out.dim) - -# With external weights (shared_weights=False) -tp = cuet.ChannelWiseTensorProduct( - cue.Irreps("SO3", "32x0 + 32x1"), - cue.Irreps("SO3", "0 + 1"), - layout=cue.mul_ir, - shared_weights=False, - device="cuda", -) -w = torch.randn(batch, tp.weight_numel, device="cuda") -out = tp(x1, x2, weight=w) - -# With gather/scatter for graph neural networks -out = tp(x1, x2, weight=w, indices_1=senders, indices_out=receivers, size_out=num_nodes) -``` - -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. - -### FullyConnectedTensorProduct - -All input irrep pairs contribute to all output irreps (dense contraction). - -```python -tp = cuet.FullyConnectedTensorProduct( - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_in1 - cue.Irreps("O3", "0e + 1o"), # irreps_in2 - cue.Irreps("O3", "4x0e + 4x1o"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) +Only when dependencies are missing and installation is appropriate, create a **new** environment. An installation or backend error does not justify deleting an existing environment, replacing the user's CPU PyTorch, or bypassing system-package protection. On Linux/macOS, this fallback also works when `venv` cannot bootstrap pip through `ensurepip` (the host needs pip 22.3+): -out = tp(x1, x2) # uses internal weights -# or: out = tp(x1, x2, weight=w) # external weights +```bash +cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") +python3 -m venv --without-pip "$cueq_env" +python3 -m pip --python "$cueq_env" install torch --index-url https://download.pytorch.org/whl/cpu +python3 -m pip --python "$cueq_env" install cuequivariance-torch +"$cueq_env/bin/python" -c 'import torch, cuequivariance_torch; print(torch.__version__)' ``` -Default method: `"fused_tp"`. +Honor project version/source constraints in these commands and keep using the new environment's Python explicitly. If pip is unavailable, use an existing manager such as `uv` or report the missing prerequisite. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). -### Linear +## Limitations and backends -Equivariant linear layer (weight-only, no second input). +| Method | Use and constraints | +| --- | --- | +| `naive` | Pure PyTorch reference implementation; supported CPU path and autograd. | +| `uniform_1d` | CUDA; uniform segment shapes within each operand and a single mode after descriptor transformations. | +| `fused_tp` | CUDA contractions with 3 or 4 operands; math dtype float32/float64. Some versions fall back on CPU, but select `naive` explicitly for a CPU reference. | +| `indexed_linear` | CUDA indexed linear operations; indices must be sorted. | -```python -linear = cuet.Linear( - cue.Irreps("SO3", "4x0 + 2x1"), # irreps_in - cue.Irreps("SO3", "3x0 + 5x1"), # irreps_out - layout=cue.mul_ir, - internal_weights=True, - device="cuda", -) - -out = linear(x) +High-level defaults vary: `SphericalHarmonics` selects `uniform_1d`, `FullyConnectedTensorProduct` selects `fused_tp`, `Linear` selects `naive`, and `ChannelWiseTensorProduct`/`SymmetricContraction` select according to descriptor compatibility. Moving tensors to CPU does not change every constructor's backend choice. CUDA acceleration requires compatible PyTorch, GPU, and `cuequivariance_ops_torch` builds; CPU checks do not establish GPU performance or export compatibility. -# Species-indexed weights (different weights per atom type) -linear = cuet.Linear( - irreps_in, irreps_out, - weight_classes=50, # 50 different weight sets - internal_weights=True, - device="cuda", -) -out = linear(x, weight_indices=species_indices) # species_indices: (batch,) int tensor -``` +## Layouts and weights -Default method: `"naive"`. Use `method="fused_tp"` for CUDA acceleration. +- `cue.mul_ir` is the high-level default, compatible with e3nn: each block is `(multiplicity, irrep_dimension)`. Core descriptors commonly use `cue.ir_mul`: `(irrep_dimension, multiplicity)`. Check each descriptor operand's `.layout`. +- Convert **each irrep block**, not the whole flattened tensor with a reshape. `cuet.TransposeIrrepsLayout(irreps, source=..., target=..., use_fallback=True)` performs the blockwise conversion using PyTorch. +- For externally supplied weights set `internal_weights=False`. Shared weights have shape `(1, module.weight_numel)`; distinct weights per item have shape `(batch, module.weight_numel)` and require `shared_weights=False`. A gathered graph operation uses the **edge count** for this batch dimension. +- `filter_irreps_out` selects allowed irrep types, not a requested output representation with prescribed multiplicities. Read `module.irreps_out`; preserve its actual order when constructing the next layer or reference. +- For training, create learned parameters or an edge-weight network in `__init__`. Do not draw new random weights inside `forward()`. -### SymmetricContraction +## Examples -MACE-style symmetric contraction with element-indexed weights. +The CPU examples below share these imports. Replace the small representations and tensors with the task's inputs; keep the layout, indexing, and gradient contracts. ```python -sc = cuet.SymmetricContraction( - cue.Irreps("O3", "32x0e + 32x1o"), # irreps_in (uniform mul required) - cue.Irreps("O3", "32x0e"), # irreps_out (uniform mul required) - contraction_degree=3, - num_elements=95, # number of chemical elements - layout=cue.ir_mul, - dtype=torch.float32, - device="cuda", -) +import numpy as np +import torch +import cuequivariance as cue +import cuequivariance_torch as cuet -# indices: (batch,) int tensor selecting which element weights to use -out = sc(x, indices) -# out.shape == (batch, irreps_out.dim) +torch.manual_seed(0) +dtype = torch.float64 ``` -Default method: `"uniform_1d"` if segments are uniform, else `"naive"`. +### Spherical harmonics and a NumPy reference -### SphericalHarmonics +`normalize=False` evaluates the raw polynomial; `normalize=True` first normalizes the input vectors. Match that choice in the reference and use nonzero vectors for gradient checks. ```python -sh = cuet.SphericalHarmonics( - ls=[0, 1, 2, 3], - normalize=True, - device="cuda", +ls = [0, 1, 2] +sh = cuet.SphericalHarmonics(ls, normalize=False, method="naive") +vectors = torch.randn(2, 3, dtype=dtype, requires_grad=True) +values = sh(vectors) +descriptor = cue.descriptors.spherical_harmonics(cue.SO3(1), ls) +np.testing.assert_allclose( + values.detach().numpy(), descriptor(vectors.detach().numpy())[0], atol=1e-12 ) - -vectors = torch.randn(batch, 3, device="cuda") -out = sh(vectors) -# out.shape == (batch, 1 + 3 + 5 + 7) -- sum of 2l+1 +assert values.shape == (2, sum(2 * l + 1 for l in ls)) +assert torch.autograd.gradcheck(sh, (vectors,)) ``` -Default method: `"uniform_1d"`. +### Linear: layout conversion and rotations -### Rotation and Inversion +Use identical explicit weights for a layout comparison. For row-vector features, apply the representation matrix as `x @ D.T`. ```python -# Rotation (SO3 or O3 irreps) -rot = cuet.Rotation( - cue.Irreps("SO3", "4x0 + 2x1 + 1x2"), - layout=cue.ir_mul, - device="cuda", +irreps_in = cue.Irreps("SO3", "3x0 + 2x1") +irreps_out = cue.Irreps("SO3", "2x0 + 4x1") +options = dict(internal_weights=False, method="naive", dtype=dtype) +linear = cuet.Linear(irreps_in, irreps_out, layout=cue.mul_ir, **options) +linear_ir = cuet.Linear(irreps_in, irreps_out, layout=cue.ir_mul, **options) +to_ir = cuet.TransposeIrrepsLayout( + irreps_in, source=cue.mul_ir, target=cue.ir_mul, use_fallback=True ) - -# Euler angles (YXY convention) -gamma = torch.tensor([0.1], device="cuda") -beta = torch.tensor([0.2], device="cuda") -alpha = torch.tensor([0.3], device="cuda") -out = rot(gamma, beta, alpha, x) - -# Helper: encode angle for rotation -encoded = cuet.encode_rotation_angle(angle, ell=3) # cos/sin encoding - -# Helper: 3D vector to Euler angles -beta, alpha = cuet.vector_to_euler_angles(vector) - -# Inversion (O3 irreps only) -inv = cuet.Inversion( - cue.Irreps("O3", "4x0e + 2x1o"), - layout=cue.ir_mul, - device="cuda", +to_mul = cuet.TransposeIrrepsLayout( + irreps_out, source=cue.ir_mul, target=cue.mul_ir, use_fallback=True ) -out = inv(x) +x = torch.randn(2, irreps_in.dim, dtype=dtype, requires_grad=True) +w = torch.randn(1, linear.weight_numel, dtype=dtype, requires_grad=True) +y = linear(x, w) +torch.testing.assert_close(y, to_mul(linear_ir(to_ir(x), w))) +rotation = np.array([0.2, -0.3, 0.1]) +din, dout = [torch.from_numpy(cue.IrrepsAndLayout(r, cue.mul_ir).exp_map(rotation, [])) + for r in (irreps_in, irreps_out)] +torch.testing.assert_close(linear(x @ din.T, w), y @ dout.T) +for g in torch.autograd.grad(y.square().sum(), (x, w)): + assert torch.isfinite(g).all() and g.abs().sum() > 0 ``` -## Layers +### Channelwise product with external per-example weights -### BatchNorm - -Batch normalization for equivariant representations (adapted from e3nn). +The core descriptor takes `(weights, input1, input2)` and returns a list of outputs. The high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids an extra transpose in this reference. ```python -bn = cuet.layers.BatchNorm( - cue.Irreps("O3", "4x0e + 4x1o"), - layout=cue.mul_ir, - eps=1e-5, - momentum=0.1, - affine=True, +a, b = cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1") +keep = [cue.SO3(0), cue.SO3(1)] +tp = cuet.ChannelWiseTensorProduct( + a, b, keep, layout=cue.ir_mul, shared_weights=False, + internal_weights=False, method="naive", dtype=dtype, ) - -out = bn(x) # x.shape == (batch, irreps.dim) +x1, x2, w = [torch.randn(2, n, dtype=dtype, requires_grad=True) + for n in (a.dim, b.dim, tp.weight_numel)] +y = tp(x1, x2, weight=w) +descriptor = cue.descriptors.channelwise_tensor_product(a, b, keep) +reference = descriptor(*(t.detach().numpy() for t in (w, x1, x2)))[0] +np.testing.assert_allclose(y.detach().numpy(), reference, atol=1e-12) +assert y.shape == (2, tp.irreps_out.dim) +for g in torch.autograd.grad(y.square().sum(), (x1, x2, w)): + assert torch.isfinite(g).all() and g.abs().sum() > 0 ``` -### FullyConnectedTensorProductConv +For a fully connected product, specify the desired output irreps in `cuet.FullyConnectedTensorProduct(a, b, irreps_out, method="naive", dtype=dtype)`. Its default weights are internal trainable parameters. Use the same external-weight flags when the caller or an edge network owns the weights. -Message passing layer for equivariant GNNs (DiffDock-style). +### SegmentedPolynomial: gather and scatter -```python -conv = cuet.layers.FullyConnectedTensorProductConv( - in_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - sh_irreps=cue.Irreps("O3", "0e + 1o"), - out_irreps=cue.Irreps("O3", "4x0e + 4x1o"), - mlp_channels=[16, 32, 32], - mlp_activation=torch.nn.ReLU(), - batch_norm=True, - layout=cue.ir_mul, -) - -# graph = ((src, dst), (num_src_nodes, num_dst_nodes)) -graph = ((src, dst), (num_src_nodes, num_dst_nodes)) - -out = conv(src_features, edge_sh, edge_emb, graph, reduce="mean") -# out.shape == (num_dst_nodes, out_irreps.dim) - -# Optional: separate scalar features for efficient first-layer GEMM -out = conv(src_features, edge_sh, edge_emb, graph, - src_scalars=src_scalars, dst_scalars=dst_scalars) -``` - -## Triangle operations (AlphaFold2-style) - -Require `cuequivariance_ops_torch`. +Input/output index keys refer to **operand positions**, not feature channels. Supply `torch.int32` or `torch.int64` indices on the computation's device. `output_shapes` takes a two-dimensional **tensor**, whose first dimension determines the destination count; it does not take a shape tuple. Repeated receivers sum, and destinations without incoming edges remain zero. ```python -# Triangle attention with pair bias -kv_lengths = cuet.mask_to_kv_lengths(prefix_mask) -out = cuet.triangle_attention(q, k, v, bias, scale=scale, kv_lengths=kv_lengths) -# Arbitrary dense masks are supported through the fallback path: -out = cuet.triangle_attention(q, k, v, bias, mask=holey_mask, scale=scale) -# q, k, v: (B, N, H, Q/K, D), bias: (B, 1, H, Q, K) - -# Triangle multiplicative update -out = cuet.triangle_multiplicative_update( - x, # (B, I, J, C) - mask=mask, # (B, I, J) - precision=cuet.TriMulPrecision.DEFAULT, -) - -# Attention with pair bias (strict OpenFold3/Boltz contract) -out, _ = cuet.attention_pair_bias( - single_repr, pair_repr, mask, num_heads, - w_ln_a, b_ln_a, - w_proj_q, b_proj_q, w_proj_k, w_proj_v, - w_proj_g, w_proj_o, w_proj_z, - w_ln_z=w_ln_z, b_ln_z=b_ln_z, -) -# Optional generalized projection params are keyword-only: -# b_proj_k, b_proj_v, w_ln_q, b_ln_q, w_ln_k, b_ln_k, b_proj_g (default None), -# and norm_kind (default "layer_norm"). +descriptor = cue.descriptors.linear(cue.Irreps("SO3", "3x0"), cue.Irreps("SO3", "2x0")) +sp = cuet.SegmentedPolynomial(descriptor.polynomial, method="naive") +nodes = torch.randn(5, 3, dtype=dtype, requires_grad=True) +w = torch.randn(1, descriptor.inputs[0].dim, dtype=dtype, requires_grad=True) +senders = torch.tensor([0, 2, 1, 0], dtype=torch.int64) +receivers = torch.tensor([1, 0, 1, 2], dtype=torch.int64) +[actual] = sp([w, nodes], input_indices={1: senders}, + output_indices={0: receivers}, output_shapes={0: nodes.new_empty(4, 1)}) +[edges] = sp([w, nodes[senders]]) +reference = nodes.new_zeros(4, descriptor.outputs[0].dim).index_add_(0, receivers, edges) +torch.testing.assert_close(actual, reference) +assert torch.count_nonzero(actual[3]) == 0 +grads = torch.autograd.grad(actual.square().sum(), (nodes, w)) +ref_grads = torch.autograd.grad(reference.square().sum(), (nodes, w)) +for g, ref in zip(grads, ref_grads): + torch.testing.assert_close(g, ref) ``` -## ONNX and TensorRT export - -```python -# ONNX export -table = cuet.onnx_custom_translation_table() -onnx_program = torch.onnx.export(model, inputs, custom_translation_table=table) +Reuse the descriptor to build a numerical reference: its coefficients include normalization factors, so a hand-written matrix product without those factors can be wrong. For high-level channelwise gathering/scattering, use `indices_1`, `indices_2`, `indices_out`, and `size_out` instead of the primitive's dictionaries. -# TensorRT plugin registration -cuet.register_tensorrt_plugins() -``` +### Trainable message passing -## Complete GNN example +This CPU reference uses a radial MLP to produce one weight vector per edge. Its parameters persist across calls, and `irreps_sh` matches the single copy of each spherical harmonic degree. ```python -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet - class SimpleGNN(torch.nn.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out): + def __init__(self, irreps_in, irreps_out): super().__init__() self.tp = cuet.ChannelWiseTensorProduct( - irreps_in, irreps_sh, layout=cue.mul_ir, - shared_weights=False, device="cuda", + irreps_in, cue.Irreps("SO3", "0 + 1 + 2"), layout=cue.mul_ir, + shared_weights=False, internal_weights=False, method="naive", ) - self.linear = cuet.Linear( - self.tp.irreps_out, irreps_out, - layout=cue.mul_ir, internal_weights=True, device="cuda", + self.radial = torch.nn.Sequential( + torch.nn.Linear(1, 16), torch.nn.SiLU(), + torch.nn.Linear(16, self.tp.weight_numel), ) - self.sh = cuet.SphericalHarmonics( - ls=[ir.l for _, ir in irreps_sh], normalize=True, device="cuda", + self.sh = cuet.SphericalHarmonics([0, 1, 2], normalize=True, method="naive") + self.linear = cuet.Linear( + self.tp.irreps_out, irreps_out, layout=cue.mul_ir, method="naive" ) - def forward(self, node_feats, edge_vec, edge_index, num_nodes): - src, dst = edge_index - edge_sh = self.sh(edge_vec) - w = torch.randn(1, self.tp.weight_numel, device=node_feats.device) - - # Message: tensor product on edges with scatter to destination nodes + def forward(self, node_features, edge_vectors, senders, receivers, num_dst): + weights = self.radial(edge_vectors.norm(dim=-1, keepdim=True)) messages = self.tp( - node_feats, edge_sh, weight=w, - indices_1=src, indices_2=None, - indices_out=dst, size_out=num_nodes, + node_features, self.sh(edge_vectors), weight=weights, + indices_1=senders, indices_out=receivers, size_out=num_dst, ) return self.linear(messages) ``` -## Key file locations - -| Component | Path | -|-----------|------| -| `SegmentedPolynomial` | `cuequivariance_torch/primitives/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_torch/primitives/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_torch/primitives/segmented_polynomial_naive.py` | -| `fused_tp` backend | `cuequivariance_torch/primitives/segmented_polynomial_fused_tp.py` | -| `indexed_linear` backend | `cuequivariance_torch/primitives/segmented_polynomial_indexed_linear.py` | -| `ChannelWiseTensorProduct` | `cuequivariance_torch/operations/tp_channel_wise.py` | -| `FullyConnectedTensorProduct` | `cuequivariance_torch/operations/tp_fully_connected.py` | -| `Linear` | `cuequivariance_torch/operations/linear.py` | -| `SymmetricContraction` | `cuequivariance_torch/operations/symmetric_contraction.py` | -| `SphericalHarmonics` | `cuequivariance_torch/operations/spherical_harmonics.py` | -| `Rotation` / `Inversion` | `cuequivariance_torch/operations/rotation.py` | -| `BatchNorm` | `cuequivariance_torch/layers/batchnorm.py` | -| `FullyConnectedTensorProductConv` | `cuequivariance_torch/layers/tp_conv_fully_connected.py` | -| Triangle operations | `cuequivariance_torch/primitives/triangle.py` | -| Layout transposition | `cuequivariance_torch/primitives/transpose.py` | +Move the whole module and inputs to the same dtype/device (for example `model.double()` for float64 checks). Use nonzero edge vectors when normalizing spherical harmonics. After a scalar loss/backward pass, check gradients on the radial network as well as node features; repeated forwards should agree when no stochastic layers are present. + +## Other operations and layers + +Inspect the requested operation's installed signature/source via `inspect.signature` and `inspect.getfile`, rather than loading the entire API. These are the non-obvious contracts: + +| Operation | Contract | +| --- | --- | +| Species-indexed `Linear` | Set `weight_classes`; pass `weight_indices` of shape `(batch,)` and weights of shape `(weight_classes, weight_numel)`. | +| `SymmetricContraction` | MACE-style; uniform multiplicities, `contraction_degree`, `num_elements`, and integer element indices in `sc(x, indices)`. | +| `Rotation` / `Inversion` | `rot(gamma, beta, alpha, x)` uses YXY Euler angles; `inv(x)` uses O3 parity. Specify layout. Angle helpers: `encode_rotation_angle`, `vector_to_euler_angles`. | +| `layers.BatchNorm` | Equivariant normalization; configure irreps and layout. | +| `layers.FullyConnectedTensorProductConv` | Configure input/SH/output irreps and edge MLP; call with `(src_features, edge_sh, edge_emb, ((src, dst), (num_src, num_dst)))`. Supports `reduce="mean"` and optional `src_scalars`/`dst_scalars`. | + +### Triangle operations and export + +These paths require compatible optional GPU packages and their own shape checks. Do not install them for a CPU tensor-product request. Locate the installed `primitives/triangle.py` via `cuet.__file__`: + +- `triangle_attention`: Q/K/V `(B,N,H,Q_or_K,D)`, bias `(B,1,H,Q,K)`. `mask_to_kv_lengths(prefix_mask)` supplies `kv_lengths`; arbitrary dense masks use `mask=...` and the fallback. +- `triangle_multiplicative_update`: input `(B,I,J,C)`, mask `(B,I,J)`, precision enum `cuet.TriMulPrecision`. +- `attention_pair_bias`: OpenFold3/Boltz contract, returns `(out, auxiliary)`; projection biases and query/key normalization keywords vary by version. Inspect its signature before assembling the weights. +- ONNX: pass `custom_translation_table=cuet.onnx_custom_translation_table()` to `torch.onnx.export`. TensorRT: `cuet.register_tensorrt_plugins()`. Validate the actual exported runtime. + +## Troubleshooting + +| Symptom | Check and recovery | +| --- | --- | +| Missing CUDA kernels on CPU | Pass `method="naive"` at module construction; retain the installed CPU PyTorch. | +| `opt_einsum_fx not available` warning | Optional optimization; continue the naive reference calculation without installing it. | +| Missing import after installation | Compare `sys.executable`, module paths, and the interpreter used with pip. Reuse the working interpreter. | +| `ensurepip` unavailable / externally managed Python | Use a fresh environment and host `pip --python` as above; preserve existing environments and system packages. | +| Unexpected keyword or missing descriptor method | Inspect the installed signature/version. Use its documented equivalent or explain the mismatch; avoid repeated blind API guesses. | +| Correct shapes but wrong numerical values | Check per-block layout, output irrep order, normalization, and the shared/per-item weight convention. | +| Scatter shape/error or wrong destination sums | Use operand index keys, integral indices, a tensor in `output_shapes`, and accumulation for repeated receivers. | +| Missing/nonfinite gradients | Keep tensor operations in autograd; detach only the NumPy reference. Use nonzero vectors and a nonconstant loss; check every requested differentiable operand. | From 72b0b7384fa8a9100a1a82326ac75a1ac5db8853 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Fri, 16 Oct 2026 22:22:17 +0000 Subject: [PATCH 04/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 48 ++++++++-------- skills/cuequivariance-jax/skill-card.md | 10 ++-- skills/cuequivariance-jax/skill.oms.sig | 2 +- skills/cuequivariance-torch/BENCHMARK.md | 69 +++++++++++++---------- skills/cuequivariance-torch/skill-card.md | 25 ++++---- skills/cuequivariance-torch/skill.oms.sig | 2 +- skills/cuequivariance/BENCHMARK.md | 50 ++++++++-------- skills/cuequivariance/skill-card.md | 14 ++--- skills/cuequivariance/skill.oms.sig | 2 +- 9 files changed, 113 insertions(+), 109 deletions(-) diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md index 01f603b0..bb9ff66a 100644 --- a/skills/cuequivariance-jax/BENCHMARK.md +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -1,10 +1,8 @@ # Skill Benchmark: cuequivariance-jax -> ✅ **Overall verdict: PASS — Recommended for publication** +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. ## Evaluation Metadata @@ -35,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 50.0% (+16.7 points) | 100.0% → 100.0% (±0.0 points) | +| Overall | 84.8% — baseline ran, but no comparable score was available; uplift unavailable | 78.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 66.7% → 66.7% (±0.0 points) | 66.7% → 50.0% (-16.7 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 100.0% (+2.1 points) | 90.8% → 87.9% (-2.9 points) | -| Efficiency | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | 63.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 96.3% → 100.0% (+3.7 points) | 90.4% → 92.9% (+2.5 points) | +| Efficiency | 74.2% — baseline ran, but no comparable score was available; uplift unavailable | 62.2% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 6,770,880 | 5,344,900 | +1,425,980 | +26.68% | skill 6/6; base 6/6 | -| claude-code | jax-001 | 1,092,116 | 1,209,345 | -117,229 | -9.69% | skill 1/1; base 1/1 | -| claude-code | jax-002 | 1,669,333 | 476,301 | +1,193,032 | +250.48% | skill 1/1; base 1/1 | -| claude-code | jax-003 | 745,879 | 887,128 | -141,249 | -15.92% | skill 1/1; base 1/1 | -| claude-code | jax-004 | 2,637,995 | 2,282,069 | +355,926 | +15.60% | skill 1/1; base 1/1 | -| claude-code | jax-005 | 596,440 | 460,588 | +135,852 | +29.50% | skill 1/1; base 1/1 | -| claude-code | jax-006 | 29,117 | 29,469 | -352 | -1.19% | skill 1/1; base 1/1 | -| codex | All cases | 3,927,148 | 2,136,742 | +1,790,406 | +83.79% | skill 6/6; base 6/6 | -| codex | jax-001 | 793,152 | 614,925 | +178,227 | +28.98% | skill 1/1; base 1/1 | -| codex | jax-002 | 304,987 | 323,916 | -18,929 | -5.84% | skill 1/1; base 1/1 | -| codex | jax-003 | 1,028,052 | 595,600 | +432,452 | +72.61% | skill 1/1; base 1/1 | -| codex | jax-004 | 1,727,473 | 556,340 | +1,171,133 | +210.51% | skill 1/1; base 1/1 | -| codex | jax-005 | 60,043 | 32,627 | +27,416 | +84.03% | skill 1/1; base 1/1 | -| codex | jax-006 | 13,441 | 13,334 | +107 | +0.80% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 10,698,028 | 7,481,642 | +3,216,386 | +42.99% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,008,776 | 4,893,147 | -884,371 | -18.07% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 819,414 | 1,039,684 | -220,270 | -21.19% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 644,824 | 519,762 | +125,062 | +24.06% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 803,688 | 837,521 | -33,833 | -4.04% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 1,165,511 | 1,871,827 | -706,316 | -37.73% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 545,612 | 594,753 | -49,141 | -8.26% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 29,727 | 29,600 | +127 | +0.43% | skill 1/1; base 1/1 | +| codex | All cases | 3,101,982 | 2,979,561 | +122,421 | +4.11% | skill 6/6; base 6/6 | +| codex | jax-001 | 1,059,589 | 862,896 | +196,693 | +22.79% | skill 1/1; base 1/1 | +| codex | jax-002 | 253,060 | 359,521 | -106,461 | -29.61% | skill 1/1; base 1/1 | +| codex | jax-003 | 645,104 | 513,418 | +131,686 | +25.65% | skill 1/1; base 1/1 | +| codex | jax-004 | 990,442 | 1,141,417 | -150,975 | -13.23% | skill 1/1; base 1/1 | +| codex | jax-005 | 140,349 | 88,989 | +51,360 | +57.71% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,438 | 13,320 | +118 | +0.89% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 7,110,758 | 7,872,708 | -761,950 | -9.68% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -78,7 +76,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | ## Findings and Observations diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md index 616fc853..6dc17b59 100644 --- a/skills/cuequivariance-jax/skill-card.md +++ b/skills/cuequivariance-jax/skill-card.md @@ -66,12 +66,12 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 50.0% (+16.7 points) | 100.0% → 100.0% (±0.0 points) | +| Overall | 84.8% — baseline ran, but no comparable score was available; uplift unavailable | 78.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 66.7% → 66.7% (±0.0 points) | 66.7% → 50.0% (-16.7 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 100.0% (+2.1 points) | 90.8% → 87.9% (-2.9 points) | -| Efficiency | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | 63.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 96.3% → 100.0% (+3.7 points) | 90.4% → 92.9% (+2.5 points) | +| Efficiency | 74.2% — baseline ran, but no comparable score was available; uplift unavailable | 62.2% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig index e69cc951..444901c4 100644 --- a/skills/cuequivariance-jax/skill.oms.sig +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjliMjA0M2U4M2RkZDNlYmQwYzFhNTI0ODlmMmI1Zjc2MWMzZjgzNmJhMDk4MjhmOTVlMDM5MmM0MGRjYjY0NDUiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImQyODY3Y2ZkZjc1ZTA2MzFiOTMwYTZiZjY3Y2NiODQ4ZjJjYzY4MDM4YjZkZjlmMzZhNjNhNWI0MDM4MTAwZDQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiZmE3MWJkYjgzYTlkZGI4MWE4OWQ3ODlhOTk3ZGRiYzdlYTk3NmNlNzY1OTYwYmJjMzVjNTBjZDUyY2EyNTYwMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YjY1ZjFmODVjZGY1ZGQ2NDkyMjJkOGEzNWQ4ZmY4Mzg2N2Y3MWU3NDUzOWQxNGM5Y2JiZWI4MWVlNDQ5OTg1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZTFkN2JkM2YzY2JiOTdjMzljZWY1ZWU3ZGQyODFmMTVkMmE1YWRhMjQ0ZDViNTQzMDc2NjIyNDEyMGY2ZGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiOGQ5Mjg4YzExOTExNDlkODk4ODBhYzhkMGU3OTVmYThkZjFiZmE5MzM0NjYzMzZlZjNjZWM3NTFiMzBkMjE0YSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEp0qcsQ+nfR/fEx1/MlAYk7r+kfMuSbZUzBrUM2aqko7jdkPrVf2bcGftAzm6GmRQIwaII+wQG2ppFMVZEyAyAqfXiCrl6F4V5L/2+CRlze6RWflpSkXyye7y50iGnm7mch","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjBhNzlmMjg4NDRkYzgyNDU3OTllNzY1ZGY5YTU2OGI5NDg3ZmNkOWRlOWM0MTNjNGJlM2U3OGZhZWMwNjEwZjAiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZjI1MmJmOTliZmMxM2MzZDE3YWYzZjUwNzc3YjdmODJmNTczYWY0ZmZmODliN2VjYTQzMzdhY2FiMTc1YjczIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZhNzFiZGI4M2E5ZGRiODFhODlkNzg5YTk5N2RkYmM3ZWE5NzZjZTc2NTk2MGJiYzM1YzUwY2Q1MmNhMjU2MDAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZmUxZDdiZDNmM2NiYjk3YzM5Y2VmNWVlN2RkMjgxZjE1ZDJhNWFkYTI0NGQ1YjU0MzA3NjYyMjQxMjBmNmRlNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5NDQ5MmE0ZDkwNTJmYjk5MDg2NGZlOGI4MTg4OTIzYWY5ZTMxYmYxMDc4YzQyOTdhMGIxZGIxNmRmZjliNjEiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCe/a0d2R+ibkK76aoY0mSiFiAXmnTnzpKt+YdyIp+arFHgEmqDUvEGU0rTAzaP/JQCMQCpJXOb3wkLxi9wadI0cJ2fF4A6s8Zyfn//qYMI5LqPrvPHLR4LDcQe6kVWRiaolp0=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md index eb34d33c..cecfd6bc 100644 --- a/skills/cuequivariance-torch/BENCHMARK.md +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -1,8 +1,10 @@ # Skill Benchmark: cuequivariance-torch -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata @@ -33,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Security | 50.0% → 50.0% (±0.0 points) | Not available | -| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | -| Discoverability | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Effectiveness | 90.1% → 100.0% (+9.9 points) | Not available | -| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Overall | 86.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 83.3% (+33.3 points) | 50.0% → 50.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 90.1% → 84.2% (-5.9 points) | 81.8% → 91.3% (+9.5 points) | +| Efficiency | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | 58.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 7,423,526 | 6,490,723 | +932,803 | +14.37% | skill 6/6; base 6/6 | -| claude-code | torch-001 | 1,379,936 | 1,308,078 | +71,858 | +5.49% | skill 1/1; base 1/1 | -| claude-code | torch-002 | 1,182,955 | 603,673 | +579,282 | +95.96% | skill 1/1; base 1/1 | -| claude-code | torch-003 | 1,161,987 | 928,849 | +233,138 | +25.10% | skill 1/1; base 1/1 | -| claude-code | torch-004 | 1,286,524 | 1,044,323 | +242,201 | +23.19% | skill 1/1; base 1/1 | -| claude-code | torch-005 | 2,382,097 | 2,576,008 | -193,911 | -7.53% | skill 1/1; base 1/1 | -| claude-code | torch-006 | 30,027 | 29,792 | +235 | +0.79% | skill 1/1; base 1/1 | -| codex | All cases | 5,264,185 | 4,068,564 | N/A | N/A | skill 5/6; base 6/6 | -| codex | torch-001 | N/A | 1,321,564 | N/A | N/A | skill 0/1; base 1/1 | -| codex | torch-002 | 1,651,497 | 768,903 | +882,594 | +114.79% | skill 1/1; base 1/1 | -| codex | torch-003 | 1,103,907 | 689,022 | +414,885 | +60.21% | skill 1/1; base 1/1 | -| codex | torch-004 | 520,123 | 425,385 | +94,738 | +22.27% | skill 1/1; base 1/1 | -| codex | torch-005 | 1,974,911 | 850,079 | +1,124,832 | +132.32% | skill 1/1; base 1/1 | -| codex | torch-006 | 13,747 | 13,611 | +136 | +1.00% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 12,687,711 | 10,559,287 | N/A | N/A | skill 11/12; base 12/12 | +| claude-code | All cases | 4,604,027 | 5,037,482 | -433,455 | -8.60% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 1,230,733 | 1,103,162 | +127,571 | +11.56% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 543,013 | 848,884 | -305,871 | -36.03% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 780,753 | 978,259 | -197,506 | -20.19% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 686,592 | 1,355,474 | -668,882 | -49.35% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 1,333,100 | 721,931 | +611,169 | +84.66% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 29,836 | 29,772 | +64 | +0.21% | skill 1/1; base 1/1 | +| codex | All cases | 5,122,185 | 7,095,117 | -1,972,932 | -27.81% | skill 6/6; base 6/6 | +| codex | torch-001 | 1,462,137 | 2,800,219 | -1,338,082 | -47.78% | skill 1/1; base 1/1 | +| codex | torch-002 | 1,022,997 | 564,020 | +458,977 | +81.38% | skill 1/1; base 1/1 | +| codex | torch-003 | 761,771 | 1,198,690 | -436,919 | -36.45% | skill 1/1; base 1/1 | +| codex | torch-004 | 680,450 | 1,134,224 | -453,774 | -40.01% | skill 1/1; base 1/1 | +| codex | torch-005 | 1,181,156 | 1,384,401 | -203,245 | -14.68% | skill 1/1; base 1/1 | +| codex | torch-006 | 13,674 | 13,563 | +111 | +0.82% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 9,726,212 | 12,132,599 | -2,406,387 | -19.83% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -74,21 +76,26 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 8 finding(s) | +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations
Show detailed findings and successful checks -- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance-torch/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: Description very long (363 chars, recommend 50-150) (`skills/cuequivariance-torch/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance-torch/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No mention of error handling or validation (`skills/cuequivariance-torch/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance-torch/SKILL.md`) -- 3 additional finding(s) are available in the full evaluation artifacts. +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 2 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code
diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md index b36976e2..f0b6b7e5 100644 --- a/skills/cuequivariance-torch/skill-card.md +++ b/skills/cuequivariance-torch/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Execute equivariant tensor products in PyTorch using SegmentedPolynomial (naive/uniform_1d/fused_tp/indexed_linear), high-level operations (ChannelWiseTensorProduct, FullyConnectedTensorProduct, Linear, SymmetricContraction, SphericalHarmonics, Rotation), and layers (BatchNorm, FullyConnectedTensorProductConv).
+Use when building or debugging cuEquivariance PyTorch tensor products or equivariant GNNs, especially CPU references, layouts, indexing, and gradients.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and researchers building equivariant geometric neural networks in PyTorch use this skill to write and validate cuequivariance_torch code, including SegmentedPolynomial execution backends, high-level equivariant operations and layers, triangle operations, and ONNX/TensorRT export, on CPU or GPU.
+Developers and researchers building equivariant geometric neural networks in PyTorch use this skill to write, debug, and validate cuequivariance_torch code, including CPU reference calculations, backend selection, irrep layouts and weights, gather/scatter indexing, gradient checks, and trainable message-passing layers.
### Deployment Geography for Use:
Global
@@ -25,13 +25,14 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [pip interpreter selection (--python option)](https://pip.pypa.io/en/stable/topics/python-option/)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python code blocks]
+**Output Type(s):** [Code, Shell commands, Analysis]
+**Output Format:** [Markdown with inline Python and bash code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -66,15 +67,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Security | 50.0% → 50.0% (±0.0 points) | Not available | -| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | -| Discoverability | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Effectiveness | 90.1% → 100.0% (+9.9 points) | Not available | -| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Overall | 86.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 83.3% (+33.3 points) | 50.0% → 50.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 90.1% → 84.2% (-5.9 points) | 81.8% → 91.3% (+9.5 points) | +| Efficiency | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | 58.0% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
+0.13.0 (source: VERSION file, git commit a126d11 2026-10-16)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig index b2d22eb3..fbad218b 100644 --- a/skills/cuequivariance-torch/skill.oms.sig +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNDczOTRmMWYzYjczYzRjZTFhMDNmNmZhNDU5YjE2OGZlYmU2ZWY5ZWQyMGNlNzM4MzI0YjI1MjYxZDczZGU5MSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTkyM2ZjMjhjYWEwNjIyZjgxZTVhNjEyNjBhZmY0NDc5ODQzNGQzZGRmMWFlNTliNGU2MzJhNjZiODVlNDhmOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5N2RlNTE5MDRkNGFmYTNlMmM5ZDIxYzE3NWJhYWRhZjYxYmNiYTZkNDVjNDgxYjlmNzkxMGQyNzEzMmM3NmEyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogImE5NjliMzAxMTgzZDBmMDJiNzI4ZGY5YzVlYzY5ZGVkNjEyMDk2YzgyMzQzODVhZjcxYzg4YzBjYmMyMjYzNDAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIxODQyYTI3YzBkYTRjNmU1MjQzMTk0NDZhYmJiZGIyNDE1YzQ3N2MzMDI3MzI5OGU4MDBjMTFiNzZjZjM2NjFmIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEJHGYkDyjFJwQprikuv5nCbQp6maPBQkswheYxv1TthPTlwQoNCpmnEGJQiRn+dPgIwfW03qgsYdOS9bwuhuz+5TC4PklmcUw6vml5jg94MsMmYpdDofxP3KEWu9PTk5T+p","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZmY2NTQ3YTc0NzE5ZDIyN2MxZWMwYWZhMzA0MDZkOWQ5MTFjOWM1OGU0NjI2Y2EzMDkyN2JiNzdkMWJlYzRkNiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjk2NjJmMjFlYWUyMzRmNTIxNTkzZGU3YjU1NTYyMTkxZTA0NTdkNTgzNTBiM2E1YzVhNjhhYTQxYmIxOTA3N2YiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQ2YTRlOWM2Y2ExN2M3NGJjMWJkMmYzMmQ1MGZmYjlkNDQ0MTY1M2M2ZWNjMWQ0NTNmNjljZTQ0OGY2N2Y2OWQiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4NGNiODQ5MDgwOGE4NjUxNzlmMDBkZDc5M2Y0ZTI0OWY5MTVjYTIyYzY5YzVhYzE3ODRiZDA1ODEzOWJhZGVkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCAoci2cjG5CRptLA73JrqC3qhnuSlirfHbm4HaNG9fI3NbJKhqIYHkVJ1/IxmocmMCMQCF9i6X1sGirTFFErk8eSUd/6XROLthuYK/ivvBRwWjIcY7dxJnpmuT8Rwwiy/icW4=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md index 2549d4fd..e7813c04 100644 --- a/skills/cuequivariance/BENCHMARK.md +++ b/skills/cuequivariance/BENCHMARK.md @@ -1,10 +1,8 @@ # Skill Benchmark: cuequivariance -> ✅ **Overall verdict: PASS — Recommended for publication** +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. ## Evaluation Metadata @@ -35,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 83.3% (+50.0 points) | 66.7% → 83.3% (+16.6 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | -| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.5% → 96.7% (-0.8 points) | 90.0% → 94.2% (+4.2 points) | -| Efficiency | 68.2% — baseline ran, but no comparable score was available; uplift unavailable | 72.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 82.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 75.0% (-25.0 points) | 83.3% → 83.3% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.4% → 95.8% (+0.4 points) | 88.8% → 80.0% (-8.8 points) | +| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | 71.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 6,178,511 | 6,967,056 | -788,545 | -11.32% | skill 6/6; base 6/6 | -| claude-code | core-001 | 1,116,183 | 1,092,606 | +23,577 | +2.16% | skill 1/1; base 1/1 | -| claude-code | core-002 | 590,901 | 1,036,330 | -445,429 | -42.98% | skill 1/1; base 1/1 | -| claude-code | core-003 | 729,662 | 1,931,071 | -1,201,409 | -62.21% | skill 1/1; base 1/1 | -| claude-code | core-004 | 994,549 | 779,822 | +214,727 | +27.54% | skill 1/1; base 1/1 | -| claude-code | core-005 | 2,716,801 | 2,096,799 | +620,002 | +29.57% | skill 1/1; base 1/1 | -| claude-code | core-006 | 30,415 | 30,428 | -13 | -0.04% | skill 1/1; base 1/1 | -| codex | All cases | 2,256,123 | 2,801,003 | -544,880 | -19.45% | skill 6/6; base 6/6 | -| codex | core-001 | 556,778 | 590,845 | -34,067 | -5.77% | skill 1/1; base 1/1 | -| codex | core-002 | 425,769 | 637,619 | -211,850 | -33.23% | skill 1/1; base 1/1 | -| codex | core-003 | 657,989 | 621,907 | +36,082 | +5.80% | skill 1/1; base 1/1 | -| codex | core-004 | 552,745 | 453,685 | +99,060 | +21.83% | skill 1/1; base 1/1 | -| codex | core-005 | 49,004 | 467,286 | -418,282 | -89.51% | skill 1/1; base 1/1 | -| codex | core-006 | 13,838 | 29,661 | -15,823 | -53.35% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 8,434,634 | 9,768,059 | -1,333,425 | -13.65% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,494,094 | 6,764,067 | -2,269,973 | -33.56% | skill 6/6; base 6/6 | +| claude-code | core-001 | 935,725 | 1,050,205 | -114,480 | -10.90% | skill 1/1; base 1/1 | +| claude-code | core-002 | 558,016 | 1,212,941 | -654,925 | -53.99% | skill 1/1; base 1/1 | +| claude-code | core-003 | 924,549 | 1,870,785 | -946,236 | -50.58% | skill 1/1; base 1/1 | +| claude-code | core-004 | 717,396 | 1,190,142 | -472,746 | -39.72% | skill 1/1; base 1/1 | +| claude-code | core-005 | 1,327,938 | 1,409,548 | -81,610 | -5.79% | skill 1/1; base 1/1 | +| claude-code | core-006 | 30,470 | 30,446 | +24 | +0.08% | skill 1/1; base 1/1 | +| codex | All cases | 2,407,936 | 3,244,738 | -836,802 | -25.79% | skill 6/6; base 6/6 | +| codex | core-001 | 472,531 | 660,932 | -188,401 | -28.51% | skill 1/1; base 1/1 | +| codex | core-002 | 626,920 | 924,625 | -297,705 | -32.20% | skill 1/1; base 1/1 | +| codex | core-003 | 561,064 | 742,408 | -181,344 | -24.43% | skill 1/1; base 1/1 | +| codex | core-004 | 644,768 | 817,679 | -172,911 | -21.15% | skill 1/1; base 1/1 | +| codex | core-005 | 88,355 | 85,281 | +3,074 | +3.60% | skill 1/1; base 1/1 | +| codex | core-006 | 14,298 | 13,813 | +485 | +3.51% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 6,902,030 | 10,008,805 | -3,106,775 | -31.04% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -78,7 +76,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | ## Findings and Observations diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md index a4314d24..fec8335d 100644 --- a/skills/cuequivariance/skill-card.md +++ b/skills/cuequivariance/skill-card.md @@ -66,15 +66,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 83.3% (+50.0 points) | 66.7% → 83.3% (+16.6 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | -| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.5% → 96.7% (-0.8 points) | 90.0% → 94.2% (+4.2 points) | -| Efficiency | 68.2% — baseline ran, but no comparable score was available; uplift unavailable | 72.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 82.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 75.0% (-25.0 points) | 83.3% → 83.3% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.4% → 95.8% (+0.4 points) | 88.8% → 80.0% (-8.8 points) | +| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | 71.3% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit 5e144b4 2026-10-16)
+0.13.0 (source: VERSION file, git commit a126d11 2026-10-16)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig index 826d4502..b3e3e149 100644 --- a/skills/cuequivariance/skill.oms.sig +++ b/skills/cuequivariance/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNDcxMjgzNTdiZWYxMmQ0ZmUwYmU1MDEyMjA0ZWJmZjI3ZGJhY2M4ZjFiODM5ZmNhNjBhMDU0OTVlOWExY2VhMiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMWE4Y2E4MTMyNGM5NTZlNjM3NjFiM2UwNGJmOTNhOWJkZGIzMmEyMmYyZWNmNzExODE5ZTE4YjlhOTQzYjQ0NyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYmEyYzg3MjVmNDk5MGVhYmRhNGEwZDIwZDVhYjkxOTRkMmEyZTZmOWE2NzdhOGNiNzFlOTgyNDFkMTdmZjRiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjYxOGFkMWMxYTg5ZjE1ZGIxZGIxMTNiMGI5YmY0NDcxMzMwMTEyZTUzOWIxMGZkOGEyNGE3YzQ0ZDI5YjcwMDkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmNThlYjA5ZTBhN2Q5OGJmZDMxZjUwODMxMGE1YzJmM2Y0ZWYwMTE1NDFkNDY3ZmRlOWZkYjU3ZjdjODM4MDdkIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGPNLwQ70vkIpgNWgKFMlKTGVFJiuY+RCSFNs1y7opxKTfiBKn/6ZdD2kOyfgKxa1AIwBScy//PeN4xrjNEvJbP3kkvyicKp7QYIp0a1ra3lIyrSj314C8uJ/B9WfUYtU3d+","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNGY4OTQ5Mjg3YzU5OGEwNTQ1YWNhZDJiZTEwMTRhOThmYzc5NWQ0YjFlYjdhYTIzZWYxZmRiZDE1ZDg0NzgyMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImE3M2Q0MDY3M2Y2YmE5OWUwNWFmMjUwYzMyNzE5YjllYjdiYmUwNTRhZjkwODY1NjVhZWJiMmZiNWM2ZThiMzUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2JhMmM4NzI1ZjQ5OTBlYWJkYTRhMGQyMGQ1YWI5MTk0ZDJhMmU2ZjlhNjc3YThjYjcxZTk4MjQxZDE3ZmY0YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJiOWNlNmY0NzU0YzFlN2FjMmQ1NDczYzIzMDhkYjc5NjNlOTEyOGIzNTUyNjYxZmEyMTQ1NDExMmQwMTU5ZWUzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMGUxZGZkYWRjOWRiZjE2MWY1MTBlMTdkY2VjNTIxNDJhM2NiNDFmMzYwNmY0MzVlZDdmY2ZmZjM5MTMyYzNhZiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMHWeZCI6Ch5LTSIuRPIAfBqMbSh8K2GDwwNKcXPYg3kpnM3R8AJUpI/YU1eo9RX/PgIwBlKBiPZ42Zj0L7oD3rr5HYgWs0vGzF7Tusz/RVB51XytmkEIM7hK2F2taCcNcgYk","keyid":""}]}} \ No newline at end of file From 531da35b16af756dc68737cc0353373538d073e5 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Sun, 18 Oct 2026 12:01:22 -0700 Subject: [PATCH 05/19] Improve JAX and Torch skill guidance and add example regression tests Signed-off-by: Ohad Mosafi --- .github/tests/test_jax_skill_examples.py | 121 +++++ .github/tests/test_torch_skill_examples.py | 51 +- .github/workflows/tests.yml | 2 +- skills/README.md | 53 +- skills/cuequivariance-jax/SKILL.md | 588 ++++++++------------- skills/cuequivariance-torch/SKILL.md | 36 +- 6 files changed, 470 insertions(+), 381 deletions(-) create mode 100644 .github/tests/test_jax_skill_examples.py diff --git a/.github/tests/test_jax_skill_examples.py b/.github/tests/test_jax_skill_examples.py new file mode 100644 index 00000000..a955c523 --- /dev/null +++ b/.github/tests/test_jax_skill_examples.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Execute the published CPU recipes and exercise their layout helpers.""" + +import importlib +import re +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +import cuequivariance as cue +import cuequivariance_jax as cuex +from cuequivariance_jax import ir_dict + +SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-jax/SKILL.md" +EXAMPLES = [ + compile(code, f"{SKILL}:example-{index}", "exec") + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", SKILL.read_text(), re.S) + ) +] + + +def run_example(index): + namespace = {} + exec(EXAMPLES[1], namespace) # Common imports and precision configuration. + exec(EXAMPLES[index], namespace) + return namespace + + +@pytest.mark.parametrize("index", range(2, len(EXAMPLES))) +def test_published_cpu_example(index): + # Run each example independently: accidental dependencies on earlier local + # variables must fail here, just as they would for someone copying a recipe. + run_example(index) + + +@pytest.mark.parametrize( + "failed_module,error", + [ + ("jax", ImportError("JAX cannot be imported")), + ("cuequivariance", OSError("shared library cannot be loaded")), + ("cuequivariance_jax", ModuleNotFoundError("frontend is absent")), + ], +) +def test_environment_probe_preserves_partial_results(monkeypatch, failed_module, error): + modules = { + name: importlib.import_module(name) + for name in ("jax", "cuequivariance", "cuequivariance_jax") + } + + def import_module(name): + if name == failed_module: + raise error + return modules[name] + + monkeypatch.setattr(importlib, "import_module", import_module) + namespace = {} + exec(EXAMPLES[0], namespace) + assert namespace["available"] == { + name: module for name, module in modules.items() if name != failed_module + } + assert namespace["import_errors"] == { + failed_module: f"{type(error).__name__}: {error}" + } + + +def test_layout_helper_with_duplicate_irreps_and_multiple_batch_axes(): + to_ir_mul = run_example(3)["dict_to_ir_mul_flat"] + irreps = cue.Irreps("SO3", "2x1 + 0 + 1 + 2x2") + flat = jnp.arange(2 * 3 * irreps.dim).reshape(2, 3, irreps.dim) + regrouped = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() + blocks = ir_dict.flat_to_dict(regrouped.irreps, regrouped.array, layout="ir_mul") + # Combine separated l=1 blocks along the channel axis, keeping every value. + expected_vectors = np.concatenate( + [ + np.asarray(flat[..., :6]).reshape(2, 3, 3, 2), + np.asarray(flat[..., 7:10]).reshape(2, 3, 3, 1), + ], + axis=-1, + ).swapaxes(-1, -2) + np.testing.assert_array_equal(blocks[cue.SO3(1)], expected_vectors) + np.testing.assert_array_equal(to_ir_mul(regrouped.irreps, blocks), regrouped.array) + + +@pytest.mark.parametrize("batch_shapes", [((), (), ()), ((), (2, 1, 1), (1, 3, 2))]) +def test_split_recipe_with_parity_higher_degrees_and_jit(batch_shapes): + split_naive = run_example(5)["split_naive"] + desc = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("O3", "2x0e + 2x1o + 2x2e"), + cue.Irreps("O3", "0e + 1o + 2e"), + [cue.O3(degree, (-1) ** degree) for degree in range(4)], + ) + rng = np.random.default_rng(41) + groups = [ + {ir: jnp.asarray(rng.normal(size=batch + (mul, ir.dim))) for mul, ir in irreps} + for irreps, batch in zip(desc.input_irreps, batch_shapes) + ] + operands = [ + np.asarray(group[ir]).swapaxes(-1, -2).reshape(batch + (mul * ir.dim,)) + for group, irreps, batch in zip(groups, desc.input_irreps, batch_shapes) + for mul, ir in irreps + ] + expected = desc.polynomial(*operands) + batch = np.broadcast_shapes(*batch_shapes) + for compute in ( + lambda g: split_naive(desc, g), + jax.jit(lambda g: split_naive(desc, g)), + ): + (result,) = compute(groups) + assert set(result) == {ir for _, ir in desc.output_irreps[0]} + for (mul, ir), reference in zip(desc.output_irreps[0], expected): + assert result[ir].shape == batch + (mul, ir.dim) + np.testing.assert_allclose( + result[ir].swapaxes(-1, -2).reshape(batch + (mul * ir.dim,)), + reference, + rtol=1e-12, + atol=1e-12, + ) diff --git a/.github/tests/test_torch_skill_examples.py b/.github/tests/test_torch_skill_examples.py index 2b80c869..cdee2449 100644 --- a/.github/tests/test_torch_skill_examples.py +++ b/.github/tests/test_torch_skill_examples.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 """Run the published skill examples and check the trainable GNN's invariants.""" +import importlib import re from pathlib import Path @@ -12,16 +13,56 @@ @pytest.fixture(scope="module") -def skill_namespace(): +def skill_examples(): skill = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" + return [ + compile(code, f"{skill}:example-{index}", "exec") + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", skill.read_text(), re.S) + ) + ] + + +@pytest.fixture(scope="module") +def skill_namespace(skill_examples): namespace = {} - for index, code in enumerate( - re.findall(r"```python\n(.*?)\n```", skill.read_text(), re.S) - ): - exec(compile(code, f"{skill}:example-{index}", "exec"), namespace) + for code in skill_examples: + exec(code, namespace) return namespace +@pytest.mark.parametrize( + "failed_module, error", + [ + ("cuequivariance_torch", ModuleNotFoundError("cuEquivariance is absent")), + ("cuequivariance", OSError("shared library cannot be loaded")), + ("torch", ImportError("PyTorch cannot be imported")), + ], +) +def test_environment_probe_preserves_partial_results( + skill_examples, monkeypatch, failed_module, error +): + modules = { + name: importlib.import_module(name) + for name in ("torch", "cuequivariance", "cuequivariance_torch") + } + + def import_module(name): + if name == failed_module: + raise error + return modules[name] + + monkeypatch.setattr(importlib, "import_module", import_module) + namespace = {} + exec(skill_examples[0], namespace) + assert namespace["available"] == { + name: module for name, module in modules.items() if name != failed_module + } + assert namespace["import_errors"] == { + failed_module: f"{type(error).__name__}: {error}" + } + + def test_documented_gnn_is_trainable_and_equivariant(skill_namespace): torch.manual_seed(41) irreps_in = cue.Irreps("SO3", "2x0+2x1") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8aa66551..5585972c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,7 +72,7 @@ jobs: env: JAX_PLATFORMS: cpu run: | - pytest -q .github/tests/test_jax_skill_evals.py + pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py uv pip install hatchling python .github/scripts/check_skill_packaging.py --package cuequivariance_jax diff --git a/skills/README.md b/skills/README.md index fbc82f7e..f22e63a6 100644 --- a/skills/README.md +++ b/skills/README.md @@ -41,23 +41,34 @@ frontends rather than reusing the core skill's expected routing. | PyTorch | Spherical harmonics and gradcheck; linear layouts and rotation; external per-example weights; gather/scatter with repeated receivers and gradients; CPU backend fallback | `evals/config.yml` requests an isolated skill workspace and a rebuilt Harbor -environment. SkillEvaluator installs `scripts/requirements.txt` when generating -that environment. Those requirements pin cuEquivariance to the immutable source -revision whose APIs were checked, and pin the JAX/Flax or CPU PyTorch versions. +environment. In SkillEvaluator 1.5.6, Docker task generation installs +`scripts/requirements.txt`. The `k8s-sandbox` path instead stages skill files into +an existing sandbox; `base_image_mode: rebuild` alone does **not** install these +dependencies there. Verify the actual interpreter and package versions in each +arm before treating a run as an evaluation of the pinned environment. A shared +custom evaluation image or common `harbor.pre_agent_setup` is needed when the +sandbox does not already provide them. Provision it equally for both arms and +establish a new baseline when changing the environment. + +The requirements pin cuEquivariance to the immutable source revision whose APIs +were checked, and pin the JAX/Flax or CPU PyTorch versions. They do not require sibling skills or a repository checkout inside the sandbox. When updating the API baseline, update the pins and run the numerical checks before changing the assertions. The CUDA backend constraints are checked as guidance; these datasets do not benchmark GPU performance or validate CUDA kernels. The tests under `.github/tests/test_*_skill_evals.py` verify the mathematical and -API expectations behind the prompts. `test_torch_skill_examples.py` also executes -the published Torch examples and checks the GNN's determinism, equivariance, and -gradients. These are maintenance checks, not live agent scores. To run them against +API expectations behind the prompts. `test_jax_skill_examples.py` executes each +published CPU example independently and checks layout conversion with duplicate +irreps and split polynomials with additional batch axes, parity, and JIT. +`test_torch_skill_examples.py` executes the published Torch examples and checks +the GNN's determinism, equivariance, and gradients. These are maintenance checks, +not live agent scores. To run them against a development checkout with the relevant frontend dependencies installed: ```bash pytest -q .github/tests/test_core_skill_evals.py -JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py +JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py ``` @@ -80,6 +91,34 @@ skillevaluator tier3 validate skills/cuequivariance-jax skillevaluator tier3 validate skills/cuequivariance-torch ``` +### Interpreting Tier 3 comparisons + +Use the exported JSON and per-trial evidence to compare revisions. The pipeline +gate, static quality score, execution coverage, and skill uplift answer different +questions; a green pipeline does not establish positive uplift. + +- Check `tier3.coverage` first. Compare the same eligible agents and models across + revisions. An agent excluded for `agent_process_exit` has no valid quality + score; a displayed zero is not a measured skill regression. Exit 137 alone + does not identify whether memory pressure, timeout, or another kill caused it. +- Use the precision recorded in `tier3.quality`. With default grading in 1.5.6, gate + lift averages the with-skill minus baseline differences for the shared metrics + `security`, `accuracy`, `goal_accuracy`, and `behavior_check`. Skill execution + and tool productivity are N/A for baseline; token efficiency is a separate + diagnostic and is outside that gate calculation. +- Pair trials by case ID. Inspect security evidence and numerical/workflow + checks before using aggregate recommendations. Record total and productive + tool calls plus uncached prompt and completion tokens to locate setup and + exploration overhead. +- Hold the dataset, dependency versions, environment, agent models, evaluator, + thresholds, and attempt policy fixed for a skill-only comparison. Capture + package versions from the actual run: requirements files alone are insufficient. + Preserve all attempts and infrastructure failures when repeating a comparison. +- One attempt per case is exploratory evidence. Repeat both arms before claiming + a small improvement is reliable; do not select only the best rerun. Keep new + evaluation cases in a separately identified benchmark until both revisions + have been measured on it. + ## Packaging The release workflow materializes symlinks **before** running Hatchling, including diff --git a/skills/cuequivariance-jax/SKILL.md b/skills/cuequivariance-jax/SKILL.md index a5ebaa88..49605c85 100644 --- a/skills/cuequivariance-jax/SKILL.md +++ b/skills/cuequivariance-jax/SKILL.md @@ -1,435 +1,311 @@ --- name: cuequivariance-jax -description: Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear). Use when writing JAX code with cuequivariance. +description: Use when writing or debugging cuEquivariance JAX polynomials, irrep dictionaries, Flax NNX layers, or CPU/CUDA execution and layouts. license: Apache-2.0 metadata: author: Mario Geiger + tags: [jax, equivariance, tensor-products] --- -# cuequivariance_jax: Executing Equivariant Polynomials in JAX +# cuEquivariance in JAX -## Overview +## Purpose -`cuequivariance_jax` (imported as `cuex`) executes `cuequivariance` polynomials on CPU or GPU via JAX. It provides: +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_jax` (`cuex`), including autodiff, JIT, irrep dictionaries, and Flax NNX layers. Preserve the user's framework, environment, and device constraints. Custom representations and Clebsch–Gordan coefficients belong to the core `cuequivariance` API; PyTorch modules use `cuequivariance_torch`. -1. **Core primitive**: `cuex.segmented_polynomial()` — JAX primitive with full AD/vmap/JIT support -2. **Two data representations** (both built on `segmented_polynomial`): - - `cuex.equivariant_polynomial()` + `RepArray` — the original interface, a single contiguous array with representation metadata - - `cuex.ir_dict` module — `dict[Irrep, Array]` interface, uses `IrDictPolynomial` descriptors, works naturally with `jax.tree` -3. **NNX layers**: `cuex.nnx` module — Flax NNX `Module` wrappers using `dict[Irrep, Array]` +## Inputs -## Instructions - -1. Check `jax.devices()`, installed versions, and whether Flax NNX is needed. Install `cuequivariance-jax` with JAX; add Flax for NNX layers. Follow the installed release's compatibility requirements. -2. Start with `method="naive"` for a portable CPU reference. Select a CUDA method only when a compatible GPU, kernel package, dtype, and descriptor are available; do not assume default methods run on CPU. -3. Choose dense `RepArray` or `ir_dict` explicitly. High-level irrep dictionaries store `(..., multiplicity, ir.dim)` blocks; low-level CUDA segmented arrays have a different contract. Match the dense layout in both conversion directions. -4. Derive operand sizes and output specs from the descriptor. Split PRNG keys, check broadcasting, compare to the core NumPy reference, and verify `jax.jit` and gradients before optimizing. -5. For custom group definitions and CG coefficients, use the `cuequivariance` skill. For PyTorch modules, use `cuequivariance-torch`. +Required inputs are the operation, SO3/O3 irreps, layout, batch shapes, dtype, device, and weight ownership; derive them from the request or existing code and state defaults. Graph operations also need sender/receiver indices and destination count. Optional inputs include a preferred backend or performance target. Derive operand sizes and output irreps from descriptors rather than guessing multiplicities. -Source paths below are relative to the `cuequivariance_jax/` project inside a source checkout. For a separately installed skill, locate code with `cuex.__file__` or https://github.com/NVIDIA/cuEquivariance. - -## Examples - -Examples build on the imports and descriptors shown earlier. CUDA examples are marked by their method or device; use the naive method for CPU checks. +## Instructions -## Execution methods +1. Probe the selected interpreter once; reuse working packages. Choose `method="naive"` for CPU. Install only missing requirements, with Flax needed only for NNX layers. +2. Choose dense `RepArray` or `dict[Irrep, Array]`. High-level dictionary blocks are `(..., mul, ir.dim)`; the low-level uniform CUDA wrapper has a different shape contract. +3. Build the descriptor outside transformed functions. Derive output specs from its outputs and the broadcast batch shape. Enable x64 **before creating arrays** for float64 numerical checks. +4. Run the relevant small example with the selected interpreter, then adapt it. Compare against NumPy with identical operands; check JIT or derivatives when relevant. Report executed shapes/devices and any checks that could not run. -| Method | Backend | Requirements | -|--------|---------|-------------| -| `"naive"` | Pure JAX | Always works, any platform | -| `"uniform_1d"` | CUDA kernel | GPU, all segments uniform shape within each operand, single mode | -| `"indexed_linear"` | CUDA kernel | GPU, linear operations with `cuex.Repeats` indexing | +## Prerequisites and environment recovery -## Core primitive: segmented_polynomial +CPU polynomial execution needs JAX, `cuequivariance`, and `cuequivariance-jax`; it needs no CUDA kernel package. Check imports independently so one missing package does not hide a working JAX installation: ```python -import jax -import jax.numpy as jnp -import numpy as np -import cuequivariance as cue -import cuequivariance_jax as cuex - -# Build a descriptor -e = cue.descriptors.channelwise_tensor_product( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) -poly = e.polynomial - -batch = 64 -key_x, key_y = jax.random.split(jax.random.key(0)) -key = jax.random.key(1) # independent key for subsequent indexing examples -w = jnp.ones((poly.inputs[0].size,)) # weights (shared across batch) -x = jax.random.normal(key_x, (batch, poly.inputs[1].size)) # batched input 1 -y = jax.random.normal(key_y, (batch, poly.inputs[2].size)) # batched input 2 - -# Execute with naive method -[out] = cuex.segmented_polynomial( - poly, - [w, x, y], # inputs - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], # output spec - method="naive", -) - -# Execute with uniform_1d (GPU, requires uniform segments) -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) +import importlib +import sys + +print("python:", sys.executable) +available, import_errors = {}, {} +for name in ("jax", "cuequivariance", "cuequivariance_jax"): + try: + module = importlib.import_module(name) + available[name] = module + print(name, getattr(module, "__version__", "unknown"), module.__file__) + except (ImportError, OSError) as error: + import_errors[name] = f"{type(error).__name__}: {error}" +print("import errors:", import_errors) +if "jax" in available: + print("devices:", available["jax"].devices()) ``` -### Multiple batch axes with broadcasting +If these imports work, continue without reinstalling. For NNX, additionally check `from flax import nnx`. A binary/version import error is different from an absent package: inspect that error and the project's dependency constraints before changing versions. In a checkout, follow its installation instructions; keep core and frontend source revisions compatible. When supplied, honor `scripts/requirements.txt` beside this skill or the project's lockfile instead of substituting latest releases. -Inputs can have any number of batch axes (everything before the last axis). Standard NumPy broadcasting applies: each batch axis is either size-1 or a common size. Inputs with fewer batch dimensions are implicitly prepended with size-1 axes: +If an isolated environment is needed, create a **new unique directory**, preserving existing environments and project files. On Linux, this recipe bypasses `ensurepip` and uses host pip 22.3+: -```python -# Fewer batch dims: weights with no batch axis broadcast across all -w = jnp.ones((poly.inputs[0].size,)) # 0 batch axes -> broadcasts -x = jnp.ones((5, 10, poly.inputs[1].size)) -y = jnp.ones((5, 10, poly.inputs[2].size)) - -[out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((5, 10, poly.outputs[0].size), jnp.float32)], - method="uniform_1d", -) +```bash +set -e +cueq_jax_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-jax.XXXXXX") +python3 -m venv --without-pip "$cueq_jax_env" +python3 -m pip --python "$cueq_jax_env" install "jax[cpu]" cuequivariance-jax +JAX_PLATFORMS=cpu "$cueq_jax_env/bin/python" -c 'import jax, cuequivariance_jax; print(jax.devices())' ``` -### Indexing (gather/scatter) - -Index arrays provide gather (for inputs) and scatter (for outputs). One index per operand (inputs + outputs), `None` means no indexing: +Replace the install package list with `-r` and the supplied requirements path when versions are prescribed. Add Flax only for an NNX task unless those requirements already include it. Keep using the chosen interpreter explicitly. Stop on a setup failure and retain its error; do not delete `.venv` or change system packages to repair `ensurepip`. If host pip is unavailable, use an existing manager such as `uv`. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/) and [JAX installation](https://docs.jax.dev/en/latest/installation.html). -```python -i = jax.random.randint(key, (100, 50), 0, 10) # gather b along axis 0 -j1 = jax.random.randint(key, (100, 50), 0, 11) # scatter output axis 0 -j2 = jax.random.randint(key, (100, 1), 0, 12) # scatter output axis 1 - -[out] = cuex.segmented_polynomial( - poly, [a, b, c], - [jax.ShapeDtypeStruct((11, 12, poly.outputs[0].size), jnp.float32)], - indices=[None, np.s_[i, :], None, np.s_[j1, j2]], - method="uniform_1d", -) -``` +## Limitations and backend selection -### Gradients +| Route | Supported use | +| --- | --- | +| `segmented_polynomial(..., method="naive")` | Portable JAX execution, including CPU, JIT, gradients, and vmap. | +| `cuex.nnx.IrrepsLinear` | CPU/GPU channel mixing within matching irreps using JAX einsum. | +| `cuex.nnx.SphericalHarmonics` | CPU/GPU; normalizes vectors and uses the naive polynomial implementation. | +| `method="uniform_1d"` | CUDA; uniform segment shapes within each operand and a single mode after preparation. | +| `method="indexed_linear"`, `cuex.nnx.IrrepsIndexedLinear` | CUDA indexed linear route, using `cuex.Repeats`; not the ordinary CPU linear layer. | -Fully differentiable — supports `jax.grad`, `jax.jacobian`, `jax.jvp`, `jax.vmap`: +A CPU-only task should use the first three routes without installing GPU dependencies. CUDA execution requires compatible JAX, GPU, and kernel builds; CPU checks do not establish CUDA compatibility or performance. Do not infer a CPU fallback from moving arrays to CPU. Avoid zero input vectors with NNX spherical harmonics at `eps=0`, since normalization divides by their norm; choose a positive `eps` if zero vectors are possible. -```python -def loss(w, x, y): - [out] = cuex.segmented_polynomial( - poly, [w, x, y], - [jax.ShapeDtypeStruct((batch, poly.outputs[0].size), jnp.float32)], - method="naive", - ) - return jnp.sum(out ** 2) - -grad_w = jax.grad(loss, 0)(w, x, y) -``` - -## ir_dict interface - -Uses `dict[Irrep, Array]` where each value has shape `(..., multiplicity, irrep_dim)`. This is the standard representation for NNX layers and works naturally with `jax.tree` operations. - -### Getting an ir_dict-ready polynomial +## Examples -Use `_ir_dict` descriptor variants, which return `IrDictPolynomial` with the polynomial already split by irrep: +Each example below runs with this common preamble; the NNX example adds Flax. For a CPU check, launch the script with `JAX_PLATFORMS=cpu`. The examples use small nonconstant arrays so layout errors remain visible. ```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) +import jax +jax.config.update("jax_enable_x64", True) +import jax.numpy as jnp +import numpy as np +import cuequivariance as cue +import cuequivariance_jax as cuex +from cuequivariance_jax import ir_dict -poly = desc.polynomial # SegmentedPolynomial, already split by irrep -weight_irreps, irreps1, irreps2 = desc.input_irreps -(irreps_out,) = desc.output_irreps # tuple unpacking to get the single output group +dtype = jnp.float64 +print("devices:", jax.devices()) ``` -Each polynomial operand corresponds to exactly one `(mul, ir)` block. The `input_irreps` and `output_irreps` tuples describe how operands group into logical operand groups (weights, node features, spherical harmonics, output). +### Spherical harmonics without Flax -### Executing with segmented_polynomial_uniform_1d +The descriptor takes an input irrep and a list of degrees. This evaluates the **raw polynomial**; normalize vectors first when unit-direction harmonics are required. O3 keys retain parity, unlike SO3 keys. ```python -from einops import rearrange - -num_edges, num_nodes = 100, 30 - -# Weights: reshape to (batch, num_segments, segment_size) -w_flat = jax.random.normal(key, (num_edges, poly.inputs[0].size)) -w = rearrange(w_flat, "e (s m) -> e s m", s=poly.inputs[0].num_segments) - -# Node features: dict[Irrep, Array] reshaped to (nodes, ir.dim, mul) for ir_mul layout -node_feats = { - cue.SO3(0): jnp.ones((num_nodes, 32, 1)), # 32x scalar - cue.SO3(1): jnp.ones((num_nodes, 32, 3)), # 32x vector -} -x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - -# Spherical harmonics: (edges, ir.dim) — no multiplicity dimension -sph = { - cue.SO3(0): jnp.ones((num_edges, 1)), - cue.SO3(1): jnp.ones((num_edges, 3)), +desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2]) +vectors = jnp.array([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], dtype=dtype) +values = cuex.segmented_polynomial( + desc.polynomial, [vectors], + [jax.ShapeDtypeStruct(vectors.shape[:-1] + (op.size,), dtype) + for op in desc.polynomial.outputs], + method="naive", +) +(irreps_out,) = desc.output_irreps +harmonics = { + ir: value.reshape(vectors.shape[:-1] + (mul, ir.dim)) + for (mul, ir), value in zip(irreps_out, values) } - -# Build output template -senders = jax.random.randint(key, (num_edges,), 0, num_nodes) -receivers = jax.random.randint(key, (num_edges,), 0, num_nodes) -out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(irreps_out, poly.outputs) +for actual, expected in zip(values, desc.polynomial(np.asarray(vectors))): + np.testing.assert_allclose(actual, expected, rtol=1e-12, atol=1e-12) +assert {ir.l: v.shape for ir, v in harmonics.items()} == { + 0: (2, 1, 1), 1: (2, 1, 3), 2: (2, 1, 5), } - -# Execute with gather (senders) and scatter (receivers) -y = cuex.ir_dict.segmented_polynomial_uniform_1d( - poly, - [w, x1, sph], - out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", -) -# y is dict[Irrep, Array] with accumulated results at receiver nodes +print({str(ir): (v.shape, v.devices()) for ir, v in harmonics.items()}) ``` -### ir_dict utility functions +### Dictionary layouts and exact round trips -```python -# Validate dict matches irreps -cuex.ir_dict.assert_mul_ir_dict(irreps, x) # asserts shape (..., mul, ir.dim) - -# Convert flat array <-> dict -d = cuex.ir_dict.flat_to_dict(irreps, flat_array) # layout="mul_ir" default -d = cuex.ir_dict.flat_to_dict(irreps, flat_array, layout="ir_mul") -flat = cuex.ir_dict.dict_to_flat(irreps, d) - -# Arithmetic -z = cuex.ir_dict.irreps_add(x, y) -z = cuex.ir_dict.irreps_zeros_like(x) - -# Create template dict -template = cuex.ir_dict.mul_ir_dict(irreps, jax.ShapeDtypeStruct(shape, dtype)) -``` - -`dict_to_flat` always emits `mul_ir`; it has no `layout` argument. To emit -`ir_mul`, swap the last two axes of each dictionary block, flatten each block, -then concatenate in `irreps` order. Regroup duplicate irreps before storing them -as dictionary keys, so a later block cannot overwrite an earlier one. - -## RepArray interface: equivariant_polynomial +`flat_to_dict(..., layout="ir_mul")` transposes each block to `(..., mul, ir.dim)`. **`dict_to_flat` has no layout argument and always emits `mul_ir`.** Returning to `ir_mul` requires a transpose within every block; a whole-array reshape changes the meaning of the data. -The original interface. Wraps `segmented_polynomial` with `RepArray` — a single contiguous array with representation metadata: +**Regroup duplicate irreps before using irrep keys**, or later blocks overwrite earlier ones. Regroup the corresponding feature data too: changing only the irreps metadata does not reorder an existing buffer. A `RepArray` can regroup metadata and data together. Use descriptor/irreps order when assembling dictionaries and operands. ```python -e = cue.descriptors.fully_connected_tensor_product( - 4 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - 4 * cue.Irreps("SO3", "0 + 1"), -) - -inputs = [ - cuex.randn(jax.random.key(i), rep, (batch,), jnp.float32) - for i, rep in enumerate(e.inputs) -] - -# Returns a RepArray with representation metadata -out = cuex.equivariant_polynomial(e, inputs, method="naive") -out.array # the raw jax.Array -out.reps # dict mapping axes to Rep objects +def dict_to_ir_mul_flat(irreps, blocks): + return jnp.concatenate([ + blocks[ir].swapaxes(-1, -2).reshape( + blocks[ir].shape[:-2] + (mul * ir.dim,) + ) + for mul, ir in irreps + ], axis=-1) + +irreps = cue.Irreps("SO3", "3x0 + 2x1").regroup() +flat_ir = jnp.arange(2 * irreps.dim).reshape(2, irreps.dim) +blocks = ir_dict.flat_to_dict(irreps, flat_ir, layout="ir_mul") +offset = 0 +for mul, ir in irreps: + size = mul * ir.dim + expected = np.asarray(flat_ir[..., offset:offset + size]).reshape(2, ir.dim, mul) + assert blocks[ir].shape == (2, mul, ir.dim) + np.testing.assert_array_equal(blocks[ir], expected.swapaxes(-1, -2)) + offset += size +flat_mul = ir_dict.dict_to_flat(irreps, blocks) +restored_blocks = ir_dict.flat_to_dict(irreps, flat_mul, layout="mul_ir") +for ir in blocks: + np.testing.assert_array_equal(restored_blocks[ir], blocks[ir]) +np.testing.assert_array_equal(dict_to_ir_mul_flat(irreps, restored_blocks), flat_ir) ``` -## NNX layers +`ir_dict.assert_mul_ir_dict` checks block conventions; `irreps_add` and `irreps_zeros_like` operate on matching dictionaries. `mul_ir_dict(irreps, data)` broadcasts a template object to each key; it does not infer feature dimensions. -### IrrepsLinear +### Dense polynomial: broadcasting, JIT, and a gradient -Equivariant linear layer using `dict[Irrep, Array]`: +`segmented_polynomial` takes a list of inputs and returns a list of outputs. Channelwise products put weights first. Shared weights need **no batch axis**; broadcast all input shapes except the final feature axis. `filter_irreps_out` selects irrep types, not output multiplicities. Dense descriptor operands carry their layout in `e.inputs` and `e.outputs`. ```python -from flax import nnx - -linear = cuex.nnx.IrrepsLinear( - irreps_in=cue.Irreps(cue.SO3, "4x0 + 2x1").regroup(), # must be regrouped - irreps_out=cue.Irreps(cue.SO3, "3x0 + 5x1").regroup(), - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), +e = cue.descriptors.channelwise_tensor_product( + cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1"), + [cue.SO3(0), cue.SO3(1)], ) - -# Input/output: dict[Irrep, Array] with shape (batch, mul, ir.dim) -x = { - cue.SO3(0): jnp.ones((batch, 4, 1)), - cue.SO3(1): jnp.ones((batch, 2, 3)), -} -y = linear(x) -# y[cue.SO3(0)].shape == (batch, 3, 1) -# y[cue.SO3(1)].shape == (batch, 5, 3) +poly = e.polynomial +rng = np.random.default_rng(7) +w = jnp.asarray(rng.normal(size=poly.inputs[0].size)) +x = jnp.asarray(rng.normal(size=(3, 1, poly.inputs[1].size))) +y = jnp.asarray(rng.normal(size=(1, 2, poly.inputs[2].size))) + +def run(weights, a, b): + batch = np.broadcast_shapes(weights.shape[:-1], a.shape[:-1], b.shape[:-1]) + return cuex.segmented_polynomial( + poly, [weights, a, b], + [jax.ShapeDtypeStruct(batch + (op.size,), jnp.result_type(weights, a, b)) + for op in poly.outputs], + method="naive", + )[0] + +reference = e(*(np.asarray(v) for v in (w, x, y)))[0] +np.testing.assert_allclose(run(w, x, y), reference, rtol=1e-12, atol=1e-12) +np.testing.assert_allclose(jax.jit(run)(w, x, y), reference, rtol=1e-12, atol=1e-12) +assert reference.shape == (3, 2, poly.outputs[0].size) +gradient = jax.grad(lambda v: jnp.sum(run(v, x, y) ** 2))(w) +index = int(jnp.argmax(jnp.abs(gradient))) +assert abs(gradient[index]) > 1e-8 +epsilon = 1e-5 +loss_plus = np.sum(e(np.asarray(w.at[index].add(epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) +loss_minus = np.sum(e(np.asarray(w.at[index].add(-epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) +np.testing.assert_allclose(gradient[index], (loss_plus - loss_minus) / (2 * epsilon), + rtol=1e-6, atol=1e-7) ``` -Implementation uses `jnp.einsum("uv,...ui->...vi", w, x[ir])` per irrep with `1/sqrt(mul_in)` normalization. +For gather/scatter, supply one `indices` entry per polynomial operand, inputs then outputs; `None` leaves an operand unindexed. Output indices require an explicit destination shape. Repeated destinations accumulate. Compare against an edgewise calculation with explicit summation before optimizing a graph contraction. -### SphericalHarmonics +### Split irrep polynomials on CPU -Uses `spherical_harmonics_ir_dict` internally for the `dict[Irrep, Array]` output: +`channelwise_tensor_product_ir_dict` returns an `IrDictPolynomial`. Its `input_irreps` groups are weights, features, and the second input; each `(mul, ir)` becomes one polynomial operand. Its `output_irreps` gives the split output order. These split descriptors use `ir_mul`: transpose the high-level blocks before flattening, then reverse that conversion on outputs. There is no `ir_dict.equivariant_polynomial` entrypoint. ```python -sh = cuex.nnx.SphericalHarmonics(max_degree=3, eps=0.0) - -vectors = jax.random.normal(key, (batch, 3)) # 3D vectors -y = sh(vectors) -# y[cue.O3(0, 1)].shape == (batch, 1, 1) # L=0 -# y[cue.O3(1, -1)].shape == (batch, 1, 3) # L=1 -# y[cue.O3(2, 1)].shape == (batch, 1, 5) # L=2 -# y[cue.O3(3, -1)].shape == (batch, 1, 7) # L=3 -``` - -### IrrepsNormalize - -```python -norm = cuex.nnx.IrrepsNormalize(eps=1e-6, scale=1.0, skip_scalars=True) -y = norm(x) # normalizes non-scalar irreps by RMS over ir.dim, averaged over mul -``` - -### MLP (scalar only) +def split_naive(desc, groups): + assert len(groups) == len(desc.input_irreps) + operands = [ + group[ir].swapaxes(-1, -2).reshape(group[ir].shape[:-2] + (mul * ir.dim,)) + for irreps, group in zip(desc.input_irreps, groups) + for mul, ir in irreps + ] + batch = np.broadcast_shapes(*(v.shape[:-1] for v in operands)) + outputs = cuex.segmented_polynomial( + desc.polynomial, operands, + [jax.ShapeDtypeStruct(batch + (op.size,), jnp.result_type(*operands)) + for op in desc.polynomial.outputs], + method="naive", + ) + values = iter(outputs) + return tuple({ + ir: next(values).reshape(batch + (ir.dim, mul)).swapaxes(-1, -2) + for mul, ir in irreps + } for irreps in desc.output_irreps) -```python -mlp = cuex.nnx.MLP( - layer_sizes=[64, 128, 64], - activation=jax.nn.silu, - output_activation=False, - dtype=jnp.float32, - rngs=nnx.Rngs(0), +desc = cue.descriptors.channelwise_tensor_product_ir_dict( + cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1"), + [cue.SO3(0), cue.SO3(1)], ) -y = mlp(x_scalar) # standard dense MLP with 1/sqrt(fan_in) normalization +rng = np.random.default_rng(8) +groups = [ + {ir: jnp.asarray(rng.normal(size=batch + (mul, ir.dim))) for mul, ir in irreps} + for irreps, batch in zip(desc.input_irreps, [(), (3, 1), (1, 2)]) +] +(result,) = split_naive(desc, groups) +numpy_inputs = [ + np.asarray(group[ir]).swapaxes(-1, -2).reshape(group[ir].shape[:-2] + (mul * ir.dim,)) + for irreps, group in zip(desc.input_irreps, groups) for mul, ir in irreps +] +expected = desc.polynomial(*numpy_inputs) +for (mul, ir), flat in zip(desc.output_irreps[0], expected): + assert result[ir].shape == (3, 2, mul, ir.dim) + np.testing.assert_allclose(result[ir], flat.reshape(3, 2, ir.dim, mul).swapaxes(-1, -2), + rtol=1e-12, atol=1e-12) ``` -### IrrepsIndexedLinear +### Flax NNX layers -For species-indexed linear layers (different weights per atom type): +`IrrepsLinear` requires regrouped irreps and explicit `nnx.Rngs`. It mixes channels only within the same irrep, with `scale / sqrt(mul_in)` normalization. An output irrep absent from the input receives zeros, not a learned cross-irrep mapping. `nnx.Param` weights can be read with `[...]` for an independent reference. Row-vector features rotate as `x @ D.T`. ```python -indexed_linear = cuex.nnx.IrrepsIndexedLinear( - irreps_in=cue.Irreps(cue.O3, "8x0e").regroup(), - irreps_out=cue.Irreps(cue.O3, "16x0e").regroup(), - num_indices=50, # number of species - scale=1.0, - dtype=jnp.float32, - rngs=nnx.Rngs(0), -) - -# num_index_counts: how many atoms of each species -species_counts = jnp.array([3, 4, 3, ...]) # sum = batch_size -y = indexed_linear(x, species_counts) +from flax import nnx +from cuequivariance_jax import nnx as cue_nnx + +irreps_in = cue.Irreps("SO3", "3x0 + 2x1").regroup() +irreps_out = cue.Irreps("SO3", "2x0 + 4x1 + 2").regroup() +linear = cue_nnx.IrrepsLinear(irreps_in, irreps_out, scale=0.7, dtype=dtype, rngs=nnx.Rngs(9)) +rng = np.random.default_rng(10) +features = {ir: jnp.asarray(rng.normal(size=(2, mul, ir.dim))) for mul, ir in irreps_in} +output = linear(features) +for mul, ir in irreps_out: + assert output[ir].shape == (2, mul, ir.dim) +np.testing.assert_array_equal(output[cue.SO3(2)], 0) +for mul, ir in irreps_in: + expected = np.einsum("uv,...ui->...vi", np.asarray(linear.w[str(ir)][...]), features[ir]) + np.testing.assert_allclose(output[ir], expected * linear.scale / np.sqrt(mul), + rtol=1e-12, atol=1e-12) +rotation = np.array([0.3, -0.2, 0.1]) +rotated_output = linear({ir: v @ ir.exp_map(rotation, []).T for ir, v in features.items()}) +for ir, value in output.items(): + np.testing.assert_allclose(rotated_output[ir], value @ ir.exp_map(rotation, []).T, + rtol=1e-12, atol=1e-12) + +vectors = jnp.array([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], dtype=dtype) +sh = cue_nnx.SphericalHarmonics(max_degree=2) +harmonics = sh(vectors) +for ir, value in harmonics.items(): + assert value.shape == (2, 1, ir.dim) + assert bool(jnp.isfinite(value).all()) +print({str(ir): (v.shape, v.devices()) for ir, v in harmonics.items()}) ``` -Uses `method="indexed_linear"` internally with `cuex.Repeats`. - -## Preparing polynomials for uniform_1d - -The `uniform_1d` CUDA kernel requires: -1. All segments within each operand have **the same shape** -2. A **single mode** in the subscripts (after preprocessing) +`IrrepsNormalize(eps=..., scale=..., skip_scalars=...)` normalizes using the sum of squared components averaged over multiplicities. `MLP(layer_sizes=..., activation=..., dtype=..., rngs=...)` operates on scalar features with fan-in normalization. Use `IrrepsIndexedLinear` only for the supported CUDA route: species counts must match grouped input rows and their sum must match the batch size. -### From EquivariantPolynomial to uniform_1d-ready - -For `equivariant_polynomial()` (RepArray interface): - -```python -e = cue.descriptors.channelwise_tensor_product(...) -e = e.squeeze_modes().flatten_coefficient_modes() -out = cuex.equivariant_polynomial(e, inputs, method="uniform_1d") -``` +### Dense RepArray interface -For `ir_dict` (dict[Irrep, Array] interface), use `_ir_dict` descriptors directly: +Use representation metadata from the descriptor to avoid a layout mismatch. A single output normally returns one `RepArray`; its `.array` is the JAX array and `.reps` maps axes to representations. ```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out +e = cue.descriptors.fully_connected_tensor_product( + cue.Irreps("SO3", "2x0 + 1"), cue.Irreps("SO3", "0 + 1"), + cue.Irreps("SO3", "0 + 2x1"), ) -poly = desc.polynomial -# Each operand has a single irrep type -> maps naturally to dict[Irrep, Array] +keys = jax.random.split(jax.random.key(11), len(e.inputs)) +inputs = [cuex.randn(key, rep, (2,), dtype) for key, rep in zip(keys, e.inputs)] +output = cuex.equivariant_polynomial(e, inputs, method="naive") +np.testing.assert_allclose(output.array, e(*(np.asarray(v.array) for v in inputs))[0], + rtol=1e-12, atol=1e-12) ``` -### Why splitting by irrep matters +## CUDA preparation and message passing -Without splitting, a dense operand like `32x0+32x1` requires all irreps packed into a single contiguous buffer. After splitting, each irrep gets its own separate buffer passed to the CUDA kernel via FFI. The buffers no longer need to be contiguous with each other. +For a compatible dense descriptor, `e.squeeze_modes().flatten_coefficient_modes()` prepares its modes before `equivariant_polynomial(..., method="uniform_1d")`. Check uniform segment shapes within each operand. Splitting by irrep separates buffers; it does **not** remove the need to convert their layouts. -This is especially useful when the polynomial is preceded or followed by per-irrep linear layers (like `IrrepsLinear`). With split operands, no transpose or copy is needed between the linear layers and the polynomial — the `dict[Irrep, Array]` flows directly through the pipeline. +`ir_dict.segmented_polynomial_uniform_1d(poly, inputs, outputs, input_indices=..., output_indices=...)` expects pytree leaves shaped `(..., op.num_segments, *op.segment_shape)`. This is not the high-level `(..., mul, ir.dim)` convention. For each `_ir_dict` operand, transpose and flatten as in `split_naive`, then reshape using that operand's segment metadata. Ensure pytree leaf order matches polynomial operand order. Output specs need the same segment suffix. Reverse both reshapes and the transpose before passing results into `IrrepsLinear`. -## Complete GNN message-passing example +For GNN message passing, weight arrays belong to edges, features to source nodes, and spherical harmonics to edge directions. Supply gather indices for node features (senders) and scatter indices for outputs (receivers); destination count determines output batch size. Construct learned weights or an edge-weight network once in the NNX module initializer. Use explicit split random keys for independent inputs. The source examples `cuequivariance_jax/examples/nequip_nnx.py` and `mace_nnx.py` show complete architectures and CUDA execution. -This pattern is used in NequIP, MACE, and similar equivariant GNN models: +## Troubleshooting -```python -class MessagePassing(nnx.Module): - def __init__(self, irreps_in, irreps_sh, irreps_out, epsilon, *, name, dtype, rngs): - self.name = name - desc = cue.descriptors.channelwise_tensor_product_ir_dict( - irreps_in, irreps_sh, irreps_out - ) - (self.irreps_out,) = desc.output_irreps - self.poly = desc.polynomial * epsilon - self.weight_numel = self.poly.inputs[0].size - - def __call__(self, weights, node_feats, sph, senders, receivers, num_nodes): - # weights: (num_edges, weight_numel) - w = rearrange(weights, "e (s m) -> e s m", s=self.poly.inputs[0].num_segments) - # node_feats: dict[Irrep, Array] with (nodes, mul, ir.dim) - x1 = jax.tree.map(lambda v: rearrange(v, "n m i -> n i m"), node_feats) - # sph: dict[Irrep, Array] with (edges, 1, ir.dim) or (edges, ir.dim) - x2 = jax.tree.map(lambda v: rearrange(v, "e 1 i -> e i"), sph) - - out_template = { - ir: jax.ShapeDtypeStruct( - (num_nodes, desc.num_segments) + desc.segment_shape, w.dtype - ) - for (_, ir), desc in zip(self.irreps_out, self.poly.outputs) - } - - y = cuex.ir_dict.segmented_polynomial_uniform_1d( - self.poly, [w, x1, x2], out_template, - input_indices=[None, senders, None], - output_indices=receivers, - name="tensor_product", - ) - # Rearrange output back to (nodes, mul, ir.dim) for downstream layers - return { - ir: rearrange(v, "n (i s) m -> n (s m) i", i=ir.dim) - for ir, v in y.items() - } -``` - -## RepArray - -Representation-aware JAX array: - -```python -rep = cue.IrrepsAndLayout(cue.Irreps("SO3", "4x0 + 2x1"), cue.ir_mul) -x = cuex.RepArray(rep, jnp.ones((batch, rep.dim))) -x = cuex.randn(jax.random.key(0), rep, (batch,), jnp.float32) - -x.array # raw jax.Array -x.reps # {axis: Rep} -x.irreps # Irreps (if last axis is IrrepsAndLayout) -``` +| Symptom | Check | +| --- | --- | +| `ensurepip` unavailable or a partial environment exists | Preserve it; use a new directory with `--without-pip` and host pip `--python`, or an installed environment manager. | +| Missing CUDA kernels on CPU | Select `naive` or ordinary NNX layers. Changing the device alone does not change the method. | +| Float64 truncated or numerical comparison fails | Enable x64 before array creation, use the same dtype/normalization in both computations, then inspect layout and operand order. | +| Dictionary shape/value mismatch | Regroup duplicate irreps with their data, preserve irrep order, and transpose per block. `dict_to_flat` emits `mul_ir`. | +| Missing output or incorrect broadcast | One output spec per polynomial output; broadcast leading axes of all inputs, including weights. | +| An API is missing in the installed version | Inspect that symbol's signature/source and installed versions; do not invent keyword arguments or reinstall the entire stack blindly. | -## Key file locations - -| Component | Path | -|-----------|------| -| `segmented_polynomial` primitive | `cuequivariance_jax/segmented_polynomials/segmented_polynomial.py` | -| `uniform_1d` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_uniform_1d.py` | -| `naive` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_naive.py` | -| `indexed_linear` backend | `cuequivariance_jax/segmented_polynomials/segmented_polynomial_indexed_linear.py` | -| `equivariant_polynomial` | `cuequivariance_jax/equivariant_polynomial.py` | -| `ir_dict` module | `cuequivariance_jax/ir_dict.py` | -| `nnx` module | `cuequivariance_jax/nnx.py` | -| `RepArray` | `cuequivariance_jax/rep_array/rep_array_.py` | -| `Repeats` / utilities | `cuequivariance_jax/segmented_polynomials/utils.py` | -| NequIP example | `cuequivariance_jax/examples/nequip_nnx.py` | -| MACE example | `cuequivariance_jax/examples/mace_nnx.py` | +Locate installed source via `cuex.__file__`. In a checkout, relevant files under `cuequivariance_jax/cuequivariance_jax/` are `ir_dict.py`, `nnx.py`, `equivariant_polynomial.py`, `rep_array/rep_array_.py`, and `segmented_polynomials/segmented_polynomial*.py`. Package API documentation is at https://docs.nvidia.com/cuda/cuequivariance/. diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md index ca86c49d..b529e996 100644 --- a/skills/cuequivariance-torch/SKILL.md +++ b/skills/cuequivariance-torch/SKILL.md @@ -22,29 +22,41 @@ Required for the computation: operation, SO3/O3 irreps, layout, device, dtype, a 1. Check the selected interpreter and imports once using the probe below. Reuse a working environment. A missing CUDA backend is usually a method-selection issue, not a reason to reinstall PyTorch. 2. Set `method="naive"` for CPU calculations. Match module parameters, features, weights, and reference computations in device and dtype. Use float64 for small numerical and gradient checks. 3. Specify layout and weight ownership explicitly. Derive sizes and output order from the module/descriptor; do not guess multiplicities from a filter. -4. Build the smallest relevant computation, run it, and report the actual checks and any failure. Compare the requested forward/gradient/equivariance properties before optimizing. Avoid repeatedly inspecting unrelated modules or producing several competing implementations. +4. Adapt the relevant example to the task and run its requested checks together. Report the interpreter, shapes, numerical error, and gradient results actually observed. Investigate a failed check before broadening the implementation. 5. If an API differs from this guide, inspect that installed object's signature and source once. Adapt to the installed version or explain the required version; do not silently replace cuEquivariance with another library. ## Prerequisites and environment reuse -Run this with the project's Python interpreter (including its existing virtual environment, if any): +Run this with the project's Python interpreter (including its existing virtual environment, if any). Check imports independently so one missing package does not hide a working CPU PyTorch: ```python +import importlib import sys -import torch -import cuequivariance as cue -import cuequivariance_torch as cuet print("python:", sys.executable) -print("torch:", torch.__version__, "CUDA build:", torch.version.cuda) -print("CUDA available:", torch.cuda.is_available()) -print("cue:", cue.__file__) -print("cuet:", cuet.__file__) +available, import_errors = {}, {} +for name in ("torch", "cuequivariance", "cuequivariance_torch"): + try: + module = importlib.import_module(name) + available[name] = module + print(name, getattr(module, "__version__", "unknown"), module.__file__) + except (ImportError, OSError) as error: + import_errors[name] = f"{type(error).__name__}: {error}" + print(name, import_errors[name]) +if "torch" in available: + torch = available["torch"] + print("CUDA build:", torch.version.cuda, "available:", torch.cuda.is_available()) ``` -If these imports work, continue with this interpreter; CPU execution does not require CUDA kernels. When an import fails, first check whether a different project interpreter already has the dependencies. Use that same interpreter for installation and execution. +Choose the next step from the probe: + +- All imports work: continue with this interpreter and `method="naive"` for CPU; install nothing. +- PyTorch works but cuEquivariance is missing: use another existing project interpreter if it has both. Otherwise install only the missing cuEquivariance packages into the intended environment, honoring its version constraints. `cuequivariance-torch` is the distribution name; `cuequivariance_torch` is the import name. +- An import reports a missing shared library or binary mismatch: use the reported package/path to diagnose compatibility. This differs from an absent package; repeated installations or a backend change may not fix it. + +Use the same interpreter for pip and execution. Installing cuEquivariance does not require replacing an already working CPU PyTorch. Optional CUDA/optimization warnings alone are not installation failures. -Only when dependencies are missing and installation is appropriate, create a **new** environment. An installation or backend error does not justify deleting an existing environment, replacing the user's CPU PyTorch, or bypassing system-package protection. On Linux/macOS, this fallback also works when `venv` cannot bootstrap pip through `ensurepip` (the host needs pip 22.3+): +If no usable project environment is available, or it cannot be modified normally, create a **new** environment when installation is appropriate. Preserve existing environments and system-package protection. On Linux/macOS, this fallback also works when `venv` cannot bootstrap pip through `ensurepip` (the host needs pip 22.3+): ```bash cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") @@ -54,7 +66,7 @@ python3 -m pip --python "$cueq_env" install cuequivariance-torch "$cueq_env/bin/python" -c 'import torch, cuequivariance_torch; print(torch.__version__)' ``` -Honor project version/source constraints in these commands and keep using the new environment's Python explicitly. If pip is unavailable, use an existing manager such as `uv` or report the missing prerequisite. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). +Honor project version/source constraints and keep using the new environment's Python explicitly. Stop at the first failed setup command and diagnose its error; preserve the failure output. If pip is unavailable, use an existing manager such as `uv` or report the missing prerequisite. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). ## Limitations and backends From ae3411652ef1b2e136dc4ba48264e65276252de2 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Sun, 18 Oct 2026 19:53:20 +0000 Subject: [PATCH 06/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 69 +++++++++++++---------- skills/cuequivariance-jax/skill-card.md | 20 ++++--- skills/cuequivariance-jax/skill.oms.sig | 2 +- skills/cuequivariance-torch/BENCHMARK.md | 54 +++++++++--------- skills/cuequivariance-torch/skill-card.md | 14 ++--- skills/cuequivariance-torch/skill.oms.sig | 2 +- skills/cuequivariance/BENCHMARK.md | 52 +++++++++-------- skills/cuequivariance/skill-card.md | 14 ++--- skills/cuequivariance/skill.oms.sig | 2 +- 9 files changed, 119 insertions(+), 110 deletions(-) diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md index bb9ff66a..0dcc5195 100644 --- a/skills/cuequivariance-jax/BENCHMARK.md +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -1,13 +1,15 @@ # Skill Benchmark: cuequivariance-jax -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata - Skill: `cuequivariance-jax` -- Evaluation date: 2026-10-16 +- Evaluation date: 2026-10-18 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -33,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 84.8% — baseline ran, but no comparable score was available; uplift unavailable | 78.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 66.7% → 66.7% (±0.0 points) | 66.7% → 50.0% (-16.7 points) | +| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 75.0% (+25.0 points) | 83.3% → 83.3% (±0.0 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 96.3% → 100.0% (+3.7 points) | 90.4% → 92.9% (+2.5 points) | -| Efficiency | 74.2% — baseline ran, but no comparable score was available; uplift unavailable | 62.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 89.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 87.9% (-10.0 points) | 82.9% → 92.9% (+10.0 points) | +| Efficiency | 78.9% — baseline ran, but no comparable score was available; uplift unavailable | 68.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,008,776 | 4,893,147 | -884,371 | -18.07% | skill 6/6; base 6/6 | -| claude-code | jax-001 | 819,414 | 1,039,684 | -220,270 | -21.19% | skill 1/1; base 1/1 | -| claude-code | jax-002 | 644,824 | 519,762 | +125,062 | +24.06% | skill 1/1; base 1/1 | -| claude-code | jax-003 | 803,688 | 837,521 | -33,833 | -4.04% | skill 1/1; base 1/1 | -| claude-code | jax-004 | 1,165,511 | 1,871,827 | -706,316 | -37.73% | skill 1/1; base 1/1 | -| claude-code | jax-005 | 545,612 | 594,753 | -49,141 | -8.26% | skill 1/1; base 1/1 | -| claude-code | jax-006 | 29,727 | 29,600 | +127 | +0.43% | skill 1/1; base 1/1 | -| codex | All cases | 3,101,982 | 2,979,561 | +122,421 | +4.11% | skill 6/6; base 6/6 | -| codex | jax-001 | 1,059,589 | 862,896 | +196,693 | +22.79% | skill 1/1; base 1/1 | -| codex | jax-002 | 253,060 | 359,521 | -106,461 | -29.61% | skill 1/1; base 1/1 | -| codex | jax-003 | 645,104 | 513,418 | +131,686 | +25.65% | skill 1/1; base 1/1 | -| codex | jax-004 | 990,442 | 1,141,417 | -150,975 | -13.23% | skill 1/1; base 1/1 | -| codex | jax-005 | 140,349 | 88,989 | +51,360 | +57.71% | skill 1/1; base 1/1 | -| codex | jax-006 | 13,438 | 13,320 | +118 | +0.89% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 7,110,758 | 7,872,708 | -761,950 | -9.68% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,947,603 | 4,390,017 | +557,586 | +12.70% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 1,583,310 | 908,185 | +675,125 | +74.34% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 692,110 | 531,175 | +160,935 | +30.30% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 1,202,926 | 819,390 | +383,536 | +46.81% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 829,453 | 1,416,254 | -586,801 | -41.43% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 610,198 | 685,455 | -75,257 | -10.98% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 29,606 | 29,558 | +48 | +0.16% | skill 1/1; base 1/1 | +| codex | All cases | 1,936,562 | 2,876,893 | -940,331 | -32.69% | skill 6/6; base 6/6 | +| codex | jax-001 | 310,054 | 759,473 | -449,419 | -59.18% | skill 1/1; base 1/1 | +| codex | jax-002 | 273,170 | 469,747 | -196,577 | -41.85% | skill 1/1; base 1/1 | +| codex | jax-003 | 673,888 | 324,300 | +349,588 | +107.80% | skill 1/1; base 1/1 | +| codex | jax-004 | 611,120 | 1,265,573 | -654,453 | -51.71% | skill 1/1; base 1/1 | +| codex | jax-005 | 54,698 | 44,481 | +10,217 | +22.97% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,632 | 13,319 | +313 | +2.35% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 6,884,165 | 7,266,910 | -382,745 | -5.27% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -74,21 +76,26 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations
Show detailed findings and successful checks -- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance-jax/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: Description very long (276 chars, recommend 50-150) (`skills/cuequivariance-jax/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance-jax/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance-jax/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No limitations documented (`skills/cuequivariance-jax/SKILL.md`) -- 2 additional finding(s) are available in the full evaluation artifacts. +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 2 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance-jax': 133 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent codex
diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md index 6dc17b59..f8a4bfbd 100644 --- a/skills/cuequivariance-jax/skill-card.md +++ b/skills/cuequivariance-jax/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Execute equivariant polynomials in JAX using segmented_polynomial (naive/uniform_1d), the ir_dict workflow with IrDictPolynomial and dict[Irrep, Array], and Flax NNX layers (IrrepsLinear, SphericalHarmonics, IrrepsIndexedLinear).
+Use when writing or debugging cuEquivariance JAX polynomials, irrep dictionaries, Flax NNX layers, or CPU/CUDA execution and layouts.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers use this skill when writing JAX code with cuequivariance to execute equivariant polynomials on CPU or GPU, work with dict[Irrep, Array] or RepArray data, and build Flax NNX equivariant layers for geometric neural networks such as NequIP- and MACE-style GNNs.
+Developers and engineers use this skill when writing or debugging JAX code with cuequivariance to execute equivariant polynomials on CPU or GPU, work with dict[Irrep, Array] or RepArray data and layouts, and build Flax NNX equivariant layers for geometric neural networks such as NequIP- and MACE-style GNNs.
### Deployment Geography for Use:
Global
@@ -27,11 +27,13 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [JAX installation](https://docs.jax.dev/en/latest/installation.html)
+- [pip interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python code blocks]
+**Output Type(s):** [Code, Shell commands, Analysis]
+**Output Format:** [Markdown with inline Python and bash code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -66,12 +68,12 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 84.8% — baseline ran, but no comparable score was available; uplift unavailable | 78.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 66.7% → 66.7% (±0.0 points) | 66.7% → 50.0% (-16.7 points) | +| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 75.0% (+25.0 points) | 83.3% → 83.3% (±0.0 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 96.3% → 100.0% (+3.7 points) | 90.4% → 92.9% (+2.5 points) | -| Efficiency | 74.2% — baseline ran, but no comparable score was available; uplift unavailable | 62.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 89.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 87.9% (-10.0 points) | 82.9% → 92.9% (+10.0 points) | +| Efficiency | 78.9% — baseline ran, but no comparable score was available; uplift unavailable | 68.9% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig index 444901c4..58629ed8 100644 --- a/skills/cuequivariance-jax/skill.oms.sig +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjBhNzlmMjg4NDRkYzgyNDU3OTllNzY1ZGY5YTU2OGI5NDg3ZmNkOWRlOWM0MTNjNGJlM2U3OGZhZWMwNjEwZjAiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZjI1MmJmOTliZmMxM2MzZDE3YWYzZjUwNzc3YjdmODJmNTczYWY0ZmZmODliN2VjYTQzMzdhY2FiMTc1YjczIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZhNzFiZGI4M2E5ZGRiODFhODlkNzg5YTk5N2RkYmM3ZWE5NzZjZTc2NTk2MGJiYzM1YzUwY2Q1MmNhMjU2MDAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZmUxZDdiZDNmM2NiYjk3YzM5Y2VmNWVlN2RkMjgxZjE1ZDJhNWFkYTI0NGQ1YjU0MzA3NjYyMjQxMjBmNmRlNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5NDQ5MmE0ZDkwNTJmYjk5MDg2NGZlOGI4MTg4OTIzYWY5ZTMxYmYxMDc4YzQyOTdhMGIxZGIxNmRmZjliNjEiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCe/a0d2R+ibkK76aoY0mSiFiAXmnTnzpKt+YdyIp+arFHgEmqDUvEGU0rTAzaP/JQCMQCpJXOb3wkLxi9wadI0cJ2fF4A6s8Zyfn//qYMI5LqPrvPHLR4LDcQe6kVWRiaolp0=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjdmNzBjZmU3NWQ4MDkzMDlmMTkyMjIwZTViODEwMjdhNDE0OWIyNjYxOWUzMzc0NTY1ODY4NmE4MThkYTg5YzgiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImNiY2FiYzQ1NWZmNWYxY2FiMWM4OTg5ZDVkNjhkMWM5MzA1MTI2NzIzODUzMjIwODZkMjc5ZTc5ZDljZGMxMTAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzJjMDkxNjA4ZTQ3NWJlMTFiMTY5YjU2YWJiY2E0ZGUwNmE0YjAyZmI5NDZmZjI1YmNmYzFlZTQzZTE5OWQ2YyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YjY1ZjFmODVjZGY1ZGQ2NDkyMjJkOGEzNWQ4ZmY4Mzg2N2Y3MWU3NDUzOWQxNGM5Y2JiZWI4MWVlNDQ5OTg1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZTFkN2JkM2YzY2JiOTdjMzljZWY1ZWU3ZGQyODFmMTVkMmE1YWRhMjQ0ZDViNTQzMDc2NjIyNDEyMGY2ZGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTRhYzI5MTEyZmVjZWM2M2I0ODVjMzZmM2RiZjU4ODY3MWFkMTZiZmYxNWVlY2E3OGNmZjVhZjBjN2RkMTZmYSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCa8AKiRgzWkRJ+HXLHGkBB69TSi1+8DCdYKpmzTmZ9eEVTgcww+OnrUXHggxMUuFECMGdNPUQrGRRk/DRfZ0BoRvSer7oFBlRGWaqyFEWZ5d3Z8H1Va2HQ04Qi6hCqRZY4dQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md index cecfd6bc..a93339b6 100644 --- a/skills/cuequivariance-torch/BENCHMARK.md +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -1,15 +1,13 @@ # Skill Benchmark: cuequivariance-torch -> ✅ **Overall verdict: PASS — Recommended for publication** +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. ## Evaluation Metadata - Skill: `cuequivariance-torch` -- Evaluation date: 2026-10-16 +- Evaluation date: 2026-10-18 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -35,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 83.3% (+33.3 points) | 50.0% → 50.0% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 90.1% → 84.2% (-5.9 points) | 81.8% → 91.3% (+9.5 points) | -| Efficiency | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | 58.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 74.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 58.3% (+8.3 points) | 50.0% → 41.7% (-8.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | +| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 92.2% → 92.9% (+0.7 points) | 82.4% → 91.7% (+9.3 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 58.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,604,027 | 5,037,482 | -433,455 | -8.60% | skill 6/6; base 6/6 | -| claude-code | torch-001 | 1,230,733 | 1,103,162 | +127,571 | +11.56% | skill 1/1; base 1/1 | -| claude-code | torch-002 | 543,013 | 848,884 | -305,871 | -36.03% | skill 1/1; base 1/1 | -| claude-code | torch-003 | 780,753 | 978,259 | -197,506 | -20.19% | skill 1/1; base 1/1 | -| claude-code | torch-004 | 686,592 | 1,355,474 | -668,882 | -49.35% | skill 1/1; base 1/1 | -| claude-code | torch-005 | 1,333,100 | 721,931 | +611,169 | +84.66% | skill 1/1; base 1/1 | -| claude-code | torch-006 | 29,836 | 29,772 | +64 | +0.21% | skill 1/1; base 1/1 | -| codex | All cases | 5,122,185 | 7,095,117 | -1,972,932 | -27.81% | skill 6/6; base 6/6 | -| codex | torch-001 | 1,462,137 | 2,800,219 | -1,338,082 | -47.78% | skill 1/1; base 1/1 | -| codex | torch-002 | 1,022,997 | 564,020 | +458,977 | +81.38% | skill 1/1; base 1/1 | -| codex | torch-003 | 761,771 | 1,198,690 | -436,919 | -36.45% | skill 1/1; base 1/1 | -| codex | torch-004 | 680,450 | 1,134,224 | -453,774 | -40.01% | skill 1/1; base 1/1 | -| codex | torch-005 | 1,181,156 | 1,384,401 | -203,245 | -14.68% | skill 1/1; base 1/1 | -| codex | torch-006 | 13,674 | 13,563 | +111 | +0.82% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 9,726,212 | 12,132,599 | -2,406,387 | -19.83% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,992,274 | 6,906,967 | -1,914,693 | -27.72% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 795,386 | 1,870,442 | -1,075,056 | -57.48% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 475,055 | 947,637 | -472,582 | -49.87% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 896,363 | 1,413,656 | -517,293 | -36.59% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 912,314 | 1,249,324 | -337,010 | -26.98% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 1,883,055 | 1,396,040 | +487,015 | +34.89% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 30,101 | 29,868 | +233 | +0.78% | skill 1/1; base 1/1 | +| codex | All cases | 5,514,111 | 7,798,241 | -2,284,130 | -29.29% | skill 6/6; base 6/6 | +| codex | torch-001 | 1,261,467 | 1,871,775 | -610,308 | -32.61% | skill 1/1; base 1/1 | +| codex | torch-002 | 1,070,595 | 970,046 | +100,549 | +10.37% | skill 1/1; base 1/1 | +| codex | torch-003 | 1,172,773 | 1,222,710 | -49,937 | -4.08% | skill 1/1; base 1/1 | +| codex | torch-004 | 556,283 | 2,789,080 | -2,232,797 | -80.05% | skill 1/1; base 1/1 | +| codex | torch-005 | 1,439,357 | 931,065 | +508,292 | +54.59% | skill 1/1; base 1/1 | +| codex | torch-006 | 13,636 | 13,565 | +71 | +0.52% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 10,506,385 | 14,705,208 | -4,198,823 | -28.55% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -78,7 +76,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -95,7 +93,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code +- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md index f0b6b7e5..d62e36bb 100644 --- a/skills/cuequivariance-torch/skill-card.md +++ b/skills/cuequivariance-torch/skill-card.md @@ -67,15 +67,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 83.3% (+33.3 points) | 50.0% → 50.0% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 90.1% → 84.2% (-5.9 points) | 81.8% → 91.3% (+9.5 points) | -| Efficiency | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | 58.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 74.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 58.3% (+8.3 points) | 50.0% → 41.7% (-8.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | +| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 92.2% → 92.9% (+0.7 points) | 82.4% → 91.7% (+9.3 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 58.1% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit a126d11 2026-10-16)
+0.13.0 (source: VERSION file, git commit 531da35 2026-10-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig index fbad218b..963b988f 100644 --- a/skills/cuequivariance-torch/skill.oms.sig +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZmY2NTQ3YTc0NzE5ZDIyN2MxZWMwYWZhMzA0MDZkOWQ5MTFjOWM1OGU0NjI2Y2EzMDkyN2JiNzdkMWJlYzRkNiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjk2NjJmMjFlYWUyMzRmNTIxNTkzZGU3YjU1NTYyMTkxZTA0NTdkNTgzNTBiM2E1YzVhNjhhYTQxYmIxOTA3N2YiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQ2YTRlOWM2Y2ExN2M3NGJjMWJkMmYzMmQ1MGZmYjlkNDQ0MTY1M2M2ZWNjMWQ0NTNmNjljZTQ0OGY2N2Y2OWQiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4NGNiODQ5MDgwOGE4NjUxNzlmMDBkZDc5M2Y0ZTI0OWY5MTVjYTIyYzY5YzVhYzE3ODRiZDA1ODEzOWJhZGVkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCAoci2cjG5CRptLA73JrqC3qhnuSlirfHbm4HaNG9fI3NbJKhqIYHkVJ1/IxmocmMCMQCF9i6X1sGirTFFErk8eSUd/6XROLthuYK/ivvBRwWjIcY7dxJnpmuT8Rwwiy/icW4=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWYzNDA4NDM0MDI5ODE3N2FlN2IxZmFkZTAzZWFiYmE3NTI4NTFkNDU4NDhmZmQ3OWJiOGRmOTZjZGU0M2E0MCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjdiNmZkNmU4ZjY3OWEwNThiOTIxNzA3OTRiZmMzYWE1ZWFkNDdmNjA0MWZlMDlmYjg3OTA5NTJiZDEyMWUwOGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0ZTdiZWUyMDBhN2NlOWViZjQ3MTQ3Mjk1NzI1MWQyZWIwNjczMmFkNmE5NTU4Mjk4ZWJhYTk4OTRlNGQzZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICJkY2M4ZDFhYmZmZWI1MzgwNzljNzg5YWI1ODhjNjI0Y2QzNjkwNTdiYzUyMDlkMjVkOWYzM2M1MzQzMTBkMDRlIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiOGJiMmI3MjJlYzdlOGNmZjkyMTJmNTc3ZTk5ODIxY2JiOWIzMDUyNjhkZTdjZjllMDAyNmUwODE2Njk2MGU5ZSIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCuKgz+vxIAZoKLxMwohkWxWJrj7Q4yN4nTBeSKISk1ZfPOUivEg5Wvs7PWrNqLwOsCMQCCyh8Q3ovPeeCZW5NW1HKI/bpCoyJQreon9og5496/zIlZZ9jlOfn2L6Ds6RFe8x4=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md index e7813c04..7ad70541 100644 --- a/skills/cuequivariance/BENCHMARK.md +++ b/skills/cuequivariance/BENCHMARK.md @@ -1,13 +1,15 @@ # Skill Benchmark: cuequivariance -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata - Skill: `cuequivariance` -- Evaluation date: 2026-10-16 +- Evaluation date: 2026-10-18 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -33,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 82.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 75.0% (-25.0 points) | 83.3% → 83.3% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | -| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 95.4% → 95.8% (+0.4 points) | 88.8% → 80.0% (-8.8 points) | -| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | 71.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | 82.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 83.3% (+33.3 points) | 83.3% → 66.7% (-16.6 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 90.0% → 98.8% (+8.8 points) | 86.7% → 92.1% (+5.4 points) | +| Efficiency | 75.6% — baseline ran, but no comparable score was available; uplift unavailable | 66.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,494,094 | 6,764,067 | -2,269,973 | -33.56% | skill 6/6; base 6/6 | -| claude-code | core-001 | 935,725 | 1,050,205 | -114,480 | -10.90% | skill 1/1; base 1/1 | -| claude-code | core-002 | 558,016 | 1,212,941 | -654,925 | -53.99% | skill 1/1; base 1/1 | -| claude-code | core-003 | 924,549 | 1,870,785 | -946,236 | -50.58% | skill 1/1; base 1/1 | -| claude-code | core-004 | 717,396 | 1,190,142 | -472,746 | -39.72% | skill 1/1; base 1/1 | -| claude-code | core-005 | 1,327,938 | 1,409,548 | -81,610 | -5.79% | skill 1/1; base 1/1 | -| claude-code | core-006 | 30,470 | 30,446 | +24 | +0.08% | skill 1/1; base 1/1 | -| codex | All cases | 2,407,936 | 3,244,738 | -836,802 | -25.79% | skill 6/6; base 6/6 | -| codex | core-001 | 472,531 | 660,932 | -188,401 | -28.51% | skill 1/1; base 1/1 | -| codex | core-002 | 626,920 | 924,625 | -297,705 | -32.20% | skill 1/1; base 1/1 | -| codex | core-003 | 561,064 | 742,408 | -181,344 | -24.43% | skill 1/1; base 1/1 | -| codex | core-004 | 644,768 | 817,679 | -172,911 | -21.15% | skill 1/1; base 1/1 | -| codex | core-005 | 88,355 | 85,281 | +3,074 | +3.60% | skill 1/1; base 1/1 | -| codex | core-006 | 14,298 | 13,813 | +485 | +3.51% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 6,902,030 | 10,008,805 | -3,106,775 | -31.04% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,991,254 | 7,417,446 | -2,426,192 | -32.71% | skill 6/6; base 6/6 | +| claude-code | core-001 | 1,552,225 | 1,548,257 | +3,968 | +0.26% | skill 1/1; base 1/1 | +| claude-code | core-002 | 596,064 | 1,089,910 | -493,846 | -45.31% | skill 1/1; base 1/1 | +| claude-code | core-003 | 872,820 | 1,146,419 | -273,599 | -23.87% | skill 1/1; base 1/1 | +| claude-code | core-004 | 663,184 | 865,437 | -202,253 | -23.37% | skill 1/1; base 1/1 | +| claude-code | core-005 | 1,276,221 | 2,736,942 | -1,460,721 | -53.37% | skill 1/1; base 1/1 | +| claude-code | core-006 | 30,740 | 30,481 | +259 | +0.85% | skill 1/1; base 1/1 | +| codex | All cases | 2,155,811 | 4,007,025 | -1,851,214 | -46.20% | skill 6/6; base 6/6 | +| codex | core-001 | 420,443 | 845,088 | -424,645 | -50.25% | skill 1/1; base 1/1 | +| codex | core-002 | 473,578 | 731,568 | -257,990 | -35.27% | skill 1/1; base 1/1 | +| codex | core-003 | 436,256 | 849,609 | -413,353 | -48.65% | skill 1/1; base 1/1 | +| codex | core-004 | 361,551 | 1,414,735 | -1,053,184 | -74.44% | skill 1/1; base 1/1 | +| codex | core-005 | 450,019 | 152,246 | +297,773 | +195.59% | skill 1/1; base 1/1 | +| codex | core-006 | 13,964 | 13,779 | +185 | +1.34% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 7,147,065 | 11,424,471 | -4,277,406 | -37.44% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -76,7 +78,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md index fec8335d..202d9c03 100644 --- a/skills/cuequivariance/skill-card.md +++ b/skills/cuequivariance/skill-card.md @@ -66,15 +66,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 82.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 75.0% (-25.0 points) | 83.3% → 83.3% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | -| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 95.4% → 95.8% (+0.4 points) | 88.8% → 80.0% (-8.8 points) | -| Efficiency | 71.9% — baseline ran, but no comparable score was available; uplift unavailable | 71.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | 82.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 83.3% (+33.3 points) | 83.3% → 66.7% (-16.6 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 90.0% → 98.8% (+8.8 points) | 86.7% → 92.1% (+5.4 points) | +| Efficiency | 75.6% — baseline ran, but no comparable score was available; uplift unavailable | 66.9% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit a126d11 2026-10-16)
+0.13.0 (source: VERSION file, git commit 531da35 2026-10-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig index b3e3e149..bc7c1ea6 100644 --- a/skills/cuequivariance/skill.oms.sig +++ b/skills/cuequivariance/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNGY4OTQ5Mjg3YzU5OGEwNTQ1YWNhZDJiZTEwMTRhOThmYzc5NWQ0YjFlYjdhYTIzZWYxZmRiZDE1ZDg0NzgyMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImE3M2Q0MDY3M2Y2YmE5OWUwNWFmMjUwYzMyNzE5YjllYjdiYmUwNTRhZjkwODY1NjVhZWJiMmZiNWM2ZThiMzUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2JhMmM4NzI1ZjQ5OTBlYWJkYTRhMGQyMGQ1YWI5MTk0ZDJhMmU2ZjlhNjc3YThjYjcxZTk4MjQxZDE3ZmY0YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJiOWNlNmY0NzU0YzFlN2FjMmQ1NDczYzIzMDhkYjc5NjNlOTEyOGIzNTUyNjYxZmEyMTQ1NDExMmQwMTU5ZWUzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMGUxZGZkYWRjOWRiZjE2MWY1MTBlMTdkY2VjNTIxNDJhM2NiNDFmMzYwNmY0MzVlZDdmY2ZmZjM5MTMyYzNhZiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMHWeZCI6Ch5LTSIuRPIAfBqMbSh8K2GDwwNKcXPYg3kpnM3R8AJUpI/YU1eo9RX/PgIwBlKBiPZ42Zj0L7oD3rr5HYgWs0vGzF7Tusz/RVB51XytmkEIM7hK2F2taCcNcgYk","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZThiZjQ2ZmFjOWY5OTg2ZDljZmVmNmM5OThjYWIyNTc3ZGQ2YTcwYmZmNTUzOGJhMjkzNjA3YzA1OWY4OTcwMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNDQ0MWIzYmE0YWY3MWUyZjQ4MzJkMzViMGMwZDdkZWMyM2Q3NTU2MjAxYTllNDE1NGNjMzE5YTk1YjM2Mjc2ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIzYmEyYzg3MjVmNDk5MGVhYmRhNGEwZDIwZDVhYjkxOTRkMmEyZTZmOWE2NzdhOGNiNzFlOTgyNDFkMTdmZjRiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogIjYxOGFkMWMxYTg5ZjE1ZGIxZGIxMTNiMGI5YmY0NDcxMzMwMTEyZTUzOWIxMGZkOGEyNGE3YzQ0ZDI5YjcwMDkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJjOWE5OTBkMDRiNzllZTU1YzVhY2UyMTRlZDJiNGU3NWRmMWZjYmMyNzA2NGNhM2YyOGM1ZGMwN2M3MjU0N2UwIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCUfn+Lyj3zDbpQwZ0GQ7uB2vc2xrKN9pGmZOfXBfmKrNdeFc2DO3WvLGWuPibeliUCMQC+7jCB5fMcfN0ZnCQ1GAdJROt3QqvbX4eC/m1zylN5Lmge7eOfhMLJ7/5CDnrGEd0=","keyid":""}]}} \ No newline at end of file From 800e98a6719efb2f003763251a9aad989a790c02 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Sun, 18 Oct 2026 19:54:54 -0700 Subject: [PATCH 07/19] Improve Torch skill environment recovery and executable examples Signed-off-by: Ohad Mosafi --- .github/tests/test_torch_skill_examples.py | 40 ++++----- skills/README.md | 5 +- skills/cuequivariance-torch/SKILL.md | 100 +++++++++++++++------ 3 files changed, 96 insertions(+), 49 deletions(-) diff --git a/.github/tests/test_torch_skill_examples.py b/.github/tests/test_torch_skill_examples.py index cdee2449..e42440ae 100644 --- a/.github/tests/test_torch_skill_examples.py +++ b/.github/tests/test_torch_skill_examples.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Run the published skill examples and check the trainable GNN's invariants.""" +"""Run each published recipe independently and check the trainable GNN.""" import importlib import re @@ -12,23 +12,20 @@ import cuequivariance as cue -@pytest.fixture(scope="module") -def skill_examples(): - skill = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" - return [ - compile(code, f"{skill}:example-{index}", "exec") - for index, code in enumerate( - re.findall(r"```python\n(.*?)\n```", skill.read_text(), re.S) - ) - ] +SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" +EXAMPLES = [ + compile(code, f"{SKILL}:example-{index}", "exec") + for index, code in enumerate( + re.findall(r"```python\n(.*?)\n```", SKILL.read_text(), re.S) + ) +] -@pytest.fixture(scope="module") -def skill_namespace(skill_examples): +@pytest.mark.parametrize("index", range(2, len(EXAMPLES) - 1)) +def test_published_cpu_example(index): namespace = {} - for code in skill_examples: - exec(code, namespace) - return namespace + exec(EXAMPLES[1], namespace) + exec(EXAMPLES[index], namespace) @pytest.mark.parametrize( @@ -39,9 +36,7 @@ def skill_namespace(skill_examples): ("torch", ImportError("PyTorch cannot be imported")), ], ) -def test_environment_probe_preserves_partial_results( - skill_examples, monkeypatch, failed_module, error -): +def test_environment_probe_preserves_partial_results(monkeypatch, failed_module, error): modules = { name: importlib.import_module(name) for name in ("torch", "cuequivariance", "cuequivariance_torch") @@ -54,7 +49,7 @@ def import_module(name): monkeypatch.setattr(importlib, "import_module", import_module) namespace = {} - exec(skill_examples[0], namespace) + exec(EXAMPLES[0], namespace) assert namespace["available"] == { name: module for name, module in modules.items() if name != failed_module } @@ -63,11 +58,14 @@ def import_module(name): } -def test_documented_gnn_is_trainable_and_equivariant(skill_namespace): +def test_documented_gnn_is_trainable_and_equivariant(): + namespace = {} + exec(EXAMPLES[1], namespace) + exec(EXAMPLES[-1], namespace) torch.manual_seed(41) irreps_in = cue.Irreps("SO3", "2x0+2x1") irreps_out = cue.Irreps("SO3", "3x0+1") - model = skill_namespace["SimpleGNN"](irreps_in, irreps_out).double() + model = namespace["SimpleGNN"](irreps_in, irreps_out).double() features = torch.randn(5, irreps_in.dim, dtype=torch.float64, requires_grad=True) vectors = torch.randn(6, 3, dtype=torch.float64, requires_grad=True) senders = torch.tensor([0, 1, 3, 0, 1, 2]) diff --git a/skills/README.md b/skills/README.md index f22e63a6..ee6b77d3 100644 --- a/skills/README.md +++ b/skills/README.md @@ -61,8 +61,9 @@ The tests under `.github/tests/test_*_skill_evals.py` verify the mathematical an API expectations behind the prompts. `test_jax_skill_examples.py` executes each published CPU example independently and checks layout conversion with duplicate irreps and split polynomials with additional batch axes, parity, and JIT. -`test_torch_skill_examples.py` executes the published Torch examples and checks -the GNN's determinism, equivariance, and gradients. These are maintenance checks, +`test_torch_skill_examples.py` executes each published CPU recipe independently, +including tensor-product backward, and checks the GNN's determinism, +equivariance, and gradients. These are maintenance checks, not live agent scores. To run them against a development checkout with the relevant frontend dependencies installed: diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md index b529e996..e343228d 100644 --- a/skills/cuequivariance-torch/SKILL.md +++ b/skills/cuequivariance-torch/SKILL.md @@ -11,23 +11,23 @@ metadata: ## Purpose -Use `cuequivariance_torch` (`cuet`) to execute `cuequivariance` (`cue`) descriptors with PyTorch autograd. This guide covers CPU reference calculations, CUDA backend selection, and equivariant layers. Use the project's existing framework and environment. +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_torch` (`cuet`) and PyTorch autograd: CPU references, CUDA backends, and equivariant layers. Preserve the project's framework and environment. ## Inputs -Required for the computation: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership. Derive these from the request or existing code and state any defaults. Graph operations also require source/destination counts and node/edge indexing. Optional inputs include a preferred backend or performance/export targets. Preserve the user's project constraints and installed API. +Required inputs: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership. Derive them from the request or code; state defaults. Graphs also need source/destination counts and indices. Optional inputs: backend, performance, or export targets. ## Instructions -1. Check the selected interpreter and imports once using the probe below. Reuse a working environment. A missing CUDA backend is usually a method-selection issue, not a reason to reinstall PyTorch. +1. Probe the selected interpreter once. Reuse working CPU PyTorch; a CUDA-kernel error on CPU calls for `method="naive"` at construction, not a new environment. 2. Set `method="naive"` for CPU calculations. Match module parameters, features, weights, and reference computations in device and dtype. Use float64 for small numerical and gradient checks. 3. Specify layout and weight ownership explicitly. Derive sizes and output order from the module/descriptor; do not guess multiplicities from a filter. -4. Adapt the relevant example to the task and run its requested checks together. Report the interpreter, shapes, numerical error, and gradient results actually observed. Investigate a failed check before broadening the implementation. -5. If an API differs from this guide, inspect that installed object's signature and source once. Adapt to the installed version or explain the required version; do not silently replace cuEquivariance with another library. +4. Run the relevant example and requested checks together. Retain exit status and printed shapes/errors/gradients. Assertions in unexecuted code do not prove a pass; report blocked checks as unrun. +5. For an API mismatch, inspect that installed object's signature/source once. Adapt to its version or explain the requirement; preserve the requested library. ## Prerequisites and environment reuse -Run this with the project's Python interpreter (including its existing virtual environment, if any). Check imports independently so one missing package does not hide a working CPU PyTorch: +Run this with the project's Python. Probe imports independently so one missing package does not hide working PyTorch: ```python import importlib @@ -50,23 +50,27 @@ if "torch" in available: Choose the next step from the probe: -- All imports work: continue with this interpreter and `method="naive"` for CPU; install nothing. -- PyTorch works but cuEquivariance is missing: use another existing project interpreter if it has both. Otherwise install only the missing cuEquivariance packages into the intended environment, honoring its version constraints. `cuequivariance-torch` is the distribution name; `cuequivariance_torch` is the import name. -- An import reports a missing shared library or binary mismatch: use the reported package/path to diagnose compatibility. This differs from an absent package; repeated installations or a backend change may not fix it. +- Imports work: retain this interpreter; install nothing. +- PyTorch works but cuEquivariance is missing: install only the missing frontend/core packages when allowed. Distribution: `cuequivariance-torch`; import: `cuequivariance_torch`. +- PyTorch is missing: check the project's configured interpreter (such as `.venv/bin/python`) before installing. If the requested existing installation is inaccessible, say so; a new environment cannot verify that installation. +- Shared-library/binary mismatch: diagnose the reported package/path and version compatibility; this differs from an absent package. -Use the same interpreter for pip and execution. Installing cuEquivariance does not require replacing an already working CPU PyTorch. Optional CUDA/optimization warnings alone are not installation failures. +Use the same interpreter for pip and execution. Honor the lockfile or supplied `scripts/requirements.txt`; keep checkout core/frontend revisions compatible. Retaining installed PyTorch takes priority over example install commands. Optional CUDA/optimization warnings are not installation failures. -If no usable project environment is available, or it cannot be modified normally, create a **new** environment when installation is appropriate. Preserve existing environments and system-package protection. On Linux/macOS, this fallback also works when `venv` cannot bootstrap pip through `ensurepip` (the host needs pip 22.3+): +If isolation is needed and installation is appropriate, create a **new unique** environment. This Linux/macOS recipe bypasses `ensurepip` using host pip 22.3+: ```bash +set -e cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") python3 -m venv --without-pip "$cueq_env" python3 -m pip --python "$cueq_env" install torch --index-url https://download.pytorch.org/whl/cpu python3 -m pip --python "$cueq_env" install cuequivariance-torch -"$cueq_env/bin/python" -c 'import torch, cuequivariance_torch; print(torch.__version__)' +"$cueq_env/bin/python" -c 'import sys, torch, cuequivariance_torch; print(sys.executable, torch.__version__, cuequivariance_torch.__version__)' ``` -Honor project version/source constraints and keep using the new environment's Python explicitly. Stop at the first failed setup command and diagnose its error; preserve the failure output. If pip is unavailable, use an existing manager such as `uv` or report the missing prerequisite. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). +For supplied requirements, replace both install commands with `python3 -m pip --python "$cueq_env" install -r /path/to/requirements.txt`. Execute with `"$cueq_env/bin/python"`; activation and embedded pip are unnecessary. Do not delete `.venv` or alter system packages to repair `ensurepip`; preserve existing directories. Without host pip, use an existing manager such as `uv`. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). + +On setup failure, retain the error and diagnose its cause before retrying. For an unavailable source, use a configured trusted mirror/cache; keep TLS verification enabled and avoid guessed wheel URLs. If still blocked, provide the code and name the unrun checks instead of repeating environment resets/network probes. ## Limitations and backends @@ -89,7 +93,7 @@ High-level defaults vary: `SphericalHarmonics` selects `uniform_1d`, `FullyConne ## Examples -The CPU examples below share these imports. Replace the small representations and tensors with the task's inputs; keep the layout, indexing, and gradient contracts. +Each CPU example runs independently with this preamble. Adapt its inputs while preserving layout, indexing, and gradient contracts; retain its printed results and any failure. ```python import numpy as np @@ -99,6 +103,31 @@ import cuequivariance_torch as cuet torch.manual_seed(0) dtype = torch.float64 +print("torch:", torch.__version__, "device: cpu", "dtype:", dtype, "method: naive") +``` + +### CPU tensor product and backward + +Select `naive` for **both** modules on CPU. This example uses trainable internal weights and requires no CUDA packages. + +```python +irreps = cue.Irreps("SO3", "0 + 1") +sh = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") +tp = cuet.FullyConnectedTensorProduct( + irreps, irreps, irreps, layout=cue.mul_ir, method="naive", dtype=dtype, +) +features = torch.randn(2, irreps.dim, dtype=dtype, requires_grad=True) +vectors = torch.tensor([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], + dtype=dtype, requires_grad=True) +result = tp(features, sh(vectors)) +assert result.device.type == "cpu" and result.shape == (2, irreps.dim) +result.square().sum().backward() +gradient_norms = {} +for name, tensor in [("features", features), ("vectors", vectors), *tp.named_parameters()]: + assert tensor.grad is not None and torch.isfinite(tensor.grad).all() + assert tensor.grad.abs().sum() > 0 + gradient_norms[name] = tensor.grad.norm().item() +print("CPU tensor product:", tuple(result.shape), "gradient norms:", gradient_norms) ``` ### Spherical harmonics and a NumPy reference @@ -108,14 +137,18 @@ dtype = torch.float64 ```python ls = [0, 1, 2] sh = cuet.SphericalHarmonics(ls, normalize=False, method="naive") -vectors = torch.randn(2, 3, dtype=dtype, requires_grad=True) +vectors = torch.tensor([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], + dtype=dtype, requires_grad=True) values = sh(vectors) descriptor = cue.descriptors.spherical_harmonics(cue.SO3(1), ls) -np.testing.assert_allclose( - values.detach().numpy(), descriptor(vectors.detach().numpy())[0], atol=1e-12 -) +reference = descriptor(vectors.detach().numpy())[0] +np.testing.assert_allclose(values.detach().numpy(), reference, atol=1e-12) assert values.shape == (2, sum(2 * l + 1 for l in ls)) -assert torch.autograd.gradcheck(sh, (vectors,)) +gradcheck_ok = torch.autograd.gradcheck(sh, (vectors,)) +assert gradcheck_ok +print("Spherical harmonics:", tuple(values.shape), "normalize: False", + "max error:", np.max(np.abs(values.detach().numpy() - reference)), + "gradcheck:", gradcheck_ok) ``` ### Linear: layout conversion and rotations @@ -125,7 +158,7 @@ Use identical explicit weights for a layout comparison. For row-vector features, ```python irreps_in = cue.Irreps("SO3", "3x0 + 2x1") irreps_out = cue.Irreps("SO3", "2x0 + 4x1") -options = dict(internal_weights=False, method="naive", dtype=dtype) +options = dict(internal_weights=False, shared_weights=True, method="naive", dtype=dtype) linear = cuet.Linear(irreps_in, irreps_out, layout=cue.mul_ir, **options) linear_ir = cuet.Linear(irreps_in, irreps_out, layout=cue.ir_mul, **options) to_ir = cuet.TransposeIrrepsLayout( @@ -137,18 +170,25 @@ to_mul = cuet.TransposeIrrepsLayout( x = torch.randn(2, irreps_in.dim, dtype=dtype, requires_grad=True) w = torch.randn(1, linear.weight_numel, dtype=dtype, requires_grad=True) y = linear(x, w) -torch.testing.assert_close(y, to_mul(linear_ir(to_ir(x), w))) +y_ir = to_mul(linear_ir(to_ir(x), w)) +torch.testing.assert_close(y, y_ir) rotation = np.array([0.2, -0.3, 0.1]) din, dout = [torch.from_numpy(cue.IrrepsAndLayout(r, cue.mul_ir).exp_map(rotation, [])) for r in (irreps_in, irreps_out)] -torch.testing.assert_close(linear(x @ din.T, w), y @ dout.T) -for g in torch.autograd.grad(y.square().sum(), (x, w)): +rotated, expected_rotated = linear(x @ din.T, w), y @ dout.T +torch.testing.assert_close(rotated, expected_rotated) +grads = torch.autograd.grad(y.square().sum(), (x, w)) +for g in grads: assert torch.isfinite(g).all() and g.abs().sum() > 0 +print("Linear:", tuple(y.shape), "shared external weights:", tuple(w.shape), + "layout error:", (y - y_ir).detach().abs().max().item(), + "rotation error:", (rotated - expected_rotated).detach().abs().max().item(), + "gradient norms:", [g.norm().item() for g in grads]) ``` ### Channelwise product with external per-example weights -The core descriptor takes `(weights, input1, input2)` and returns a list of outputs. The high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids an extra transpose in this reference. +The core descriptor takes `(weights, input1, input2)` and returns a list of outputs. The high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids an extra transpose in this reference. `filter_irreps_out` filters **types**, not multiplicities: report `tp.irreps_out` and retain that actual order in references and downstream layers. ```python a, b = cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1") @@ -164,8 +204,13 @@ descriptor = cue.descriptors.channelwise_tensor_product(a, b, keep) reference = descriptor(*(t.detach().numpy() for t in (w, x1, x2)))[0] np.testing.assert_allclose(y.detach().numpy(), reference, atol=1e-12) assert y.shape == (2, tp.irreps_out.dim) -for g in torch.autograd.grad(y.square().sum(), (x1, x2, w)): +grads = torch.autograd.grad(y.square().sum(), (x1, x2, w)) +for g in grads: assert torch.isfinite(g).all() and g.abs().sum() > 0 +print("Channelwise:", tuple(y.shape), "per-item weights:", tuple(w.shape), + "ir_mul output order:", tp.irreps_out, + "max error:", np.max(np.abs(y.detach().numpy() - reference)), + "gradient norms:", [g.norm().item() for g in grads]) ``` For a fully connected product, specify the desired output irreps in `cuet.FullyConnectedTensorProduct(a, b, irreps_out, method="naive", dtype=dtype)`. Its default weights are internal trainable parameters. Use the same external-weight flags when the caller or an edge network owns the weights. @@ -191,6 +236,10 @@ grads = torch.autograd.grad(actual.square().sum(), (nodes, w)) ref_grads = torch.autograd.grad(reference.square().sum(), (nodes, w)) for g, ref in zip(grads, ref_grads): torch.testing.assert_close(g, ref) +print("Gather/scatter:", tuple(actual.shape), + "forward error:", (actual - reference).detach().abs().max().item(), + "gradient errors:", [(g - r).abs().max().item() for g, r in zip(grads, ref_grads)], + "isolated destination:", actual[3].detach().tolist()) ``` Reuse the descriptor to build a numerical reference: its coefficients include normalization factors, so a hand-written matrix product without those factors can be wrong. For high-level channelwise gathering/scattering, use `indices_1`, `indices_2`, `indices_out`, and `size_out` instead of the primitive's dictionaries. @@ -255,7 +304,6 @@ These paths require compatible optional GPU packages and their own shape checks. | Missing CUDA kernels on CPU | Pass `method="naive"` at module construction; retain the installed CPU PyTorch. | | `opt_einsum_fx not available` warning | Optional optimization; continue the naive reference calculation without installing it. | | Missing import after installation | Compare `sys.executable`, module paths, and the interpreter used with pip. Reuse the working interpreter. | -| `ensurepip` unavailable / externally managed Python | Use a fresh environment and host `pip --python` as above; preserve existing environments and system packages. | | Unexpected keyword or missing descriptor method | Inspect the installed signature/version. Use its documented equivalent or explain the mismatch; avoid repeated blind API guesses. | | Correct shapes but wrong numerical values | Check per-block layout, output irrep order, normalization, and the shared/per-item weight convention. | | Scatter shape/error or wrong destination sums | Use operand index keys, integral indices, a tensor in `output_shapes`, and accumulation for repeated receivers. | From 40c003b1ccf2eb78c000a8438739c81ed49d3124 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Mon, 19 Oct 2026 03:50:16 +0000 Subject: [PATCH 08/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 54 +++++++++++------------ skills/cuequivariance-jax/skill-card.md | 12 ++--- skills/cuequivariance-jax/skill.oms.sig | 2 +- skills/cuequivariance-torch/BENCHMARK.md | 54 ++++++++++++----------- skills/cuequivariance-torch/skill-card.md | 14 +++--- skills/cuequivariance-torch/skill.oms.sig | 2 +- skills/cuequivariance/BENCHMARK.md | 44 +++++++++--------- skills/cuequivariance/skill-card.md | 14 +++--- skills/cuequivariance/skill.oms.sig | 2 +- 9 files changed, 99 insertions(+), 99 deletions(-) diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md index 0dcc5195..cdd63d6f 100644 --- a/skills/cuequivariance-jax/BENCHMARK.md +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -1,15 +1,13 @@ # Skill Benchmark: cuequivariance-jax -> ✅ **Overall verdict: PASS — Recommended for publication** +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. ## Evaluation Metadata - Skill: `cuequivariance-jax` -- Evaluation date: 2026-10-18 +- Evaluation date: 2026-10-19 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -35,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 75.0% (+25.0 points) | 83.3% → 83.3% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 89.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 87.9% (-10.0 points) | 82.9% → 92.9% (+10.0 points) | -| Efficiency | 78.9% — baseline ran, but no comparable score was available; uplift unavailable | 68.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 100.0% (+16.7 points) | 100.0% → 66.7% (-33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 96.7% (-1.2 points) | 97.9% → 92.9% (-5.0 points) | +| Efficiency | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 65.7% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,947,603 | 4,390,017 | +557,586 | +12.70% | skill 6/6; base 6/6 | -| claude-code | jax-001 | 1,583,310 | 908,185 | +675,125 | +74.34% | skill 1/1; base 1/1 | -| claude-code | jax-002 | 692,110 | 531,175 | +160,935 | +30.30% | skill 1/1; base 1/1 | -| claude-code | jax-003 | 1,202,926 | 819,390 | +383,536 | +46.81% | skill 1/1; base 1/1 | -| claude-code | jax-004 | 829,453 | 1,416,254 | -586,801 | -41.43% | skill 1/1; base 1/1 | -| claude-code | jax-005 | 610,198 | 685,455 | -75,257 | -10.98% | skill 1/1; base 1/1 | -| claude-code | jax-006 | 29,606 | 29,558 | +48 | +0.16% | skill 1/1; base 1/1 | -| codex | All cases | 1,936,562 | 2,876,893 | -940,331 | -32.69% | skill 6/6; base 6/6 | -| codex | jax-001 | 310,054 | 759,473 | -449,419 | -59.18% | skill 1/1; base 1/1 | -| codex | jax-002 | 273,170 | 469,747 | -196,577 | -41.85% | skill 1/1; base 1/1 | -| codex | jax-003 | 673,888 | 324,300 | +349,588 | +107.80% | skill 1/1; base 1/1 | -| codex | jax-004 | 611,120 | 1,265,573 | -654,453 | -51.71% | skill 1/1; base 1/1 | -| codex | jax-005 | 54,698 | 44,481 | +10,217 | +22.97% | skill 1/1; base 1/1 | -| codex | jax-006 | 13,632 | 13,319 | +313 | +2.35% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 6,884,165 | 7,266,910 | -382,745 | -5.27% | skill 12/12; base 12/12 | +| claude-code | All cases | 5,437,471 | 5,353,635 | +83,836 | +1.57% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 794,408 | 1,156,143 | -361,735 | -31.29% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 1,006,269 | 471,710 | +534,559 | +113.32% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 797,985 | 611,625 | +186,360 | +30.47% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 1,596,338 | 1,666,259 | -69,921 | -4.20% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 1,212,898 | 1,418,401 | -205,503 | -14.49% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 29,573 | 29,497 | +76 | +0.26% | skill 1/1; base 1/1 | +| codex | All cases | 1,725,111 | 3,333,306 | -1,608,195 | -48.25% | skill 6/6; base 6/6 | +| codex | jax-001 | 309,970 | 837,967 | -527,997 | -63.01% | skill 1/1; base 1/1 | +| codex | jax-002 | 263,600 | 398,498 | -134,898 | -33.85% | skill 1/1; base 1/1 | +| codex | jax-003 | 263,329 | 573,627 | -310,298 | -54.09% | skill 1/1; base 1/1 | +| codex | jax-004 | 777,827 | 1,456,163 | -678,336 | -46.58% | skill 1/1; base 1/1 | +| codex | jax-005 | 96,998 | 53,508 | +43,490 | +81.28% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,387 | 13,543 | -156 | -1.15% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 7,162,582 | 8,686,941 | -1,524,359 | -17.55% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -78,7 +76,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -95,7 +93,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-jax': 133 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent codex +- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md index f8a4bfbd..7c29b6b9 100644 --- a/skills/cuequivariance-jax/skill-card.md +++ b/skills/cuequivariance-jax/skill-card.md @@ -68,12 +68,12 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | 87.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 75.0% (+25.0 points) | 83.3% → 83.3% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 89.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 87.9% (-10.0 points) | 82.9% → 92.9% (+10.0 points) | -| Efficiency | 78.9% — baseline ran, but no comparable score was available; uplift unavailable | 68.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 100.0% (+16.7 points) | 100.0% → 66.7% (-33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 97.9% → 96.7% (-1.2 points) | 97.9% → 92.9% (-5.0 points) | +| Efficiency | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 65.7% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig index 58629ed8..33f7ae77 100644 --- a/skills/cuequivariance-jax/skill.oms.sig +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjdmNzBjZmU3NWQ4MDkzMDlmMTkyMjIwZTViODEwMjdhNDE0OWIyNjYxOWUzMzc0NTY1ODY4NmE4MThkYTg5YzgiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImNiY2FiYzQ1NWZmNWYxY2FiMWM4OTg5ZDVkNjhkMWM5MzA1MTI2NzIzODUzMjIwODZkMjc5ZTc5ZDljZGMxMTAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzJjMDkxNjA4ZTQ3NWJlMTFiMTY5YjU2YWJiY2E0ZGUwNmE0YjAyZmI5NDZmZjI1YmNmYzFlZTQzZTE5OWQ2YyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YjY1ZjFmODVjZGY1ZGQ2NDkyMjJkOGEzNWQ4ZmY4Mzg2N2Y3MWU3NDUzOWQxNGM5Y2JiZWI4MWVlNDQ5OTg1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZTFkN2JkM2YzY2JiOTdjMzljZWY1ZWU3ZGQyODFmMTVkMmE1YWRhMjQ0ZDViNTQzMDc2NjIyNDEyMGY2ZGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTRhYzI5MTEyZmVjZWM2M2I0ODVjMzZmM2RiZjU4ODY3MWFkMTZiZmYxNWVlY2E3OGNmZjVhZjBjN2RkMTZmYSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCa8AKiRgzWkRJ+HXLHGkBB69TSi1+8DCdYKpmzTmZ9eEVTgcww+OnrUXHggxMUuFECMGdNPUQrGRRk/DRfZ0BoRvSer7oFBlRGWaqyFEWZ5d3Z8H1Va2HQ04Qi6hCqRZY4dQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQzMDJlZWRhODM3YmIzZmMzMGU4ZDBhOTBmNTNmOTc4N2ZjMDcyNmY3ZjEyYjlmZjk4MDY5OWI3MGFlMWE1ZDkiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGRkNGUwMmIxMWIyNzVhNGQ4M2FhNmU0MzRmYThkNGU2MmVhZmFlMDYyZDQ3MmRmZTYzNDkyMTFlMzQzM2QwMCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzJjMDkxNjA4ZTQ3NWJlMTFiMTY5YjU2YWJiY2E0ZGUwNmE0YjAyZmI5NDZmZjI1YmNmYzFlZTQzZTE5OWQ2YyIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZlMWQ3YmQzZjNjYmI5N2MzOWNlZjVlZTdkZDI4MWYxNWQyYTVhZGEyNDRkNWI1NDMwNzY2MjI0MTIwZjZkZTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjM5YzcwZDkxOGVjODBjZGYyY2M1OGM1YmY2MTVlN2Q1NTY4ZjcwYzllNDU4YTBjOTU4NWZiMDQ4ZDY5NzI3MGMiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMAE0ZPTdg57WtHYqRIpM251bpJKAvpL/a8/pe0MYAXpuAnhGTcEMQoTkbCKMOPml5QIxAI1Jn3NOiyTG7AvuH5dQ72KKjWcpbK8OlsycBBwB7LkyzVIOAKXAzrJeAh0HPNXN0g==","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md index a93339b6..26a8ac56 100644 --- a/skills/cuequivariance-torch/BENCHMARK.md +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -1,13 +1,15 @@ # Skill Benchmark: cuequivariance-torch -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata - Skill: `cuequivariance-torch` -- Evaluation date: 2026-10-18 +- Evaluation date: 2026-10-19 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -33,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 74.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 58.3% (+8.3 points) | 50.0% → 41.7% (-8.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | -| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 92.2% → 92.9% (+0.7 points) | 82.4% → 91.7% (+9.3 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 58.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 83.3% (+50.0 points) | 50.0% → 83.3% (+33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 85.4% → 88.8% (+3.4 points) | 88.9% → 68.1% (-20.8 points) | +| Efficiency | 76.9% — baseline ran, but no comparable score was available; uplift unavailable | 57.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,992,274 | 6,906,967 | -1,914,693 | -27.72% | skill 6/6; base 6/6 | -| claude-code | torch-001 | 795,386 | 1,870,442 | -1,075,056 | -57.48% | skill 1/1; base 1/1 | -| claude-code | torch-002 | 475,055 | 947,637 | -472,582 | -49.87% | skill 1/1; base 1/1 | -| claude-code | torch-003 | 896,363 | 1,413,656 | -517,293 | -36.59% | skill 1/1; base 1/1 | -| claude-code | torch-004 | 912,314 | 1,249,324 | -337,010 | -26.98% | skill 1/1; base 1/1 | -| claude-code | torch-005 | 1,883,055 | 1,396,040 | +487,015 | +34.89% | skill 1/1; base 1/1 | -| claude-code | torch-006 | 30,101 | 29,868 | +233 | +0.78% | skill 1/1; base 1/1 | -| codex | All cases | 5,514,111 | 7,798,241 | -2,284,130 | -29.29% | skill 6/6; base 6/6 | -| codex | torch-001 | 1,261,467 | 1,871,775 | -610,308 | -32.61% | skill 1/1; base 1/1 | -| codex | torch-002 | 1,070,595 | 970,046 | +100,549 | +10.37% | skill 1/1; base 1/1 | -| codex | torch-003 | 1,172,773 | 1,222,710 | -49,937 | -4.08% | skill 1/1; base 1/1 | -| codex | torch-004 | 556,283 | 2,789,080 | -2,232,797 | -80.05% | skill 1/1; base 1/1 | -| codex | torch-005 | 1,439,357 | 931,065 | +508,292 | +54.59% | skill 1/1; base 1/1 | -| codex | torch-006 | 13,636 | 13,565 | +71 | +0.52% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 10,506,385 | 14,705,208 | -4,198,823 | -28.55% | skill 12/12; base 12/12 | +| claude-code | All cases | 5,775,302 | 7,088,640 | -1,313,338 | -18.53% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 1,469,023 | 2,210,485 | -741,462 | -33.54% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 969,772 | 859,972 | +109,800 | +12.77% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 1,406,878 | 1,501,971 | -95,093 | -6.33% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 1,164,531 | 894,121 | +270,410 | +30.24% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 735,274 | 1,592,286 | -857,012 | -53.82% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 29,824 | 29,805 | +19 | +0.06% | skill 1/1; base 1/1 | +| codex | All cases | 7,380,609 | 6,306,245 | +1,074,364 | +17.04% | skill 6/6; base 6/6 | +| codex | torch-001 | 3,116,920 | 1,992,382 | +1,124,538 | +56.44% | skill 1/1; base 1/1 | +| codex | torch-002 | 770,021 | 799,219 | -29,198 | -3.65% | skill 1/1; base 1/1 | +| codex | torch-003 | 1,043,846 | 970,765 | +73,081 | +7.53% | skill 1/1; base 1/1 | +| codex | torch-004 | 842,750 | 557,473 | +285,277 | +51.17% | skill 1/1; base 1/1 | +| codex | torch-005 | 1,593,490 | 1,972,843 | -379,353 | -19.23% | skill 1/1; base 1/1 | +| codex | torch-006 | 13,582 | 13,563 | +19 | +0.14% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 13,155,911 | 13,394,885 | -238,974 | -1.78% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -76,7 +78,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -93,7 +95,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md index d62e36bb..99a95b11 100644 --- a/skills/cuequivariance-torch/skill-card.md +++ b/skills/cuequivariance-torch/skill-card.md @@ -67,15 +67,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 74.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 58.3% (+8.3 points) | 50.0% → 41.7% (-8.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 96.7% → 100.0% (+3.3 points) | -| Discoverability | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 92.2% → 92.9% (+0.7 points) | 82.4% → 91.7% (+9.3 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 58.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 33.3% → 83.3% (+50.0 points) | 50.0% → 83.3% (+33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 85.4% → 88.8% (+3.4 points) | 88.9% → 68.1% (-20.8 points) | +| Efficiency | 76.9% — baseline ran, but no comparable score was available; uplift unavailable | 57.8% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit 531da35 2026-10-18)
+0.13.0 (source: VERSION file, git commit 800e98a 2026-10-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig index 963b988f..a577c416 100644 --- a/skills/cuequivariance-torch/skill.oms.sig +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWYzNDA4NDM0MDI5ODE3N2FlN2IxZmFkZTAzZWFiYmE3NTI4NTFkNDU4NDhmZmQ3OWJiOGRmOTZjZGU0M2E0MCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjdiNmZkNmU4ZjY3OWEwNThiOTIxNzA3OTRiZmMzYWE1ZWFkNDdmNjA0MWZlMDlmYjg3OTA5NTJiZDEyMWUwOGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0ZTdiZWUyMDBhN2NlOWViZjQ3MTQ3Mjk1NzI1MWQyZWIwNjczMmFkNmE5NTU4Mjk4ZWJhYTk4OTRlNGQzZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiZGlnZXN0IjogIjkyYzg1YTFkZTNhZGE2NGJhODA4NzVkNDQ4NjYyOTQyZTNhNzgxMjM1NTc5NzYzYzU2MDAzZTFiY2EwMDU4OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICJkY2M4ZDFhYmZmZWI1MzgwNzljNzg5YWI1ODhjNjI0Y2QzNjkwNTdiYzUyMDlkMjVkOWYzM2M1MzQzMTBkMDRlIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiOGJiMmI3MjJlYzdlOGNmZjkyMTJmNTc3ZTk5ODIxY2JiOWIzMDUyNjhkZTdjZjllMDAyNmUwODE2Njk2MGU5ZSIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCuKgz+vxIAZoKLxMwohkWxWJrj7Q4yN4nTBeSKISk1ZfPOUivEg5Wvs7PWrNqLwOsCMQCCyh8Q3ovPeeCZW5NW1HKI/bpCoyJQreon9og5496/zIlZZ9jlOfn2L6Ds6RFe8x4=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYTI0ZmZlZjU0YjlkMjljNGI0NzE3MjIzYzE5NTQ2NzNkOTIyMmNlZDZkNGI2Zjc1Y2ExMzExMDVmNDk2MDhkOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZmFjZDIwY2UzM2M0MDM3ZDMwYjNlNzg0OGNkMmZjYWQxZGNmMmRmODRjNGY1ZWFmZTBhNDBiMjg3MmU2NmNiYiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4YmI3MTZmOGM3ZjViYmM2YzIwZWNiZDU4NDEyNDkyNTFiMmZmNTA4YjJlZjFiY2Y1Nzg2NTFmMTg3M2MzNDExIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogImE5NjliMzAxMTgzZDBmMDJiNzI4ZGY5YzVlYzY5ZGVkNjEyMDk2YzgyMzQzODVhZjcxYzg4YzBjYmMyMjYzNDAiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MTg3ZTdlNjQ1Njg3YjkzM2M1NWQzMWIwZWNiNTY4ZmFjY2UzZWQzOTRjZjdiZWQ0MTFlNWU4Zjk2MGU5MzI3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMECya8pMgE66bQgAK5MrrppLXHVHFRJV7c35z4ZWFzcut7+h97cI9vpZ864aCZXOOQIwKka5lFTwc1vzdTfdaEZwp86NE/S4eSQ0NG8r+s2lRQYJzEgApYCJ1q9KxAaMGye2","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md index 7ad70541..5e5f9f31 100644 --- a/skills/cuequivariance/BENCHMARK.md +++ b/skills/cuequivariance/BENCHMARK.md @@ -9,7 +9,7 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `cuequivariance` -- Evaluation date: 2026-10-18 +- Evaluation date: 2026-10-19 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -35,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | 82.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 83.3% (+33.3 points) | 83.3% → 66.7% (-16.6 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 90.0% → 98.8% (+8.8 points) | 86.7% → 92.1% (+5.4 points) | -| Efficiency | 75.6% — baseline ran, but no comparable score was available; uplift unavailable | 66.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Security | 33.3% → 83.3% (+50.0 points) | Not available | +| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | +| Discoverability | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Effectiveness | 95.0% → 92.5% (-2.5 points) | Not available | +| Efficiency | 67.7% — baseline ran, but no comparable score was available; uplift unavailable | Not available | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,991,254 | 7,417,446 | -2,426,192 | -32.71% | skill 6/6; base 6/6 | -| claude-code | core-001 | 1,552,225 | 1,548,257 | +3,968 | +0.26% | skill 1/1; base 1/1 | -| claude-code | core-002 | 596,064 | 1,089,910 | -493,846 | -45.31% | skill 1/1; base 1/1 | -| claude-code | core-003 | 872,820 | 1,146,419 | -273,599 | -23.87% | skill 1/1; base 1/1 | -| claude-code | core-004 | 663,184 | 865,437 | -202,253 | -23.37% | skill 1/1; base 1/1 | -| claude-code | core-005 | 1,276,221 | 2,736,942 | -1,460,721 | -53.37% | skill 1/1; base 1/1 | -| claude-code | core-006 | 30,740 | 30,481 | +259 | +0.85% | skill 1/1; base 1/1 | -| codex | All cases | 2,155,811 | 4,007,025 | -1,851,214 | -46.20% | skill 6/6; base 6/6 | -| codex | core-001 | 420,443 | 845,088 | -424,645 | -50.25% | skill 1/1; base 1/1 | -| codex | core-002 | 473,578 | 731,568 | -257,990 | -35.27% | skill 1/1; base 1/1 | -| codex | core-003 | 436,256 | 849,609 | -413,353 | -48.65% | skill 1/1; base 1/1 | -| codex | core-004 | 361,551 | 1,414,735 | -1,053,184 | -74.44% | skill 1/1; base 1/1 | -| codex | core-005 | 450,019 | 152,246 | +297,773 | +195.59% | skill 1/1; base 1/1 | -| codex | core-006 | 13,964 | 13,779 | +185 | +1.34% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 7,147,065 | 11,424,471 | -4,277,406 | -37.44% | skill 12/12; base 12/12 | +| claude-code | All cases | 4,249,937 | 8,293,717 | -4,043,780 | -48.76% | skill 6/6; base 6/6 | +| claude-code | core-001 | 999,874 | 2,053,017 | -1,053,143 | -51.30% | skill 1/1; base 1/1 | +| claude-code | core-002 | 729,147 | 920,821 | -191,674 | -20.82% | skill 1/1; base 1/1 | +| claude-code | core-003 | 809,317 | 2,045,454 | -1,236,137 | -60.43% | skill 1/1; base 1/1 | +| claude-code | core-004 | 747,604 | 774,446 | -26,842 | -3.47% | skill 1/1; base 1/1 | +| claude-code | core-005 | 933,319 | 2,470,959 | -1,537,640 | -62.23% | skill 1/1; base 1/1 | +| claude-code | core-006 | 30,676 | 29,020 | +1,656 | +5.71% | skill 1/1; base 1/1 | +| codex | All cases | 1,984,253 | 2,235,151 | -250,898 | -11.23% | skill 6/6; base 6/6 | +| codex | core-001 | 522,278 | 770,087 | -247,809 | -32.18% | skill 1/1; base 1/1 | +| codex | core-002 | 364,871 | 318,140 | +46,731 | +14.69% | skill 1/1; base 1/1 | +| codex | core-003 | 390,612 | 317,225 | +73,387 | +23.13% | skill 1/1; base 1/1 | +| codex | core-004 | 448,211 | 413,494 | +34,717 | +8.40% | skill 1/1; base 1/1 | +| codex | core-005 | 244,396 | 402,418 | -158,022 | -39.27% | skill 1/1; base 1/1 | +| codex | core-006 | 13,885 | 13,787 | +98 | +0.71% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 6,234,190 | 10,528,868 | -4,294,678 | -40.79% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md index 202d9c03..82846dff 100644 --- a/skills/cuequivariance/skill-card.md +++ b/skills/cuequivariance/skill-card.md @@ -66,15 +66,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | 82.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 83.3% (+33.3 points) | 83.3% → 66.7% (-16.6 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 90.0% → 98.8% (+8.8 points) | 86.7% → 92.1% (+5.4 points) | -| Efficiency | 75.6% — baseline ran, but no comparable score was available; uplift unavailable | 66.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 86.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Security | 33.3% → 83.3% (+50.0 points) | Not available | +| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | +| Discoverability | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Effectiveness | 95.0% → 92.5% (-2.5 points) | Not available | +| Efficiency | 67.7% — baseline ran, but no comparable score was available; uplift unavailable | Not available | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit 531da35 2026-10-18)
+0.13.0 (source: VERSION file, CHANGELOG released 2026-10-15; git commit 800e98a)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig index bc7c1ea6..aa326201 100644 --- a/skills/cuequivariance/skill.oms.sig +++ b/skills/cuequivariance/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZThiZjQ2ZmFjOWY5OTg2ZDljZmVmNmM5OThjYWIyNTc3ZGQ2YTcwYmZmNTUzOGJhMjkzNjA3YzA1OWY4OTcwMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNDQ0MWIzYmE0YWY3MWUyZjQ4MzJkMzViMGMwZDdkZWMyM2Q3NTU2MjAxYTllNDE1NGNjMzE5YTk1YjM2Mjc2ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIzYmEyYzg3MjVmNDk5MGVhYmRhNGEwZDIwZDVhYjkxOTRkMmEyZTZmOWE2NzdhOGNiNzFlOTgyNDFkMTdmZjRiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogIjYxOGFkMWMxYTg5ZjE1ZGIxZGIxMTNiMGI5YmY0NDcxMzMwMTEyZTUzOWIxMGZkOGEyNGE3YzQ0ZDI5YjcwMDkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJjOWE5OTBkMDRiNzllZTU1YzVhY2UyMTRlZDJiNGU3NWRmMWZjYmMyNzA2NGNhM2YyOGM1ZGMwN2M3MjU0N2UwIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCUfn+Lyj3zDbpQwZ0GQ7uB2vc2xrKN9pGmZOfXBfmKrNdeFc2DO3WvLGWuPibeliUCMQC+7jCB5fMcfN0ZnCQ1GAdJROt3QqvbX4eC/m1zylN5Lmge7eOfhMLJ7/5CDnrGEd0=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMTVkMWE4MWFlOTE3NTE5MjIyYjUwYjNkZmFlMGU2NGZkZTJjZWYxMDRhMTUyNzlhYWI3MzI0ZWNjZjVlNjE1OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5MzQ1MTM2ZDJmYWYwNDVhNmRhZDEwZGFmMDRjMDE0MjYwMWEwMzE0NzJmOGU1YWMxYTVkMzlhMGFlMzg2NzkiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNiYTJjODcyNWY0OTkwZWFiZGE0YTBkMjBkNWFiOTE5NGQyYTJlNmY5YTY3N2E4Y2I3MWU5ODI0MWQxN2ZmNGIiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyODA3OWEyN2QwZGVhMTY3MzMzOWQzOTU2OWZkYzQ0ODUzNWY2MjIyYjY0ZGRmNjhjMjMyMDQ2ZDI4YWIxMmY3IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC2mFvbJEnl9hj9zF8VT+PajMdPUtAsX0f+4WEVP0K0XzClIx1cC83BJ+Spijsl2RgCMHAgGFsCyyUb5Wz642/+s4rzjqvoPAY+xeiYD+FAVY54t8YtfAHry4AzEKfJeBApoA==","keyid":""}]}} \ No newline at end of file From af4c1c9f92f7a5414fc450e9a6b9f1cde72880a5 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 19 Oct 2026 09:14:58 -0700 Subject: [PATCH 09/19] Improve JAX skill setup safety and execution reliability Signed-off-by: Ohad Mosafi --- .github/tests/test_jax_skill_examples.py | 56 +++++++-- .github/tests/test_torch_skill_examples.py | 1 - skills/cuequivariance-jax/SKILL.md | 126 ++++++++++++--------- 3 files changed, 116 insertions(+), 67 deletions(-) diff --git a/.github/tests/test_jax_skill_examples.py b/.github/tests/test_jax_skill_examples.py index a955c523..9336bfe8 100644 --- a/.github/tests/test_jax_skill_examples.py +++ b/.github/tests/test_jax_skill_examples.py @@ -11,8 +11,6 @@ import pytest import cuequivariance as cue -import cuequivariance_jax as cuex -from cuequivariance_jax import ir_dict SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-jax/SKILL.md" EXAMPLES = [ @@ -67,22 +65,60 @@ def import_module(name): } -def test_layout_helper_with_duplicate_irreps_and_multiple_batch_axes(): - to_ir_mul = run_example(3)["dict_to_ir_mul_flat"] +@pytest.mark.parametrize("batch", [(), (2, 3)]) +def test_layout_helpers_with_duplicate_irreps(batch): + recipe = run_example(3) irreps = cue.Irreps("SO3", "2x1 + 0 + 1 + 2x2") - flat = jnp.arange(2 * 3 * irreps.dim).reshape(2, 3, irreps.dim) - regrouped = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() - blocks = ir_dict.flat_to_dict(regrouped.irreps, regrouped.array, layout="ir_mul") + flat = jnp.arange(int(np.prod(batch)) * irreps.dim).reshape(batch + (irreps.dim,)) + regrouped, blocks = recipe["ir_mul_to_dict"](irreps, flat) + assert regrouped == irreps.regroup() # Combine separated l=1 blocks along the channel axis, keeping every value. expected_vectors = np.concatenate( [ - np.asarray(flat[..., :6]).reshape(2, 3, 3, 2), - np.asarray(flat[..., 7:10]).reshape(2, 3, 3, 1), + np.asarray(flat[..., :6]).reshape(batch + (3, 2)), + np.asarray(flat[..., 7:10]).reshape(batch + (3, 1)), ], axis=-1, ).swapaxes(-1, -2) + np.testing.assert_array_equal(blocks[cue.SO3(0)], flat[..., 6:7, None]) np.testing.assert_array_equal(blocks[cue.SO3(1)], expected_vectors) - np.testing.assert_array_equal(to_ir_mul(regrouped.irreps, blocks), regrouped.array) + np.testing.assert_array_equal( + blocks[cue.SO3(2)], + np.asarray(flat[..., 10:]).reshape(batch + (5, 2)).swapaxes(-1, -2), + ) + expected_flat = np.concatenate( + [ + flat[..., 6:7], + expected_vectors.swapaxes(-1, -2).reshape(batch + (9,)), + flat[..., 10:], + ], + axis=-1, + ) + np.testing.assert_array_equal( + recipe["dict_to_ir_mul_flat"](regrouped, blocks), expected_flat + ) + + +def test_layout_regroup_keeps_opposite_parities_separate(): + recipe = run_example(3) + irreps = cue.Irreps("O3", "1o + 0e + 2x1e + 2x1o") + flat = jnp.arange(2 * irreps.dim).reshape(2, irreps.dim) + regrouped, blocks = recipe["ir_mul_to_dict"](irreps, flat) + assert regrouped == irreps.regroup() + assert set(blocks) == {cue.O3(0, 1), cue.O3(1, -1), cue.O3(1, 1)} + odd_vectors = np.concatenate( + [ + np.asarray(flat[:, :3]).reshape(2, 3, 1), + np.asarray(flat[:, 10:]).reshape(2, 3, 2), + ], + axis=-1, + ).swapaxes(-1, -2) + np.testing.assert_array_equal(blocks[cue.O3(1, -1)], odd_vectors) + np.testing.assert_array_equal( + blocks[cue.O3(1, 1)], + np.asarray(flat[:, 4:10]).reshape(2, 3, 2).swapaxes(-1, -2), + ) + np.testing.assert_array_equal(blocks[cue.O3(0, 1)], flat[:, 3:4, None]) @pytest.mark.parametrize("batch_shapes", [((), (), ()), ((), (2, 1, 1), (1, 3, 2))]) diff --git a/.github/tests/test_torch_skill_examples.py b/.github/tests/test_torch_skill_examples.py index e42440ae..41c17557 100644 --- a/.github/tests/test_torch_skill_examples.py +++ b/.github/tests/test_torch_skill_examples.py @@ -11,7 +11,6 @@ import cuequivariance as cue - SKILL = Path(__file__).resolve().parents[2] / "skills/cuequivariance-torch/SKILL.md" EXAMPLES = [ compile(code, f"{SKILL}:example-{index}", "exec") diff --git a/skills/cuequivariance-jax/SKILL.md b/skills/cuequivariance-jax/SKILL.md index 49605c85..9183815a 100644 --- a/skills/cuequivariance-jax/SKILL.md +++ b/skills/cuequivariance-jax/SKILL.md @@ -11,28 +11,29 @@ metadata: ## Purpose -Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_jax` (`cuex`), including autodiff, JIT, irrep dictionaries, and Flax NNX layers. Preserve the user's framework, environment, and device constraints. Custom representations and Clebsch–Gordan coefficients belong to the core `cuequivariance` API; PyTorch modules use `cuequivariance_torch`. +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_jax` (`cuex`). Preserve the project's framework and devices. Custom representations use core `cuequivariance`; PyTorch modules use `cuequivariance_torch`. ## Inputs -Required inputs are the operation, SO3/O3 irreps, layout, batch shapes, dtype, device, and weight ownership; derive them from the request or existing code and state defaults. Graph operations also need sender/receiver indices and destination count. Optional inputs include a preferred backend or performance target. Derive operand sizes and output irreps from descriptors rather than guessing multiplicities. +Required inputs: operation, SO3/O3 irreps, layout, batch shapes, dtype, device, and weight ownership. Derive them from the request/code; state defaults. Graphs also need indices and destination count. Operand sizes/output irreps come from descriptors. Optional inputs: backend/performance preferences. ## Instructions -1. Probe the selected interpreter once; reuse working packages. Choose `method="naive"` for CPU. Install only missing requirements, with Flax needed only for NNX layers. -2. Choose dense `RepArray` or `dict[Irrep, Array]`. High-level dictionary blocks are `(..., mul, ir.dim)`; the low-level uniform CUDA wrapper has a different shape contract. -3. Build the descriptor outside transformed functions. Derive output specs from its outputs and the broadcast batch shape. Enable x64 **before creating arrays** for float64 numerical checks. -4. Run the relevant small example with the selected interpreter, then adapt it. Compare against NumPy with identical operands; check JIT or derivatives when relevant. Report executed shapes/devices and any checks that could not run. +1. Probe the project interpreter once and reuse working packages. **Environment setup/recovery must not recursively delete directories**, including temporary environments. Use a fresh unique path when isolation is needed. +2. Select `method="naive"` for CPU; Flax is needed only for NNX. High-level dictionary blocks are `(..., mul, ir.dim)`; uniform CUDA arrays differ. +3. Adapt one matching recipe into a script. Build descriptors outside transformed functions, broadcast leading input axes, and enable x64 **before array creation** for float64 checks. +4. Run the script and requested checks together. Collect completion of any running session; retain exit status and printed shapes/errors. Report observed results and identify unrun checks. Assertions in unexecuted code are not a pass. +5. Investigate a concrete failure using its error and the affected installed API. Rerun affected checks after a fix. Once the requested checks pass, finish; avoid repeated probes, unrelated examples, or exploration of other backends for a CPU reference. ## Prerequisites and environment recovery -CPU polynomial execution needs JAX, `cuequivariance`, and `cuequivariance-jax`; it needs no CUDA kernel package. Check imports independently so one missing package does not hide a working JAX installation: +Probe imports independently so a missing frontend does not hide working JAX. For an NNX task, include `"flax.nnx"` in the same loop: ```python import importlib import sys -print("python:", sys.executable) +print("python:", sys.executable, sys.version.split()[0]) available, import_errors = {}, {} for name in ("jax", "cuequivariance", "cuequivariance_jax"): try: @@ -46,19 +47,21 @@ if "jax" in available: print("devices:", available["jax"].devices()) ``` -If these imports work, continue without reinstalling. For NNX, additionally check `from flax import nnx`. A binary/version import error is different from an absent package: inspect that error and the project's dependency constraints before changing versions. In a checkout, follow its installation instructions; keep core and frontend source revisions compatible. When supplied, honor `scripts/requirements.txt` beside this skill or the project's lockfile instead of substituting latest releases. +Reuse working packages and install missing CPU dependencies when appropriate. Honor the project's lockfile or supplied `scripts/requirements.txt` and compatible core/frontend revisions. JAX 0.11.1 / Flax 0.12.10 require **Python >=3.12**. Imports alone do not verify primitive compatibility: cuEquivariance 0.13.0 imports with JAX 0.12.0 but fails polynomial execution with a `segmented_polynomial_dce` argument-count error. The standalone setup below uses a tested combination; project requirements take precedence. -If an isolated environment is needed, create a **new unique directory**, preserving existing environments and project files. On Linux, this recipe bypasses `ensurepip` and uses host pip 22.3+: +For isolation, keep the unique directory returned by `mktemp`; reuse it for reruns. This Linux recipe uses host pip 22.3+ and bypasses `ensurepip`: ```bash set -e +cueq_python=python3.12 # Or the selected compatible interpreter's path. cueq_jax_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-jax.XXXXXX") -python3 -m venv --without-pip "$cueq_jax_env" -python3 -m pip --python "$cueq_jax_env" install "jax[cpu]" cuequivariance-jax +"$cueq_python" -m venv --without-pip "$cueq_jax_env" +python3 -m pip --python "$cueq_jax_env" install \ + "jax[cpu]==0.11.1" "cuequivariance==0.13.0" "cuequivariance-jax==0.13.0" JAX_PLATFORMS=cpu "$cueq_jax_env/bin/python" -c 'import jax, cuequivariance_jax; print(jax.devices())' ``` -Replace the install package list with `-r` and the supplied requirements path when versions are prescribed. Add Flax only for an NNX task unless those requirements already include it. Keep using the chosen interpreter explicitly. Stop on a setup failure and retain its error; do not delete `.venv` or change system packages to repair `ensurepip`. If host pip is unavailable, use an existing manager such as `uv`. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/) and [JAX installation](https://docs.jax.dev/en/latest/installation.html). +Replace the package list with `-r` and the requirements path when prescribed; otherwise add `flax==0.12.10` only for NNX. Run with `"$cueq_jax_env/bin/python"`, including the relevant numerical recipe after the import check. If host pip is unavailable, use an installed `uv`. Diagnose failures before retrying; preserve environments/system packages. Use trusted mirrors/caches with TLS verification enabled. If blocked, report unrun checks. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). ## Limitations and backend selection @@ -70,11 +73,11 @@ Replace the install package list with `-r` and the supplied requirements path wh | `method="uniform_1d"` | CUDA; uniform segment shapes within each operand and a single mode after preparation. | | `method="indexed_linear"`, `cuex.nnx.IrrepsIndexedLinear` | CUDA indexed linear route, using `cuex.Repeats`; not the ordinary CPU linear layer. | -A CPU-only task should use the first three routes without installing GPU dependencies. CUDA execution requires compatible JAX, GPU, and kernel builds; CPU checks do not establish CUDA compatibility or performance. Do not infer a CPU fallback from moving arrays to CPU. Avoid zero input vectors with NNX spherical harmonics at `eps=0`, since normalization divides by their norm; choose a positive `eps` if zero vectors are possible. +CPU routes need no GPU dependencies; moving arrays to CPU does not change the method. CUDA requires compatible JAX/GPU/kernel builds. NNX spherical harmonics normalize vectors: use positive `eps` if zero vectors are possible. ## Examples -Each example below runs with this common preamble; the NNX example adds Flax. For a CPU check, launch the script with `JAX_PLATFORMS=cpu`. The examples use small nonconstant arrays so layout errors remain visible. +Use this preamble with one recipe and run with `JAX_PLATFORMS=cpu`. Nonconstant inputs expose layout errors; prints follow successful assertions. ```python import jax @@ -91,7 +94,7 @@ print("devices:", jax.devices()) ### Spherical harmonics without Flax -The descriptor takes an input irrep and a list of degrees. This evaluates the **raw polynomial**; normalize vectors first when unit-direction harmonics are required. O3 keys retain parity, unlike SO3 keys. +This evaluates the **raw polynomial**; normalize vectors for unit-direction harmonics. O3 keys retain parity; SO3 keys do not. ```python desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2]) @@ -117,11 +120,16 @@ print({str(ir): (v.shape, v.devices()) for ir, v in harmonics.items()}) ### Dictionary layouts and exact round trips -`flat_to_dict(..., layout="ir_mul")` transposes each block to `(..., mul, ir.dim)`. **`dict_to_flat` has no layout argument and always emits `mul_ir`.** Returning to `ir_mul` requires a transpose within every block; a whole-array reshape changes the meaning of the data. +`flat_to_dict(..., layout="ir_mul")` produces `(..., mul, ir.dim)` blocks. **`dict_to_flat` has no layout argument and emits `mul_ir`.** Return to `ir_mul` by transposing each block, not reshaping the whole array. -**Regroup duplicate irreps before using irrep keys**, or later blocks overwrite earlier ones. Regroup the corresponding feature data too: changing only the irreps metadata does not reorder an existing buffer. A `RepArray` can regroup metadata and data together. Use descriptor/irreps order when assembling dictionaries and operands. +Explain the **unique-irrep key precondition**, even for regrouped inputs. Regroup duplicate blocks **with their data** using `RepArray.regroup()`; changing only metadata corrupts alignment. The round trip below recovers the regrouped buffer. Recovering an ungrouped order also needs its original block arrangement. ```python +def ir_mul_to_dict(irreps, flat): + # Merge duplicate irrep keys and reorder their data together. + data = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() + return data.irreps, ir_dict.flat_to_dict(data.irreps, data.array, layout="ir_mul") + def dict_to_ir_mul_flat(irreps, blocks): return jnp.concatenate([ blocks[ir].swapaxes(-1, -2).reshape( @@ -132,7 +140,7 @@ def dict_to_ir_mul_flat(irreps, blocks): irreps = cue.Irreps("SO3", "3x0 + 2x1").regroup() flat_ir = jnp.arange(2 * irreps.dim).reshape(2, irreps.dim) -blocks = ir_dict.flat_to_dict(irreps, flat_ir, layout="ir_mul") +irreps, blocks = ir_mul_to_dict(irreps, flat_ir) offset = 0 for mul, ir in irreps: size = mul * ir.dim @@ -145,13 +153,14 @@ restored_blocks = ir_dict.flat_to_dict(irreps, flat_mul, layout="mul_ir") for ir in blocks: np.testing.assert_array_equal(restored_blocks[ir], blocks[ir]) np.testing.assert_array_equal(dict_to_ir_mul_flat(irreps, restored_blocks), flat_ir) +print("exact ir_mul/mul_ir round trip:", {str(ir): v.shape for ir, v in blocks.items()}) ``` -`ir_dict.assert_mul_ir_dict` checks block conventions; `irreps_add` and `irreps_zeros_like` operate on matching dictionaries. `mul_ir_dict(irreps, data)` broadcasts a template object to each key; it does not infer feature dimensions. +`ir_dict.assert_mul_ir_dict` checks block conventions. `mul_ir_dict(irreps, data)` broadcasts a template to keys without inferring feature dimensions. ### Dense polynomial: broadcasting, JIT, and a gradient -`segmented_polynomial` takes a list of inputs and returns a list of outputs. Channelwise products put weights first. Shared weights need **no batch axis**; broadcast all input shapes except the final feature axis. `filter_irreps_out` selects irrep types, not output multiplicities. Dense descriptor operands carry their layout in `e.inputs` and `e.outputs`. +`segmented_polynomial` takes/returns lists. Channelwise products put weights first; shared weights need **no batch axis**. Broadcast leading input axes. `filter_irreps_out` selects types, not multiplicities; `e.inputs`/`e.outputs` carry dense operand layouts. ```python e = cue.descriptors.channelwise_tensor_product( @@ -174,8 +183,9 @@ def run(weights, a, b): )[0] reference = e(*(np.asarray(v) for v in (w, x, y)))[0] -np.testing.assert_allclose(run(w, x, y), reference, rtol=1e-12, atol=1e-12) -np.testing.assert_allclose(jax.jit(run)(w, x, y), reference, rtol=1e-12, atol=1e-12) +eager, compiled = run(w, x, y), jax.jit(run)(w, x, y) +np.testing.assert_allclose(eager, reference, rtol=1e-12, atol=1e-12) +np.testing.assert_allclose(compiled, reference, rtol=1e-12, atol=1e-12) assert reference.shape == (3, 2, poly.outputs[0].size) gradient = jax.grad(lambda v: jnp.sum(run(v, x, y) ** 2))(w) index = int(jnp.argmax(jnp.abs(gradient))) @@ -183,15 +193,18 @@ assert abs(gradient[index]) > 1e-8 epsilon = 1e-5 loss_plus = np.sum(e(np.asarray(w.at[index].add(epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) loss_minus = np.sum(e(np.asarray(w.at[index].add(-epsilon)), np.asarray(x), np.asarray(y))[0] ** 2) -np.testing.assert_allclose(gradient[index], (loss_plus - loss_minus) / (2 * epsilon), - rtol=1e-6, atol=1e-7) +finite_difference = (loss_plus - loss_minus) / (2 * epsilon) +np.testing.assert_allclose(gradient[index], finite_difference, rtol=1e-6, atol=1e-7) +print("shape:", eager.shape, "eager/JIT max errors:", + float(jnp.max(abs(eager - reference))), float(jnp.max(abs(compiled - reference)))) +print("weight gradient / finite difference:", float(gradient[index]), finite_difference) ``` -For gather/scatter, supply one `indices` entry per polynomial operand, inputs then outputs; `None` leaves an operand unindexed. Output indices require an explicit destination shape. Repeated destinations accumulate. Compare against an edgewise calculation with explicit summation before optimizing a graph contraction. +For gather/scatter, supply one `indices` entry per operand, inputs then outputs; `None` means unindexed. Output indices need a destination shape; repeated destinations accumulate. Verify against explicit edgewise summation. ### Split irrep polynomials on CPU -`channelwise_tensor_product_ir_dict` returns an `IrDictPolynomial`. Its `input_irreps` groups are weights, features, and the second input; each `(mul, ir)` becomes one polynomial operand. Its `output_irreps` gives the split output order. These split descriptors use `ir_mul`: transpose the high-level blocks before flattening, then reverse that conversion on outputs. There is no `ir_dict.equivariant_polynomial` entrypoint. +`channelwise_tensor_product_ir_dict` returns an `IrDictPolynomial`. Follow `input_irreps` order: weights, features, second input; each `(mul, ir)` is one operand. Follow `output_irreps` for outputs. These descriptors use `ir_mul`: transpose blocks before flattening; reverse on outputs. There is no `ir_dict.equivariant_polynomial` API. ```python def split_naive(desc, groups): @@ -231,13 +244,15 @@ numpy_inputs = [ expected = desc.polynomial(*numpy_inputs) for (mul, ir), flat in zip(desc.output_irreps[0], expected): assert result[ir].shape == (3, 2, mul, ir.dim) - np.testing.assert_allclose(result[ir], flat.reshape(3, 2, ir.dim, mul).swapaxes(-1, -2), - rtol=1e-12, atol=1e-12) + reference = flat.reshape(3, 2, ir.dim, mul).swapaxes(-1, -2) + np.testing.assert_allclose(result[ir], reference, rtol=1e-12, atol=1e-12) + print("output:", str(ir), result[ir].shape, "max error:", + float(jnp.max(abs(result[ir] - reference)))) ``` ### Flax NNX layers -`IrrepsLinear` requires regrouped irreps and explicit `nnx.Rngs`. It mixes channels only within the same irrep, with `scale / sqrt(mul_in)` normalization. An output irrep absent from the input receives zeros, not a learned cross-irrep mapping. `nnx.Param` weights can be read with `[...]` for an independent reference. Row-vector features rotate as `x @ D.T`. +`IrrepsLinear` requires regrouped irreps and `nnx.Rngs`. It mixes matching-irrep channels with `scale / sqrt(mul_in)` normalization; unmatched outputs are zero. Read `nnx.Param` weights with `[...]`. Row vectors rotate as `x @ D.T`. ```python from flax import nnx @@ -253,29 +268,27 @@ for mul, ir in irreps_out: assert output[ir].shape == (2, mul, ir.dim) np.testing.assert_array_equal(output[cue.SO3(2)], 0) for mul, ir in irreps_in: - expected = np.einsum("uv,...ui->...vi", np.asarray(linear.w[str(ir)][...]), features[ir]) - np.testing.assert_allclose(output[ir], expected * linear.scale / np.sqrt(mul), - rtol=1e-12, atol=1e-12) + factor = linear.scale / np.sqrt(mul) + expected = factor * np.einsum( + "uv,...ui->...vi", np.asarray(linear.w[str(ir)][...]), features[ir]) + np.testing.assert_allclose(output[ir], expected, rtol=1e-12, atol=1e-12) + print("irrep:", str(ir), "scale/sqrt(mul_in):", factor, + "reference max error:", float(jnp.max(abs(output[ir] - expected)))) rotation = np.array([0.3, -0.2, 0.1]) rotated_output = linear({ir: v @ ir.exp_map(rotation, []).T for ir, v in features.items()}) for ir, value in output.items(): - np.testing.assert_allclose(rotated_output[ir], value @ ir.exp_map(rotation, []).T, - rtol=1e-12, atol=1e-12) - -vectors = jnp.array([[0.3, -0.7, 1.1], [1.2, 0.4, -0.2]], dtype=dtype) -sh = cue_nnx.SphericalHarmonics(max_degree=2) -harmonics = sh(vectors) -for ir, value in harmonics.items(): - assert value.shape == (2, 1, ir.dim) - assert bool(jnp.isfinite(value).all()) -print({str(ir): (v.shape, v.devices()) for ir, v in harmonics.items()}) + expected = value @ ir.exp_map(rotation, []).T + np.testing.assert_allclose(rotated_output[ir], expected, rtol=1e-12, atol=1e-12) + print("output:", str(ir), value.shape, "rotation max error:", + float(jnp.max(abs(rotated_output[ir] - expected)))) +print("unmatched l=2 max magnitude:", float(jnp.max(abs(output[cue.SO3(2)])))) ``` -`IrrepsNormalize(eps=..., scale=..., skip_scalars=...)` normalizes using the sum of squared components averaged over multiplicities. `MLP(layer_sizes=..., activation=..., dtype=..., rngs=...)` operates on scalar features with fan-in normalization. Use `IrrepsIndexedLinear` only for the supported CUDA route: species counts must match grouped input rows and their sum must match the batch size. +`IrrepsNormalize(eps=..., scale=..., skip_scalars=...)` uses squared component norms averaged over multiplicities. `MLP(layer_sizes=..., activation=..., dtype=..., rngs=...)` uses scalar features and fan-in normalization. CUDA indexed-linear species counts must match grouped rows and sum to batch size. ### Dense RepArray interface -Use representation metadata from the descriptor to avoid a layout mismatch. A single output normally returns one `RepArray`; its `.array` is the JAX array and `.reps` maps axes to representations. +Use descriptor representation metadata. A single output returns one `RepArray`: `.array` holds data; `.reps` maps axes to representations. ```python e = cue.descriptors.fully_connected_tensor_product( @@ -287,25 +300,26 @@ inputs = [cuex.randn(key, rep, (2,), dtype) for key, rep in zip(keys, e.inputs)] output = cuex.equivariant_polynomial(e, inputs, method="naive") np.testing.assert_allclose(output.array, e(*(np.asarray(v.array) for v in inputs))[0], rtol=1e-12, atol=1e-12) +print("RepArray shape:", output.shape, "devices:", output.array.devices()) ``` ## CUDA preparation and message passing -For a compatible dense descriptor, `e.squeeze_modes().flatten_coefficient_modes()` prepares its modes before `equivariant_polynomial(..., method="uniform_1d")`. Check uniform segment shapes within each operand. Splitting by irrep separates buffers; it does **not** remove the need to convert their layouts. +Prepare compatible dense descriptors with `e.squeeze_modes().flatten_coefficient_modes()` before `equivariant_polynomial(..., method="uniform_1d")`. Segment shapes must be uniform within each operand. Splitting by irrep still requires layout conversion. -`ir_dict.segmented_polynomial_uniform_1d(poly, inputs, outputs, input_indices=..., output_indices=...)` expects pytree leaves shaped `(..., op.num_segments, *op.segment_shape)`. This is not the high-level `(..., mul, ir.dim)` convention. For each `_ir_dict` operand, transpose and flatten as in `split_naive`, then reshape using that operand's segment metadata. Ensure pytree leaf order matches polynomial operand order. Output specs need the same segment suffix. Reverse both reshapes and the transpose before passing results into `IrrepsLinear`. +`ir_dict.segmented_polynomial_uniform_1d(poly, inputs, outputs, input_indices=..., output_indices=...)` needs leaves shaped `(..., op.num_segments, *op.segment_shape)`, distinct from `(..., mul, ir.dim)`. For `_ir_dict` operands, transpose/flatten as in `split_naive`, then reshape using segment metadata. Match leaf order to operand order and output specs to segment suffixes. Reverse conversions before `IrrepsLinear`. -For GNN message passing, weight arrays belong to edges, features to source nodes, and spherical harmonics to edge directions. Supply gather indices for node features (senders) and scatter indices for outputs (receivers); destination count determines output batch size. Construct learned weights or an edge-weight network once in the NNX module initializer. Use explicit split random keys for independent inputs. The source examples `cuequivariance_jax/examples/nequip_nnx.py` and `mace_nnx.py` show complete architectures and CUDA execution. +GNN weights belong to edges, features to nodes, harmonics to edge directions. Gather by senders, scatter by receivers, and size output by destination count. Initialize learned weights/edge networks once in the NNX initializer; split RNG keys for independent inputs. Source examples: `cuequivariance_jax/examples/nequip_nnx.py` and `mace_nnx.py`. ## Troubleshooting | Symptom | Check | | --- | --- | -| `ensurepip` unavailable or a partial environment exists | Preserve it; use a new directory with `--without-pip` and host pip `--python`, or an installed environment manager. | -| Missing CUDA kernels on CPU | Select `naive` or ordinary NNX layers. Changing the device alone does not change the method. | -| Float64 truncated or numerical comparison fails | Enable x64 before array creation, use the same dtype/normalization in both computations, then inspect layout and operand order. | -| Dictionary shape/value mismatch | Regroup duplicate irreps with their data, preserve irrep order, and transpose per block. `dict_to_flat` emits `mul_ir`. | -| Missing output or incorrect broadcast | One output spec per polynomial output; broadcast leading axes of all inputs, including weights. | -| An API is missing in the installed version | Inspect that symbol's signature/source and installed versions; do not invent keyword arguments or reinstall the entire stack blindly. | - -Locate installed source via `cuex.__file__`. In a checkout, relevant files under `cuequivariance_jax/cuequivariance_jax/` are `ir_dict.py`, `nnx.py`, `equivariant_polynomial.py`, `rep_array/rep_array_.py`, and `segmented_polynomials/segmented_polynomial*.py`. Package API documentation is at https://docs.nvidia.com/cuda/cuequivariance/. +| `ensurepip` unavailable | Use a new unique directory, `--without-pip`, and host pip `--python`; preserve partial environments. | +| Missing CUDA kernels on CPU | Select `naive` or ordinary NNX layers. | +| Float64 truncated / numerical mismatch | Check early x64, identical operands, dtype/normalization, layout, and operand order. | +| Dictionary mismatch | Regroup data with metadata; transpose per block. `dict_to_flat` emits `mul_ir`. | +| Missing output / bad broadcast | Specify every output; broadcast leading axes including weights. | +| Missing API / unexpected keyword | Inspect that installed symbol's signature/source and version. | + +Locate source via `cuex.__file__`; relevant files are `ir_dict.py`, `nnx.py`, `equivariant_polynomial.py`, `rep_array/rep_array_.py`, and `segmented_polynomials/segmented_polynomial*.py`. API docs: https://docs.nvidia.com/cuda/cuequivariance/. From d9721f255682b6e9abd5c07ba342b4288caefb2d Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Mon, 19 Oct 2026 17:27:58 +0000 Subject: [PATCH 10/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 42 +++++++++--------- skills/cuequivariance-jax/skill-card.md | 27 ++++++------ skills/cuequivariance-jax/skill.oms.sig | 2 +- skills/cuequivariance-torch/BENCHMARK.md | 52 +++++++++++------------ skills/cuequivariance-torch/skill-card.md | 14 +++--- skills/cuequivariance-torch/skill.oms.sig | 2 +- skills/cuequivariance/BENCHMARK.md | 52 ++++++++++++----------- skills/cuequivariance/skill-card.md | 37 ++++++++-------- skills/cuequivariance/skill.oms.sig | 2 +- 9 files changed, 116 insertions(+), 114 deletions(-) diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md index cdd63d6f..d97403a1 100644 --- a/skills/cuequivariance-jax/BENCHMARK.md +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -33,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 100.0% (+16.7 points) | 100.0% → 66.7% (-33.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | -| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 96.7% (-1.2 points) | 97.9% → 92.9% (-5.0 points) | -| Efficiency | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 65.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.8% — baseline ran, but no comparable score was available; uplift unavailable | 83.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 100.0% (+16.7 points) | 50.0% → 83.3% (+33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 94.2% (-1.6 points) | 91.7% → 78.3% (-13.4 points) | +| Efficiency | 76.1% — baseline ran, but no comparable score was available; uplift unavailable | 69.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 5,437,471 | 5,353,635 | +83,836 | +1.57% | skill 6/6; base 6/6 | -| claude-code | jax-001 | 794,408 | 1,156,143 | -361,735 | -31.29% | skill 1/1; base 1/1 | -| claude-code | jax-002 | 1,006,269 | 471,710 | +534,559 | +113.32% | skill 1/1; base 1/1 | -| claude-code | jax-003 | 797,985 | 611,625 | +186,360 | +30.47% | skill 1/1; base 1/1 | -| claude-code | jax-004 | 1,596,338 | 1,666,259 | -69,921 | -4.20% | skill 1/1; base 1/1 | -| claude-code | jax-005 | 1,212,898 | 1,418,401 | -205,503 | -14.49% | skill 1/1; base 1/1 | -| claude-code | jax-006 | 29,573 | 29,497 | +76 | +0.26% | skill 1/1; base 1/1 | -| codex | All cases | 1,725,111 | 3,333,306 | -1,608,195 | -48.25% | skill 6/6; base 6/6 | -| codex | jax-001 | 309,970 | 837,967 | -527,997 | -63.01% | skill 1/1; base 1/1 | -| codex | jax-002 | 263,600 | 398,498 | -134,898 | -33.85% | skill 1/1; base 1/1 | -| codex | jax-003 | 263,329 | 573,627 | -310,298 | -54.09% | skill 1/1; base 1/1 | -| codex | jax-004 | 777,827 | 1,456,163 | -678,336 | -46.58% | skill 1/1; base 1/1 | -| codex | jax-005 | 96,998 | 53,508 | +43,490 | +81.28% | skill 1/1; base 1/1 | -| codex | jax-006 | 13,387 | 13,543 | -156 | -1.15% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 7,162,582 | 8,686,941 | -1,524,359 | -17.55% | skill 12/12; base 12/12 | +| claude-code | All cases | 2,977,893 | 6,629,286 | -3,651,393 | -55.08% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 527,791 | 1,099,326 | -571,535 | -51.99% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 561,852 | 404,850 | +157,002 | +38.78% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 793,159 | 788,453 | +4,706 | +0.60% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 625,633 | 3,461,747 | -2,836,114 | -81.93% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 439,809 | 845,382 | -405,573 | -47.98% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 29,649 | 29,528 | +121 | +0.41% | skill 1/1; base 1/1 | +| codex | All cases | 2,124,104 | 3,306,453 | -1,182,349 | -35.76% | skill 6/6; base 6/6 | +| codex | jax-001 | 644,663 | 876,262 | -231,599 | -26.43% | skill 1/1; base 1/1 | +| codex | jax-002 | 487,638 | 394,272 | +93,366 | +23.68% | skill 1/1; base 1/1 | +| codex | jax-003 | 440,548 | 541,449 | -100,901 | -18.64% | skill 1/1; base 1/1 | +| codex | jax-004 | 425,932 | 1,438,609 | -1,012,677 | -70.39% | skill 1/1; base 1/1 | +| codex | jax-005 | 111,915 | 42,431 | +69,484 | +163.76% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,408 | 13,430 | -22 | -0.16% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 5,101,997 | 9,935,739 | -4,833,742 | -48.65% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md index 7c29b6b9..83096f9c 100644 --- a/skills/cuequivariance-jax/skill-card.md +++ b/skills/cuequivariance-jax/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers use this skill when writing or debugging JAX code with cuequivariance to execute equivariant polynomials on CPU or GPU, work with dict[Irrep, Array] or RepArray data and layouts, and build Flax NNX equivariant layers for geometric neural networks such as NequIP- and MACE-style GNNs.
+Developers and engineers building equivariant geometric neural networks use this skill to execute cuEquivariance descriptors with cuequivariance_jax, including segmented polynomials, irrep dictionary layouts, Flax NNX layers, and CPU or CUDA backend selection, with executable numerical checks.
### Deployment Geography for Use:
Global
@@ -26,16 +26,15 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
-- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
-- [JAX installation](https://docs.jax.dev/en/latest/installation.html)
-- [pip interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/)
+- [pip interpreter selection (--python option)](https://pip.pypa.io/en/stable/topics/python-option/)
+- [scripts/requirements.txt](scripts/requirements.txt)
## Skill Output:
**Output Type(s):** [Code, Shell commands, Analysis]
**Output Format:** [Markdown with inline Python and bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Reports observed execution results (exit status, printed shapes, errors) and identifies unrun checks.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
@@ -44,7 +43,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod and compared against a no-skill baseline.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod; evaluator version 1.5.6, evaluated 2026-10-19. Overall verdict: NEUTRAL.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -52,7 +51,7 @@ Reported benchmark dimensions:
- Correctness: Is the answer correct? Scored from the `accuracy` signal.
- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
-- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` plus 50% `token_efficiency`.
+- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` and 50% `token_efficiency`.
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
@@ -68,15 +67,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 100.0% (+16.7 points) | 100.0% → 66.7% (-33.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | -| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 97.9% → 96.7% (-1.2 points) | 97.9% → 92.9% (-5.0 points) | -| Efficiency | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 65.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.8% — baseline ran, but no comparable score was available; uplift unavailable | 83.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 100.0% (+16.7 points) | 50.0% → 83.3% (+33.3 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 94.2% (-1.6 points) | 91.7% → 78.3% (-13.4 points) | +| Efficiency | 76.1% — baseline ran, but no comparable score was available; uplift unavailable | 69.3% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION, CHANGELOG, released 2026-10-15)
+af4c1c9 (source: git SHA, committed 2026-10-19)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig index 33f7ae77..d8401b6d 100644 --- a/skills/cuequivariance-jax/skill.oms.sig +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQzMDJlZWRhODM3YmIzZmMzMGU4ZDBhOTBmNTNmOTc4N2ZjMDcyNmY3ZjEyYjlmZjk4MDY5OWI3MGFlMWE1ZDkiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGRkNGUwMmIxMWIyNzVhNGQ4M2FhNmU0MzRmYThkNGU2MmVhZmFlMDYyZDQ3MmRmZTYzNDkyMTFlMzQzM2QwMCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzJjMDkxNjA4ZTQ3NWJlMTFiMTY5YjU2YWJiY2E0ZGUwNmE0YjAyZmI5NDZmZjI1YmNmYzFlZTQzZTE5OWQ2YyIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIiwKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZlMWQ3YmQzZjNjYmI5N2MzOWNlZjVlZTdkZDI4MWYxNWQyYTVhZGEyNDRkNWI1NDMwNzY2MjI0MTIwZjZkZTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjM5YzcwZDkxOGVjODBjZGYyY2M1OGM1YmY2MTVlN2Q1NTY4ZjcwYzllNDU4YTBjOTU4NWZiMDQ4ZDY5NzI3MGMiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMAE0ZPTdg57WtHYqRIpM251bpJKAvpL/a8/pe0MYAXpuAnhGTcEMQoTkbCKMOPml5QIxAI1Jn3NOiyTG7AvuH5dQ72KKjWcpbK8OlsycBBwB7LkyzVIOAKXAzrJeAh0HPNXN0g==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQyYTQ1NzQ4YmJhNzNlMmE2N2JkODI4Y2VhY2M4NWM0MjgyODBlZjQzZTVhYzIxOWQxOTYzNzU0MWIzYjM3YjQiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJlOTQxZmFjOTEzNTc3YTM4NDM5NTAwMjYxZjFiZjc3ZmEwOWQyMzhmNTE5NjllMmZhMDY3M2FmNGJhNjcyNmY0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjQ5NTU1MzY2MjAyNzdkY2UyYjY5Yzk2NGIzYTNhMDMyMjE5ODA0MzJiNTYwYmViNDQwYTNmOWFiZWI3ODliZTEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJkaWdlc3QiOiAiZmUxZDdiZDNmM2NiYjk3YzM5Y2VmNWVlN2RkMjgxZjE1ZDJhNWFkYTI0NGQ1YjU0MzA3NjYyMjQxMjBmNmRlNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjE4YTI3NjhhNmJjYWYxOGQyMDVmYzhhN2U0ZmY5MzBhZjNiNjFjZjdhYzRhYWNiMDM4NGVmZDJkMjEzZDU2ODEiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDeZhFR5vm9Qd41EAREoJ68XYH/qS0dFT9oMoJRhFrwxnTMFvTd6VMt2VcsS8eIYmQCME56gLCnF9DMpmhlQxiH/XPvvYvSr0wmDcvN/7ClUiKWjdBIlEL6939r+AW+Yq8/Xg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md index 26a8ac56..bc226f6e 100644 --- a/skills/cuequivariance-torch/BENCHMARK.md +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -1,10 +1,8 @@ # Skill Benchmark: cuequivariance-torch -> ✅ **Overall verdict: PASS — Recommended for publication** +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. ## Evaluation Metadata @@ -35,12 +33,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 83.3% (+50.0 points) | 50.0% → 83.3% (+33.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 85.4% → 88.8% (+3.4 points) | 88.9% → 68.1% (-20.8 points) | -| Efficiency | 76.9% — baseline ran, but no comparable score was available; uplift unavailable | 57.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 75.0% → 100.0% (+25.0 points) | 83.3% → 66.7% (-16.6 points) | +| Correctness | 100.0% → 93.3% (-6.7 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 88.8% → 86.4% (-2.4 points) | 80.7% → 72.5% (-8.2 points) | +| Efficiency | 75.9% — baseline ran, but no comparable score was available; uplift unavailable | 60.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +52,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 5,775,302 | 7,088,640 | -1,313,338 | -18.53% | skill 6/6; base 6/6 | -| claude-code | torch-001 | 1,469,023 | 2,210,485 | -741,462 | -33.54% | skill 1/1; base 1/1 | -| claude-code | torch-002 | 969,772 | 859,972 | +109,800 | +12.77% | skill 1/1; base 1/1 | -| claude-code | torch-003 | 1,406,878 | 1,501,971 | -95,093 | -6.33% | skill 1/1; base 1/1 | -| claude-code | torch-004 | 1,164,531 | 894,121 | +270,410 | +30.24% | skill 1/1; base 1/1 | -| claude-code | torch-005 | 735,274 | 1,592,286 | -857,012 | -53.82% | skill 1/1; base 1/1 | -| claude-code | torch-006 | 29,824 | 29,805 | +19 | +0.06% | skill 1/1; base 1/1 | -| codex | All cases | 7,380,609 | 6,306,245 | +1,074,364 | +17.04% | skill 6/6; base 6/6 | -| codex | torch-001 | 3,116,920 | 1,992,382 | +1,124,538 | +56.44% | skill 1/1; base 1/1 | -| codex | torch-002 | 770,021 | 799,219 | -29,198 | -3.65% | skill 1/1; base 1/1 | -| codex | torch-003 | 1,043,846 | 970,765 | +73,081 | +7.53% | skill 1/1; base 1/1 | -| codex | torch-004 | 842,750 | 557,473 | +285,277 | +51.17% | skill 1/1; base 1/1 | -| codex | torch-005 | 1,593,490 | 1,972,843 | -379,353 | -19.23% | skill 1/1; base 1/1 | -| codex | torch-006 | 13,582 | 13,563 | +19 | +0.14% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 13,155,911 | 13,394,885 | -238,974 | -1.78% | skill 12/12; base 12/12 | +| claude-code | All cases | 5,223,194 | 6,398,169 | -1,174,975 | -18.36% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 975,100 | 1,392,092 | -416,992 | -29.95% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 1,453,016 | 1,741,134 | -288,118 | -16.55% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 1,492,508 | 1,175,512 | +316,996 | +26.97% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 411,116 | 966,688 | -555,572 | -57.47% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 861,581 | 1,092,854 | -231,273 | -21.16% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 29,873 | 29,889 | -16 | -0.05% | skill 1/1; base 1/1 | +| codex | All cases | 4,231,771 | 5,027,418 | -795,647 | -15.83% | skill 6/6; base 6/6 | +| codex | torch-001 | 1,646,932 | 1,677,817 | -30,885 | -1.84% | skill 1/1; base 1/1 | +| codex | torch-002 | 739,871 | 612,047 | +127,824 | +20.88% | skill 1/1; base 1/1 | +| codex | torch-003 | 608,840 | 840,494 | -231,654 | -27.56% | skill 1/1; base 1/1 | +| codex | torch-004 | 786,916 | 531,744 | +255,172 | +47.99% | skill 1/1; base 1/1 | +| codex | torch-005 | 435,590 | 1,351,661 | -916,071 | -67.77% | skill 1/1; base 1/1 | +| codex | torch-006 | 13,622 | 13,655 | -33 | -0.24% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 9,454,965 | 11,425,587 | -1,970,622 | -17.25% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -78,7 +76,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -95,7 +93,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code +- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md index 99a95b11..5fdf3a3f 100644 --- a/skills/cuequivariance-torch/skill-card.md +++ b/skills/cuequivariance-torch/skill-card.md @@ -67,15 +67,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 33.3% → 83.3% (+50.0 points) | 50.0% → 83.3% (+33.3 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 85.4% → 88.8% (+3.4 points) | 88.9% → 68.1% (-20.8 points) | -| Efficiency | 76.9% — baseline ran, but no comparable score was available; uplift unavailable | 57.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 75.0% → 100.0% (+25.0 points) | 83.3% → 66.7% (-16.6 points) | +| Correctness | 100.0% → 93.3% (-6.7 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 88.8% → 86.4% (-2.4 points) | 80.7% → 72.5% (-8.2 points) | +| Efficiency | 75.9% — baseline ran, but no comparable score was available; uplift unavailable | 60.9% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit 800e98a 2026-10-18)
+0.13.0 (source: VERSION file, git commit af4c1c9 2026-10-19)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig index a577c416..9c4d0265 100644 --- a/skills/cuequivariance-torch/skill.oms.sig +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYTI0ZmZlZjU0YjlkMjljNGI0NzE3MjIzYzE5NTQ2NzNkOTIyMmNlZDZkNGI2Zjc1Y2ExMzExMDVmNDk2MDhkOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZmFjZDIwY2UzM2M0MDM3ZDMwYjNlNzg0OGNkMmZjYWQxZGNmMmRmODRjNGY1ZWFmZTBhNDBiMjg3MmU2NmNiYiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4YmI3MTZmOGM3ZjViYmM2YzIwZWNiZDU4NDEyNDkyNTFiMmZmNTA4YjJlZjFiY2Y1Nzg2NTFmMTg3M2MzNDExIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiZGlnZXN0IjogImE5NjliMzAxMTgzZDBmMDJiNzI4ZGY5YzVlYzY5ZGVkNjEyMDk2YzgyMzQzODVhZjcxYzg4YzBjYmMyMjYzNDAiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MTg3ZTdlNjQ1Njg3YjkzM2M1NWQzMWIwZWNiNTY4ZmFjY2UzZWQzOTRjZjdiZWQ0MTFlNWU4Zjk2MGU5MzI3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMECya8pMgE66bQgAK5MrrppLXHVHFRJV7c35z4ZWFzcut7+h97cI9vpZ864aCZXOOQIwKka5lFTwc1vzdTfdaEZwp86NE/S4eSQ0NG8r+s2lRQYJzEgApYCJ1q9KxAaMGye2","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYjFkNjZmNmU1OTZlMTZhMTFhOGY4ZDY0OTg3NzU4NTU1Nzk0N2MyMzUzMTBmMWE0NDQzYjRjMTExOTg1Y2Y0OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjM1YjhmOTUwZjhkMmJiODc0MTkyZjc2YjI5YmQxY2VkODI0Mzk0NTRkOTQ3ODhmNDhjYzk3YmYyMTg1M2FjZjQiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhiYjcxNmY4YzdmNWJiYzZjMjBlY2JkNTg0MTI0OTI1MWIyZmY1MDhiMmVmMWJjZjU3ODY1MWYxODczYzM0MTEiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMmExYWJlM2Y0N2EyY2MwOWIzMjA3ZTQxYWM4NzM2ZWJhNWVlMzMxOTBiOWM2YjAxZDE3YmNiYjNlZjAwYjBkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDQ9SoymEBxNnx0WT5+siyJd9+6nMd+FYS9wK3IoUwz/OHnXcqXjkMMm+iUM20UmWACMQC8bcrdtVBO3r8ZtwouswAeURFCViReU9fadmElnWg3fjtx5a05i3V8QoeDReFQkEs=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md index 5e5f9f31..122d5f91 100644 --- a/skills/cuequivariance/BENCHMARK.md +++ b/skills/cuequivariance/BENCHMARK.md @@ -35,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Security | 33.3% → 83.3% (+50.0 points) | Not available | -| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | -| Discoverability | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Effectiveness | 95.0% → 92.5% (-2.5 points) | Not available | -| Efficiency | 67.7% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Overall | 85.5% — baseline ran, but no comparable score was available; uplift unavailable | 91.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 83.3% (±0.0 points) | 83.3% → 100.0% (+16.7 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 94.2% → 92.9% (-1.3 points) | 88.3% → 99.2% (+10.9 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -54,21 +54,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 4,249,937 | 8,293,717 | -4,043,780 | -48.76% | skill 6/6; base 6/6 | -| claude-code | core-001 | 999,874 | 2,053,017 | -1,053,143 | -51.30% | skill 1/1; base 1/1 | -| claude-code | core-002 | 729,147 | 920,821 | -191,674 | -20.82% | skill 1/1; base 1/1 | -| claude-code | core-003 | 809,317 | 2,045,454 | -1,236,137 | -60.43% | skill 1/1; base 1/1 | -| claude-code | core-004 | 747,604 | 774,446 | -26,842 | -3.47% | skill 1/1; base 1/1 | -| claude-code | core-005 | 933,319 | 2,470,959 | -1,537,640 | -62.23% | skill 1/1; base 1/1 | -| claude-code | core-006 | 30,676 | 29,020 | +1,656 | +5.71% | skill 1/1; base 1/1 | -| codex | All cases | 1,984,253 | 2,235,151 | -250,898 | -11.23% | skill 6/6; base 6/6 | -| codex | core-001 | 522,278 | 770,087 | -247,809 | -32.18% | skill 1/1; base 1/1 | -| codex | core-002 | 364,871 | 318,140 | +46,731 | +14.69% | skill 1/1; base 1/1 | -| codex | core-003 | 390,612 | 317,225 | +73,387 | +23.13% | skill 1/1; base 1/1 | -| codex | core-004 | 448,211 | 413,494 | +34,717 | +8.40% | skill 1/1; base 1/1 | -| codex | core-005 | 244,396 | 402,418 | -158,022 | -39.27% | skill 1/1; base 1/1 | -| codex | core-006 | 13,885 | 13,787 | +98 | +0.71% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 6,234,190 | 10,528,868 | -4,294,678 | -40.79% | skill 12/12; base 12/12 | +| claude-code | All cases | 5,881,399 | 7,298,202 | -1,416,803 | -19.41% | skill 6/6; base 6/6 | +| claude-code | core-001 | 956,726 | 1,188,219 | -231,493 | -19.48% | skill 1/1; base 1/1 | +| claude-code | core-002 | 510,535 | 869,031 | -358,496 | -41.25% | skill 1/1; base 1/1 | +| claude-code | core-003 | 710,993 | 2,235,618 | -1,524,625 | -68.20% | skill 1/1; base 1/1 | +| claude-code | core-004 | 708,256 | 959,125 | -250,869 | -26.16% | skill 1/1; base 1/1 | +| claude-code | core-005 | 2,964,658 | 2,015,952 | +948,706 | +47.06% | skill 1/1; base 1/1 | +| claude-code | core-006 | 30,231 | 30,257 | -26 | -0.09% | skill 1/1; base 1/1 | +| codex | All cases | 2,400,367 | 2,780,675 | -380,308 | -13.68% | skill 6/6; base 6/6 | +| codex | core-001 | 720,238 | 494,550 | +225,688 | +45.64% | skill 1/1; base 1/1 | +| codex | core-002 | 474,477 | 618,515 | -144,038 | -23.29% | skill 1/1; base 1/1 | +| codex | core-003 | 429,875 | 645,457 | -215,582 | -33.40% | skill 1/1; base 1/1 | +| codex | core-004 | 356,473 | 692,463 | -335,990 | -48.52% | skill 1/1; base 1/1 | +| codex | core-005 | 405,363 | 315,871 | +89,492 | +28.33% | skill 1/1; base 1/1 | +| codex | core-006 | 13,941 | 13,819 | +122 | +0.88% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 8,281,766 | 10,078,877 | -1,797,111 | -17.83% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -77,7 +77,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| | Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 1 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -85,12 +85,16 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks +- **HIGH** DUPLICATE/duplicate: Duplicate content found within SKILL.md: + "### IrDictPolynomial descriptors" in SKILL.md (lines 247-257) + vs "### IrDictPolynomial" in SKILL.md (lines 287-301) + vs "### split_polynomial_by_irreps" in SKILL.md (lines 312-322) + vs "# Split an operand into per-irrep pieces (for ir_dict interface)" in SKILL.md (lines 336-338) (`SKILL.md:247`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance/SKILL.md`) - **LOW** QUALITY/quality_discoverability: Description very long (310 chars, recommend 50-150) (`skills/cuequivariance/SKILL.md`) - **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance/SKILL.md`) - **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No limitations documented (`skills/cuequivariance/SKILL.md`) -- 2 additional finding(s) are available in the full evaluation artifacts. +- 3 additional finding(s) are available in the full evaluation artifacts.
diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md index 82846dff..218b8628 100644 --- a/skills/cuequivariance/skill-card.md +++ b/skills/cuequivariance/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics).
+Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics) when working with cuequivariance group theory, irreps, or segmented polynomials.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and researchers building equivariant geometric neural networks use this skill to define irreps and custom symmetry groups, construct segmented tensor products and equivariant polynomial descriptors with the cuequivariance library, and validate them with CPU NumPy execution.
+Developers and engineers building equivariant geometric neural networks use this skill to define groups and irreps, construct segmented tensor products and equivariant polynomials with the cuequivariance library, and validate them with CPU NumPy execution.
### Deployment Geography for Use:
Global
@@ -25,8 +25,9 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
-- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance Documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance GitHub Repository](https://github.com/NVIDIA/cuEquivariance)
+- [scripts/requirements.txt](scripts/requirements.txt)
## Skill Output:
@@ -42,15 +43,15 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox; with-skill runs compared against no-skill baselines.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Is it safe to use?
-- Correctness: Is the answer correct?
-- Discoverability: Was the right skill loaded when needed?
-- Effectiveness: Did the skill help complete the task?
-- Efficiency: Did it avoid wasted tool calls and token usage?
+- Security: Is it safe to use? Scored from the `security` signal.
+- Correctness: Is the answer correct? Scored from the `accuracy` signal.
+- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
+- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` and 50% `token_efficiency`.
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
@@ -59,22 +60,22 @@ Underlying evaluation signals used in this run:
- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage.
+- `token_efficiency`: Actual uncached prompt plus completion token usage.
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 86.9% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Security | 33.3% → 83.3% (+50.0 points) | Not available | -| Correctness | 100.0% → 100.0% (±0.0 points) | Not available | -| Discoverability | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | Not available | -| Effectiveness | 95.0% → 92.5% (-2.5 points) | Not available | -| Efficiency | 67.7% — baseline ran, but no comparable score was available; uplift unavailable | Not available | +| Overall | 85.5% — baseline ran, but no comparable score was available; uplift unavailable | 91.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 83.3% (±0.0 points) | 83.3% → 100.0% (+16.7 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 94.2% → 92.9% (-1.3 points) | 88.3% → 99.2% (+10.9 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-0.13.0 (source: VERSION file, CHANGELOG released 2026-10-15; git commit 800e98a)
+af4c1c9 (source: git SHA, committed 2026-10-19)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig index aa326201..7f65490e 100644 --- a/skills/cuequivariance/skill.oms.sig +++ b/skills/cuequivariance/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMTVkMWE4MWFlOTE3NTE5MjIyYjUwYjNkZmFlMGU2NGZkZTJjZWYxMDRhMTUyNzlhYWI3MzI0ZWNjZjVlNjE1OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5MzQ1MTM2ZDJmYWYwNDVhNmRhZDEwZGFmMDRjMDE0MjYwMWEwMzE0NzJmOGU1YWMxYTVkMzlhMGFlMzg2NzkiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNiYTJjODcyNWY0OTkwZWFiZGE0YTBkMjBkNWFiOTE5NGQyYTJlNmY5YTY3N2E4Y2I3MWU5ODI0MWQxN2ZmNGIiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyODA3OWEyN2QwZGVhMTY3MzMzOWQzOTU2OWZkYzQ0ODUzNWY2MjIyYjY0ZGRmNjhjMjMyMDQ2ZDI4YWIxMmY3IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC2mFvbJEnl9hj9zF8VT+PajMdPUtAsX0f+4WEVP0K0XzClIx1cC83BJ+Spijsl2RgCMHAgGFsCyyUb5Wz642/+s4rzjqvoPAY+xeiYD+FAVY54t8YtfAHry4AzEKfJeBApoA==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZjFmNWUyNmI3NDIyZGEwNTlmNzQ4NjAxYzlmMzU5NTc0NThiZGIyNjBiNzYyMGQzMjdkZTcwYzM1YTIwNjMzMyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjEyNjg5MDI0MGJmYzJkOWU3ZThmOTg1NjVmYzVkMDU3MzE1NmU5MTk4N2YzOGI2YzcxNTA4NWNkNTZlOWJmOGMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNiYTJjODcyNWY0OTkwZWFiZGE0YTBkMjBkNWFiOTE5NGQyYTJlNmY5YTY3N2E4Y2I3MWU5ODI0MWQxN2ZmNGIiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiNDYwZTVhYmNkM2U2YWJkZDM4NDgzZTBiNzY2Mjc1ODJhMjQ4MjliODVjZmVjYjhlMWJkZjQyYzU0MGY4MzZjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMFyx+hockE2Zeybx5xB0kGUFv7gg6Uo697YKc9waXocvFJ5PmA0gvjJb6W57dRSZbgIwN7b9iCGzPLPepPT10WHSExXYk2hxsU8irPwQeL26CGQg9Uk0ao1+6VkTz2JReVIr","keyid":""}]}} \ No newline at end of file From c4c008658805db492bf832ee6eb197a12943d1a2 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 19 Oct 2026 15:37:52 -0700 Subject: [PATCH 11/19] Improve JAX and Torch Tier 3 evaluation reliability Signed-off-by: Ohad Mosafi --- .github/tests/test_skill_eval_setup.py | 47 ++++++++ .github/tests/test_skill_execution.py | 103 ++++++++++++++++++ .github/workflows/tests.yml | 2 +- skills/cuequivariance-jax/SKILL.md | 39 +++++-- skills/cuequivariance-jax/evals/config.yml | 45 +++++++- .../scripts/requirements.txt | 10 +- skills/cuequivariance-torch/SKILL.md | 82 +++++++++----- skills/cuequivariance-torch/evals/config.yml | 40 ++++++- .../scripts/requirements.txt | 6 +- skills/cuequivariance/SKILL.md | 46 +++----- 10 files changed, 338 insertions(+), 82 deletions(-) create mode 100644 .github/tests/test_skill_eval_setup.py create mode 100644 .github/tests/test_skill_execution.py diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py new file mode 100644 index 00000000..2300054e --- /dev/null +++ b/.github/tests/test_skill_eval_setup.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pre-agent environment failures must stop evaluation without replacing data.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +def test_pre_agent_setup_preserves_existing_incompatible_environment( + tmp_path, frontend +): + config_path = ( + Path(__file__).resolve().parents[2] + / "skills" + / f"cuequivariance-{frontend}" + / "evals" + / "config.yml" + ) + config = yaml.safe_load(config_path.read_text()) + env_dir = tmp_path / ".venv" + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(env_dir)], + check=True, + timeout=30, + ) + sentinel = env_dir / "keep.txt" + sentinel.write_text("existing environment") + original_config = (env_dir / "pyvenv.cfg").read_bytes() + command = "set -euo pipefail\n" + "\n".join(config["harbor"]["pre_agent_setup"]) + result = subprocess.run( + ["bash", "-c", command], + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert "ModuleNotFoundError" in result.stderr + assert "CPU environment ready" not in result.stdout + assert sentinel.read_text() == "existing environment" + assert (env_dir / "pyvenv.cfg").read_bytes() == original_config + assert not list(tmp_path.glob(".cueq-env.*")) diff --git a/.github/tests/test_skill_execution.py b/.github/tests/test_skill_execution.py new file mode 100644 index 00000000..8389f314 --- /dev/null +++ b/.github/tests/test_skill_execution.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the documented completion commands, including failures and live runs.""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(params=["cuequivariance-jax", "cuequivariance-torch"]) +def completion_command(request): + skill = Path(__file__).resolve().parents[2] / "skills" / request.param / "SKILL.md" + section = skill.read_text().split("### Complete and inspect a run\n", 1)[1] + return re.search(r"```bash\n(.*?)\n```", section, re.S).group(1) + + +def run_environment(tmp_path): + return {**os.environ, "TMPDIR": str(tmp_path), "cueq_python": sys.executable} + + +def test_completion_records_success_and_failure_without_reusing_logs( + completion_command, tmp_path +): + project = tmp_path / "project with spaces" + project.mkdir() + sentinel = tmp_path / ".venv" / "keep.txt" + sentinel.parent.mkdir() + sentinel.write_text("existing environment") + records = [] + for status in (0, 7): + (project / "check.py").write_text( + f"print('observed result for exit {status}')\nraise SystemExit({status})\n" + ) + result = subprocess.run( + ["bash", "-c", "set -e\n" + completion_command], + cwd=project, + env=run_environment(tmp_path), + capture_output=True, + text=True, + timeout=10, + ) + assert (result.returncode == 0) == (status == 0), result + record = Path(result.stdout.splitlines()[0].removeprefix("run directory: ")) + assert (record / "exit-code.txt").read_text().strip() == str(status) + assert ( + record / "output.txt" + ).read_text() == f"observed result for exit {status}\n" + assert f"observed result for exit {status}" in result.stdout + assert f"exit code: {status}" in result.stdout + records.append(record) + assert records[0] != records[1] + assert (records[0] / "exit-code.txt").read_text().strip() == "0" + assert sentinel.read_text() == "existing environment" + + +def test_live_run_has_no_completion_record(completion_command, tmp_path): + (tmp_path / "check.py").write_text( + "import time\nfrom pathlib import Path\n" + "print('check started', flush=True)\n" + "while not Path('release').exists(): time.sleep(0.01)\n" + "print('check completed')\n" + ) + with subprocess.Popen( + ["bash", "-c", "set -e\n" + completion_command], + cwd=tmp_path, + env=run_environment(tmp_path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + try: + first_line = process.stdout.readline().strip() + record = Path(first_line.removeprefix("run directory: ")) + assert record.is_dir(), first_line + assert process.poll() is None + assert not (record / "exit-code.txt").exists() + finally: + (tmp_path / "release").touch() + output, errors = process.communicate(timeout=10) + assert process.returncode == 0, errors + assert (record / "exit-code.txt").read_text().strip() == "0" + assert "check completed" in output + assert "exit code: 0" in output + + +def test_missing_interpreter_is_a_failed_run(completion_command, tmp_path): + env = run_environment(tmp_path) + env["cueq_python"] = str(tmp_path / "missing-python") + result = subprocess.run( + ["bash", "-c", "set -e\n" + completion_command], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=10, + ) + record = Path(result.stdout.splitlines()[0].removeprefix("run directory: ")) + assert result.returncode != 0 + assert (record / "exit-code.txt").read_text().strip() == "127" + assert "missing-python" in (record / "output.txt").read_text() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5585972c..7adb2d43 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,7 +72,7 @@ jobs: env: JAX_PLATFORMS: cpu run: | - pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py + pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_skill_execution.py .github/tests/test_skill_eval_setup.py uv pip install hatchling python .github/scripts/check_skill_packaging.py --package cuequivariance_jax diff --git a/skills/cuequivariance-jax/SKILL.md b/skills/cuequivariance-jax/SKILL.md index 9183815a..b6754d00 100644 --- a/skills/cuequivariance-jax/SKILL.md +++ b/skills/cuequivariance-jax/SKILL.md @@ -22,7 +22,7 @@ Required inputs: operation, SO3/O3 irreps, layout, batch shapes, dtype, device, 1. Probe the project interpreter once and reuse working packages. **Environment setup/recovery must not recursively delete directories**, including temporary environments. Use a fresh unique path when isolation is needed. 2. Select `method="naive"` for CPU; Flax is needed only for NNX. High-level dictionary blocks are `(..., mul, ir.dim)`; uniform CUDA arrays differ. 3. Adapt one matching recipe into a script. Build descriptors outside transformed functions, broadcast leading input axes, and enable x64 **before array creation** for float64 checks. -4. Run the script and requested checks together. Collect completion of any running session; retain exit status and printed shapes/errors. Report observed results and identify unrun checks. Assertions in unexecuted code are not a pass. +4. Finish installation before running the saved script with the completion recipe below. A running session/task ID, empty output, or a file write is not a completed check. Poll the same session until it exits; then read its results before reporting success. 5. Investigate a concrete failure using its error and the affected installed API. Rerun affected checks after a fix. Once the requested checks pass, finish; avoid repeated probes, unrelated examples, or exploration of other backends for a CPU reference. ## Prerequisites and environment recovery @@ -47,21 +47,43 @@ if "jax" in available: print("devices:", available["jax"].devices()) ``` -Reuse working packages and install missing CPU dependencies when appropriate. Honor the project's lockfile or supplied `scripts/requirements.txt` and compatible core/frontend revisions. JAX 0.11.1 / Flax 0.12.10 require **Python >=3.12**. Imports alone do not verify primitive compatibility: cuEquivariance 0.13.0 imports with JAX 0.12.0 but fails polynomial execution with a `segmented_polynomial_dce` argument-count error. The standalone setup below uses a tested combination; project requirements take precedence. +Reuse working packages; install only missing dependencies when needed. Honor the project's lockfile or supplied `scripts/requirements.txt` and compatible core/frontend revisions. JAX 0.11.1 / Flax 0.12.10 require **Python >=3.12**: select a compatible interpreter for those pins. The standalone combination below also supports Python 3.11; it does not replace project pins. Imports alone do not establish compatibility: cuEquivariance 0.13.0 with JAX 0.12.0 fails polynomial execution with a `segmented_polynomial_dce` argument-count error. For isolation, keep the unique directory returned by `mktemp`; reuse it for reruns. This Linux recipe uses host pip 22.3+ and bypasses `ensurepip`: ```bash set -e -cueq_python=python3.12 # Or the selected compatible interpreter's path. +cueq_python=python3 # Set to the selected project interpreter's path. cueq_jax_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-jax.XXXXXX") "$cueq_python" -m venv --without-pip "$cueq_jax_env" -python3 -m pip --python "$cueq_jax_env" install \ - "jax[cpu]==0.11.1" "cuequivariance==0.13.0" "cuequivariance-jax==0.13.0" -JAX_PLATFORMS=cpu "$cueq_jax_env/bin/python" -c 'import jax, cuequivariance_jax; print(jax.devices())' +python3 -m pip --python "$cueq_jax_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + "jax==0.10.2" "cuequivariance==0.13.0" "cuequivariance-jax==0.13.0" +cueq_python="$cueq_jax_env/bin/python" +JAX_PLATFORMS=cpu "$cueq_python" -c 'import jax, cuequivariance_jax; print("setup complete:", jax.__version__, cuequivariance_jax.__version__, jax.devices())' ``` -Replace the package list with `-r` and the requirements path when prescribed; otherwise add `flax==0.12.10` only for NNX. Run with `"$cueq_jax_env/bin/python"`, including the relevant numerical recipe after the import check. If host pip is unavailable, use an installed `uv`. Diagnose failures before retrying; preserve environments/system packages. Use trusted mirrors/caches with TLS verification enabled. If blocked, report unrun checks. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). +Replace the package list with `-r` and the requirements path when prescribed; otherwise add `flax==0.12.7` to the same install command only for NNX. Keep this interpreter for execution; shell variables may need restoring in a new tool call. Without host pip, use an installed `uv`. The pip command uses the Linux system CA bundle; for a private CA use the environment's approved `PIP_CERT` bundle. Retain TLS verification. Diagnose the actual error before retrying; never reset directories or switch versions while an install is still running. If no compatible interpreter or trusted package source is available, report the blocker and unrun checks. See [pip's certificates](https://pip.pypa.io/en/stable/topics/https-certificates/). + +### Complete and inspect a run + +Save the preamble and relevant checks in `check.py`. Set `cueq_python` to the probe's `sys.executable`, or retain it from setup. Run **after setup exits successfully**; a fresh directory records output and status: + +```bash +set -e +cueq_run=$(mktemp -d "${TMPDIR:-/tmp}/cueq-check.XXXXXX") +printf 'run directory: %s\n' "$cueq_run" +if JAX_PLATFORMS=cpu "$cueq_python" -u check.py >"$cueq_run/output.txt" 2>&1; then + cueq_status=0 +else + cueq_status=$? +fi +printf '%s\n' "$cueq_status" >"$cueq_run/exit-code.txt" +tail -n 30 "$cueq_run/output.txt" +printf 'exit code: %s\n' "$cueq_status" +test "$cueq_status" -eq 0 +``` + +If the tool yields, keep polling its session; the exit-code file is written only after Python finishes. Inspect that file and the numerical output. Empty output is not evidence that assertions ran. Deliver the script, actual command, and observed checks; label anything unrun. Keep setup output separate from the numerical results. ## Limitations and backend selection @@ -153,7 +175,8 @@ restored_blocks = ir_dict.flat_to_dict(irreps, flat_mul, layout="mul_ir") for ir in blocks: np.testing.assert_array_equal(restored_blocks[ir], blocks[ir]) np.testing.assert_array_equal(dict_to_ir_mul_flat(irreps, restored_blocks), flat_ir) -print("exact ir_mul/mul_ir round trip:", {str(ir): v.shape for ir, v in blocks.items()}) +print("regrouped unique irrep keys; exact ir_mul/mul_ir round trip:", + {str(ir): v.shape for ir, v in blocks.items()}) ``` `ir_dict.assert_mul_ir_dict` checks block conventions. `mul_ir_dict(irreps, data)` broadcasts a template to keys without inferring feature dimensions. diff --git a/skills/cuequivariance-jax/evals/config.yml b/skills/cuequivariance-jax/evals/config.yml index 7c9468f9..2d6ee283 100644 --- a/skills/cuequivariance-jax/evals/config.yml +++ b/skills/cuequivariance-jax/evals/config.yml @@ -1,9 +1,48 @@ -# Dependency versions below are exercised by the CPU contract checks. -# Build the task environment with scripts/requirements.txt; do not assume -# the shared Harbor base image contains these scientific packages. +# Prepare identical CPU dependencies for both arms before agent execution. +# Astra's shared image currently provides Python 3.11; use its tested stack. schema_version: 1 harbor: base_image_mode: rebuild + runtime_env: + VIRTUAL_ENV: /workspace/.venv + PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin + JAX_PLATFORMS: cpu + pre_agent_setup: + - | + cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" + cueq_env="$cueq_root/.venv" + if [ ! -e "$cueq_env" ]; then + cueq_new=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + python3 -m venv --system-site-packages --without-pip "$cueq_new" + ln -s "$cueq_new" "$cueq_env" + python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + 'cuequivariance==0.13.0' 'cuequivariance-jax==0.13.0' \ + 'jax==0.10.2' 'flax==0.12.7' + fi + JAX_PLATFORMS=cpu "$cueq_env/bin/python" - <<'PY' + import importlib.metadata as metadata + import sys + import jax + import jax.numpy as jnp + import cuequivariance as cue + import cuequivariance_jax as cuex + from cuequivariance_jax import nnx + + for package, expected in { + "cuequivariance": "0.13.0", "cuequivariance-jax": "0.13.0", + "jax": "0.10.2", "flax": "0.12.7", + }.items(): + assert metadata.version(package) == expected, (package, metadata.version(package)) + assert all(device.platform == "cpu" for device in jax.devices()) + descriptor = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1]) + output = cuex.segmented_polynomial( + descriptor.polynomial, [jnp.ones((1, 3))], + [jax.ShapeDtypeStruct((1, 4), jnp.float32)], method="naive", + )[0] + assert output.shape == (1, 4) and bool(jnp.isfinite(output).all()) + assert callable(nnx.IrrepsLinear) + print("CPU environment ready:", sys.executable, sys.version.split()[0], jax.devices()) + PY skill_workspace: mode: isolated grading: diff --git a/skills/cuequivariance-jax/scripts/requirements.txt b/skills/cuequivariance-jax/scripts/requirements.txt index e50097dc..5ec8d539 100644 --- a/skills/cuequivariance-jax/scripts/requirements.txt +++ b/skills/cuequivariance-jax/scripts/requirements.txt @@ -1,6 +1,6 @@ # CPU evaluation baseline for the APIs documented in this skill. -# Pin the source revision because main can contain APIs newer than PyPI. -cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance -cuequivariance-jax @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance_jax -jax==0.11.1 -flax==0.12.10 +# Match evals/config.yml and the tested Python 3.11-compatible release stack. +cuequivariance==0.13.0 +cuequivariance-jax==0.13.0 +jax==0.10.2 +flax==0.12.7 diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md index e343228d..0722afe9 100644 --- a/skills/cuequivariance-torch/SKILL.md +++ b/skills/cuequivariance-torch/SKILL.md @@ -11,29 +11,29 @@ metadata: ## Purpose -Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_torch` (`cuet`) and PyTorch autograd: CPU references, CUDA backends, and equivariant layers. Preserve the project's framework and environment. +Execute `cuequivariance` (`cue`) descriptors with `cuequivariance_torch` (`cuet`): CPU references, CUDA backends, equivariant layers, and autograd. Preserve the project's environment. ## Inputs -Required inputs: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership. Derive them from the request or code; state defaults. Graphs also need source/destination counts and indices. Optional inputs: backend, performance, or export targets. +Required inputs: operation, SO3/O3 irreps, layout, device, dtype, and weight ownership; infer from the request or code. Graphs also need source/destination counts and indices. Optional inputs: backend, performance, export targets. ## Instructions -1. Probe the selected interpreter once. Reuse working CPU PyTorch; a CUDA-kernel error on CPU calls for `method="naive"` at construction, not a new environment. -2. Set `method="naive"` for CPU calculations. Match module parameters, features, weights, and reference computations in device and dtype. Use float64 for small numerical and gradient checks. +1. Probe once and reuse working PyTorch. **Never recursively delete directories during setup or recovery**, including partial temporary environments; use a fresh unique path when isolation is needed. +2. Set `method="naive"` at construction for CPU. Match parameters, features, weights, and references in device and dtype. Use float64 for small numerical and gradient checks. 3. Specify layout and weight ownership explicitly. Derive sizes and output order from the module/descriptor; do not guess multiplicities from a filter. -4. Run the relevant example and requested checks together. Retain exit status and printed shapes/errors/gradients. Assertions in unexecuted code do not prove a pass; report blocked checks as unrun. -5. For an API mismatch, inspect that installed object's signature/source once. Adapt to its version or explain the requirement; preserve the requested library. +4. Finish installation before running the saved script with the completion recipe below. A session/task ID, empty output, or a file write is not completion. Poll that session until it exits and inspect its results. +5. For an API mismatch, inspect the installed signature/source once. Adapt or explain the requirement; preserve the requested library. ## Prerequisites and environment reuse -Run this with the project's Python. Probe imports independently so one missing package does not hide working PyTorch: +Probe imports independently with the project's Python; a missing frontend must not hide working PyTorch: ```python import importlib import sys -print("python:", sys.executable) +print("python:", sys.executable, sys.version.split()[0]) available, import_errors = {}, {} for name in ("torch", "cuequivariance", "cuequivariance_torch"): try: @@ -50,27 +50,50 @@ if "torch" in available: Choose the next step from the probe: -- Imports work: retain this interpreter; install nothing. -- PyTorch works but cuEquivariance is missing: install only the missing frontend/core packages when allowed. Distribution: `cuequivariance-torch`; import: `cuequivariance_torch`. -- PyTorch is missing: check the project's configured interpreter (such as `.venv/bin/python`) before installing. If the requested existing installation is inaccessible, say so; a new environment cannot verify that installation. -- Shared-library/binary mismatch: diagnose the reported package/path and version compatibility; this differs from an absent package. +- Imports work: retain the interpreter; install nothing. +- Only cuEquivariance is missing: install `cuequivariance-torch` in that interpreter; it does not require replacing PyTorch. +- PyTorch is missing: check the project interpreter (e.g. `.venv/bin/python`). If the requested installation is inaccessible, say so; a new environment cannot verify it. +- Binary/shared-library mismatch: diagnose the reported package/path and compatibility. -Use the same interpreter for pip and execution. Honor the lockfile or supplied `scripts/requirements.txt`; keep checkout core/frontend revisions compatible. Retaining installed PyTorch takes priority over example install commands. Optional CUDA/optimization warnings are not installation failures. +Use one interpreter for pip and execution. Honor project requirements and compatible core/frontend revisions. Retaining installed PyTorch takes priority over setup examples. Optional CUDA/optimization warnings are not failures. -If isolation is needed and installation is appropriate, create a **new unique** environment. This Linux/macOS recipe bypasses `ensurepip` using host pip 22.3+: +When isolation is needed, create a **new unique** environment. This Linux recipe uses host pip 22.3+ without `ensurepip`: ```bash set -e +cueq_python=python3 # Set to the selected project interpreter's path. cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") -python3 -m venv --without-pip "$cueq_env" -python3 -m pip --python "$cueq_env" install torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip --python "$cueq_env" install cuequivariance-torch -"$cueq_env/bin/python" -c 'import sys, torch, cuequivariance_torch; print(sys.executable, torch.__version__, cuequivariance_torch.__version__)' +"$cueq_python" -m venv --without-pip "$cueq_env" +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q 'torch==2.14.1+cpu' --extra-index-url https://download.pytorch.org/whl/cpu +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q cuequivariance-torch +cueq_python="$cueq_env/bin/python" +"$cueq_python" -c 'import sys, torch, cuequivariance_torch; assert torch.version.cuda is None; print("CPU setup complete:", sys.executable, torch.__version__, cuequivariance_torch.__version__)' ``` -For supplied requirements, replace both install commands with `python3 -m pip --python "$cueq_env" install -r /path/to/requirements.txt`. Execute with `"$cueq_env/bin/python"`; activation and embedded pip are unnecessary. Do not delete `.venv` or alter system packages to repair `ensurepip`; preserve existing directories. Without host pip, use an existing manager such as `uv`. See [pip's interpreter selection](https://pip.pypa.io/en/stable/topics/python-option/). +For supplied requirements, replace both install commands with one using `-r` and their path. Retain `cueq_python` for execution; restore its path in new tool calls if needed. Without host pip, use an installed `uv`. -On setup failure, retain the error and diagnose its cause before retrying. For an unavailable source, use a configured trusted mirror/cache; keep TLS verification enabled and avoid guessed wheel URLs. If still blocked, provide the code and name the unrun checks instead of repeating environment resets/network probes. +Use approved `PIP_CERT` for a private CA; retain TLS verification. If downloading fails, try a configured CPU mirror/cache or report the blocker. Do not replace CPU wheels with CUDA builds. Finish installs before retrying. See [pip's certificates](https://pip.pypa.io/en/stable/topics/https-certificates/). + +### Complete and inspect a run + +Save the preamble and checks in `check.py`. Set `cueq_python` to the probe's `sys.executable`, or retain it from setup. Run **after setup exits successfully**: + +```bash +set -e +cueq_run=$(mktemp -d "${TMPDIR:-/tmp}/cueq-check.XXXXXX") +printf 'run directory: %s\n' "$cueq_run" +if "$cueq_python" -u check.py >"$cueq_run/output.txt" 2>&1; then + cueq_status=0 +else + cueq_status=$? +fi +printf '%s\n' "$cueq_status" >"$cueq_run/exit-code.txt" +tail -n 30 "$cueq_run/output.txt" +printf 'exit code: %s\n' "$cueq_status" +test "$cueq_status" -eq 0 +``` + +The exit-code file appears only when Python finishes. If the tool yields, poll its session, then inspect the file and numerical output. Deliver the script, command, observed results, and backend explanation; label unrun checks. Keep install output separate. ## Limitations and backends @@ -93,7 +116,7 @@ High-level defaults vary: `SphericalHarmonics` selects `uniform_1d`, `FullyConne ## Examples -Each CPU example runs independently with this preamble. Adapt its inputs while preserving layout, indexing, and gradient contracts; retain its printed results and any failure. +Each CPU example runs independently with this preamble; preserve layout, indexing, and gradient contracts when adapting inputs. ```python import numpy as np @@ -103,12 +126,14 @@ import cuequivariance_torch as cuet torch.manual_seed(0) dtype = torch.float64 -print("torch:", torch.__version__, "device: cpu", "dtype:", dtype, "method: naive") +print("torch:", torch.__version__, "CUDA build:", torch.version.cuda, + "CUDA available:", torch.cuda.is_available(), "device: cpu", "dtype:", dtype, + "method: naive") ``` ### CPU tensor product and backward -Select `naive` for **both** modules on CPU. This example uses trainable internal weights and requires no CUDA packages. +Use `naive` for **both** CPU modules. Here weights are internal trainable parameters. ```python irreps = cue.Irreps("SO3", "0 + 1") @@ -188,7 +213,7 @@ print("Linear:", tuple(y.shape), "shared external weights:", tuple(w.shape), ### Channelwise product with external per-example weights -The core descriptor takes `(weights, input1, input2)` and returns a list of outputs. The high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids an extra transpose in this reference. `filter_irreps_out` filters **types**, not multiplicities: report `tp.irreps_out` and retain that actual order in references and downstream layers. +The core descriptor takes `(weights, input1, input2)` and returns a list; the high-level module takes `(input1, input2, weight=...)` and returns a tensor. Matching `ir_mul` layouts avoids transposition. Report `tp.irreps_out` and preserve its actual order: the filter selects **types**, not multiplicities. ```python a, b = cue.Irreps("SO3", "3x0 + 3x1"), cue.Irreps("SO3", "0 + 1") @@ -208,12 +233,12 @@ grads = torch.autograd.grad(y.square().sum(), (x1, x2, w)) for g in grads: assert torch.isfinite(g).all() and g.abs().sum() > 0 print("Channelwise:", tuple(y.shape), "per-item weights:", tuple(w.shape), - "ir_mul output order:", tp.irreps_out, + "filter selects types; actual ir_mul output order:", tp.irreps_out, "max error:", np.max(np.abs(y.detach().numpy() - reference)), "gradient norms:", [g.norm().item() for g in grads]) ``` -For a fully connected product, specify the desired output irreps in `cuet.FullyConnectedTensorProduct(a, b, irreps_out, method="naive", dtype=dtype)`. Its default weights are internal trainable parameters. Use the same external-weight flags when the caller or an edge network owns the weights. +`cuet.FullyConnectedTensorProduct(a, b, irreps_out, method="naive", dtype=dtype)` accepts desired output irreps. Its weights are internal by default; use the same external-weight flags for caller-owned weights. ### SegmentedPolynomial: gather and scatter @@ -237,12 +262,13 @@ ref_grads = torch.autograd.grad(reference.square().sum(), (nodes, w)) for g, ref in zip(grads, ref_grads): torch.testing.assert_close(g, ref) print("Gather/scatter:", tuple(actual.shape), + "input/output index dtypes:", senders.dtype, receivers.dtype, "forward error:", (actual - reference).detach().abs().max().item(), "gradient errors:", [(g - r).abs().max().item() for g, r in zip(grads, ref_grads)], "isolated destination:", actual[3].detach().tolist()) ``` -Reuse the descriptor to build a numerical reference: its coefficients include normalization factors, so a hand-written matrix product without those factors can be wrong. For high-level channelwise gathering/scattering, use `indices_1`, `indices_2`, `indices_out`, and `size_out` instead of the primitive's dictionaries. +Descriptor coefficients include normalization factors; retain them in references. High-level channelwise gathering/scattering uses `indices_1`, `indices_2`, `indices_out`, and `size_out` instead of dictionaries. ### Trainable message passing @@ -278,7 +304,7 @@ Move the whole module and inputs to the same dtype/device (for example `model.do ## Other operations and layers -Inspect the requested operation's installed signature/source via `inspect.signature` and `inspect.getfile`, rather than loading the entire API. These are the non-obvious contracts: +Inspect the requested operation via `inspect.signature` and `inspect.getfile`. Key contracts: | Operation | Contract | | --- | --- | diff --git a/skills/cuequivariance-torch/evals/config.yml b/skills/cuequivariance-torch/evals/config.yml index 7c9468f9..21e9737b 100644 --- a/skills/cuequivariance-torch/evals/config.yml +++ b/skills/cuequivariance-torch/evals/config.yml @@ -1,9 +1,43 @@ -# Dependency versions below are exercised by the CPU contract checks. -# Build the task environment with scripts/requirements.txt; do not assume -# the shared Harbor base image contains these scientific packages. +# Prepare identical CPU dependencies for both arms before agent execution. schema_version: 1 harbor: base_image_mode: rebuild + runtime_env: + VIRTUAL_ENV: /workspace/.venv + PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin + pre_agent_setup: + - | + cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" + cueq_env="$cueq_root/.venv" + if [ ! -e "$cueq_env" ]; then + cueq_new=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + python3 -m venv --system-site-packages --without-pip "$cueq_new" + ln -s "$cueq_new" "$cueq_env" + python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + 'torch==2.14.1+cpu' --extra-index-url https://download.pytorch.org/whl/cpu + python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' + fi + "$cueq_env/bin/python" - <<'PY' + import importlib.metadata as metadata + import sys + import torch + import cuequivariance_torch as cuet + + for package, expected in { + "cuequivariance": "0.13.0", "cuequivariance-torch": "0.13.0", + "torch": "2.14.1+cpu", + }.items(): + assert metadata.version(package) == expected, (package, metadata.version(package)) + assert torch.version.cuda is None, torch.__version__ + layer = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") + x = torch.ones((1, 3), dtype=torch.float64, requires_grad=True) + y = layer(x) + assert y.shape == (1, 4) and bool(torch.isfinite(y).all()) + y.square().sum().backward() + assert x.grad is not None and bool(torch.isfinite(x.grad).all()) + print("CPU environment ready:", sys.executable, sys.version.split()[0], torch.__version__) + PY skill_workspace: mode: isolated grading: diff --git a/skills/cuequivariance-torch/scripts/requirements.txt b/skills/cuequivariance-torch/scripts/requirements.txt index c5f8ec68..76514d21 100644 --- a/skills/cuequivariance-torch/scripts/requirements.txt +++ b/skills/cuequivariance-torch/scripts/requirements.txt @@ -1,6 +1,6 @@ # CPU evaluation baseline for the APIs documented in this skill. -# Pin the source revision because main can contain APIs newer than PyPI. -cuequivariance @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance -cuequivariance-torch @ https://github.com/NVIDIA/cuEquivariance/archive/046edf7e2ee9e9fb7868e6c32c787d9049d79ebe.tar.gz#subdirectory=cuequivariance_torch +# Match evals/config.yml; keep the CPU-only wheel build explicit. +cuequivariance==0.13.0 +cuequivariance-torch==0.13.0 --extra-index-url=https://download.pytorch.org/whl/cpu torch==2.14.1+cpu diff --git a/skills/cuequivariance/SKILL.md b/skills/cuequivariance/SKILL.md index f371e226..c9c7a6ea 100644 --- a/skills/cuequivariance/SKILL.md +++ b/skills/cuequivariance/SKILL.md @@ -254,7 +254,7 @@ e = cue.descriptors.symmetric_contraction( ### IrDictPolynomial descriptors -Each `_ir_dict` variant returns an `IrDictPolynomial` whose polynomial is already split by irrep. The `input_irreps` and `output_irreps` tuples describe the operand groups. +Each `_ir_dict` constructor returns an `IrDictPolynomial`: `.polynomial` is a `SegmentedPolynomial` already split by irrep, and `.input_irreps` / `.output_irreps` describe the operand groups. Each `(mul, ir)` block in those groups corresponds to one polynomial operand of size `mul * ir.dim`. `channelwise_tensor_product_ir_dict` always simplifies output irreps. To compare with its dense counterpart, pass `simplify_irreps3=True` to @@ -269,10 +269,19 @@ desc = cue.descriptors.channelwise_tensor_product_ir_dict( cue.Irreps("SO3", "0 + 1"), cue.Irreps("SO3", "0 + 1"), ) -# desc.polynomial — SegmentedPolynomial, already split by irrep -# desc.input_irreps — (weight_irreps, irreps1, irreps2) -# desc.output_irreps — (irreps_out,) +desc.polynomial # SegmentedPolynomial +desc.input_irreps # (weight_irreps, irreps1, irreps2) +desc.output_irreps # (irreps_out,) + +# Scale coefficients and inspect individual input operands +scaled_poly = desc.polynomial * 0.5 +for i, op in enumerate(desc.polynomial.inputs): + print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") +``` +Other constructors use the same operand metadata contract: + +```python # Fully connected tensor product desc = cue.descriptors.fully_connected_tensor_product_ir_dict(irreps1, irreps2, irreps3) @@ -292,34 +301,9 @@ desc = cue.descriptors.spherical_harmonics_ir_dict(cue.O3(1, -1), [0, 1, 2, 3]) desc = cue.descriptors.symmetric_contraction_ir_dict(irreps_in, irreps_out, (1, 2, 3)) ``` -### IrDictPolynomial - -`IrDictPolynomial` pairs a `SegmentedPolynomial` (already split by irrep) with the `Irreps` that describe each operand group. - -```python -desc = cue.descriptors.channelwise_tensor_product_ir_dict( - 32 * cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), - cue.Irreps("SO3", "0 + 1"), -) - -desc.polynomial # SegmentedPolynomial — each operand is one (mul, ir) block -desc.input_irreps # (weight_irreps, irreps1, irreps2) -desc.output_irreps # (irreps_out,) - -# Scale coefficients -scaled_poly = desc.polynomial * 0.5 - -# Access individual operand info -for i, op in enumerate(desc.polynomial.inputs): - print(f"Input {i}: size={op.size}, num_segments={op.num_segments}") -``` - -Contract: for each `(mul, ir)` block in `input_irreps` / `output_irreps`, the corresponding polynomial operand has size `mul * ir.dim`. - ### split_polynomial_by_irreps -The low-level function underlying `_ir_dict` descriptors. Splits one polynomial operand at irrep boundaries: +Use this helper to split an existing `SegmentedPolynomial` operand at irrep boundaries. Supply the operand's irreps explicitly because the bare polynomial has no representation metadata: ```python poly = e.polynomial # from an EquivariantPolynomial @@ -341,7 +325,7 @@ e.polynomial # the underlying SegmentedPolynomial # Preparing for uniform_1d execution (see cuequivariance_jax SKILL.md) e_ready = e.squeeze_modes().flatten_coefficient_modes() -# Split an operand into per-irrep pieces (for ir_dict interface) +# Split dense operands using the representation metadata already attached to e e_split = e.split_operand_by_irrep(1).split_operand_by_irrep(-1) # Scale all coefficients From 9f0aadd4d1185e8a73d26590b66780054972ba13 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 19 Oct 2026 22:15:55 -0700 Subject: [PATCH 12/19] Fix JAX and Torch Tier 3 environment setup and recovery Signed-off-by: Ohad Mosafi --- .github/tests/test_skill_eval_setup.py | 127 ++++++++++++++++++- skills/README.md | 17 ++- skills/cuequivariance-jax/evals/config.yml | 25 +++- skills/cuequivariance-torch/evals/config.yml | 38 ++++-- 4 files changed, 185 insertions(+), 22 deletions(-) diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py index 2300054e..f4eac971 100644 --- a/.github/tests/test_skill_eval_setup.py +++ b/.github/tests/test_skill_eval_setup.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 """Pre-agent environment failures must stop evaluation without replacing data.""" +import json import os import subprocess import sys @@ -10,10 +11,7 @@ import yaml -@pytest.mark.parametrize("frontend", ["jax", "torch"]) -def test_pre_agent_setup_preserves_existing_incompatible_environment( - tmp_path, frontend -): +def setup_command(frontend): config_path = ( Path(__file__).resolve().parents[2] / "skills" @@ -22,6 +20,124 @@ def test_pre_agent_setup_preserves_existing_incompatible_environment( / "config.yml" ) config = yaml.safe_load(config_path.read_text()) + return "set -euo pipefail\n" + "\n".join(config["harbor"]["pre_agent_setup"]) + + +@pytest.fixture +def setup_sandbox(tmp_path): + """Inject installer/smoke failures without downloading dependencies.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + python = bin_dir / "python3" + python.write_text( + f"#!{sys.executable}\n" + """ +import json +import os +import sys +from pathlib import Path + +root = Path(os.environ["CUEQ_EVAL_ROOT"]) +args = sys.argv[1:] +if args[:2] == ["-m", "venv"]: + stage = "venv" + target = Path(args[-1]) + (target / "bin").mkdir() + (target / "bin/python").symlink_to(Path(__file__).resolve()) +elif args[:2] == ["-m", "pip"]: + stage = "install" +else: + stage = "smoke" + sys.stdin.read() + +with (root / "calls.jsonl").open("a") as log: + log.write(json.dumps({ + "stage": stage, "args": args, "published": (root / ".venv").is_symlink() + }) + "\\n") +if os.environ.get("CUEQ_FAIL_STAGE") == stage: + print("injected " + stage + " failure", file=sys.stderr) + sys.exit(23) +if stage == "smoke": + print("CPU environment ready") +""" + ) + python.chmod(0o755) + return { + **os.environ, + "CUEQ_EVAL_ROOT": str(tmp_path), + "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"], + } + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +@pytest.mark.parametrize("failure_stage", ["venv", "install", "smoke"]) +def test_failed_setup_is_not_published_and_can_be_retried( + tmp_path, setup_sandbox, frontend, failure_stage +): + def run(**extra): + return subprocess.run( + ["bash", "-c", setup_command(frontend)], + env={**setup_sandbox, **extra}, + capture_output=True, + text=True, + timeout=30, + ) + + failed = run(CUEQ_FAIL_STAGE=failure_stage) + assert failed.returncode == 23, failed.stderr + assert f"injected {failure_stage} failure" in failed.stderr + assert not os.path.lexists(tmp_path / ".venv") + (partial,) = tmp_path.glob(".cueq-env.*") + sentinel = partial / "keep.txt" + sentinel.write_text("partial environment") + + retried = run(CUEQ_FAIL_STAGE="") + assert retried.returncode == 0, retried.stderr + assert "CPU environment ready" in retried.stdout + published = tmp_path / ".venv" + assert published.is_symlink() + assert published.resolve() != partial + assert sentinel.read_text() == "partial environment" + calls = [ + json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines() + ] + assert not any(call["published"] for call in calls) + + # Reuse checks the environment but does not reinstall or replace it. + target = published.resolve() + reused = run(CUEQ_FAIL_STAGE="install") + assert reused.returncode == 0, reused.stderr + assert published.resolve() == target + after = [ + json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines() + ] + assert [call["stage"] for call in after[len(calls) :]] == ["smoke"] + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +def test_pre_agent_setup_preserves_dangling_environment_link( + tmp_path, setup_sandbox, frontend +): + env_dir = tmp_path / ".venv" + target = tmp_path / "missing-environment" + env_dir.symlink_to(target) + result = subprocess.run( + ["bash", "-c", setup_command(frontend)], + env=setup_sandbox, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert env_dir.is_symlink() and env_dir.readlink() == target + assert not list(tmp_path.glob(".cueq-env.*")) + assert not (tmp_path / "calls.jsonl").exists() + + +@pytest.mark.parametrize("frontend", ["jax", "torch"]) +def test_pre_agent_setup_preserves_existing_incompatible_environment( + tmp_path, frontend +): env_dir = tmp_path / ".venv" subprocess.run( [sys.executable, "-m", "venv", "--without-pip", str(env_dir)], @@ -31,9 +147,8 @@ def test_pre_agent_setup_preserves_existing_incompatible_environment( sentinel = env_dir / "keep.txt" sentinel.write_text("existing environment") original_config = (env_dir / "pyvenv.cfg").read_bytes() - command = "set -euo pipefail\n" + "\n".join(config["harbor"]["pre_agent_setup"]) result = subprocess.run( - ["bash", "-c", command], + ["bash", "-c", setup_command(frontend)], env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, capture_output=True, text=True, diff --git a/skills/README.md b/skills/README.md index ee6b77d3..c4365851 100644 --- a/skills/README.md +++ b/skills/README.md @@ -50,7 +50,22 @@ custom evaluation image or common `harbor.pre_agent_setup` is needed when the sandbox does not already provide them. Provision it equally for both arms and establish a new baseline when changing the environment. -The requirements pin cuEquivariance to the immutable source revision whose APIs +The JAX and PyTorch setup commands publish `/workspace/.venv` only after the +pinned packages pass a CPU smoke check. Failed environments remain at their +unique `.cueq-env.*` paths for diagnosis; the next attempt creates a fresh one. +An existing `.venv`, including a broken symlink, is preserved and must pass the +check without being replaced. Setup uses binary wheels, skips bytecode +compilation, and fetches only the PyTorch wheel from the CPU index. A second +install resolves its dependencies and cuEquivariance from the normal index, +avoiding dependency-metadata incompatibilities with older host pip versions. +Both arms use the same setup and package pins. + +These are repository-side setup improvements, not a change to SkillEvaluator's +healthcheck timeout. A successful local setup does not establish that the shared +Astra sandbox will finish within its budget. Inspect the live preflight result +before interpreting a missing Torch score as a skill-quality failure. + +The requirements pin the cuEquivariance release or source revision whose APIs were checked, and pin the JAX/Flax or CPU PyTorch versions. They do not require sibling skills or a repository checkout inside the sandbox. When updating the API baseline, update the pins and run the numerical checks diff --git a/skills/cuequivariance-jax/evals/config.yml b/skills/cuequivariance-jax/evals/config.yml index 2d6ee283..d359552d 100644 --- a/skills/cuequivariance-jax/evals/config.yml +++ b/skills/cuequivariance-jax/evals/config.yml @@ -9,17 +9,25 @@ harbor: JAX_PLATFORMS: cpu pre_agent_setup: - | + set -eu cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" cueq_env="$cueq_root/.venv" - if [ ! -e "$cueq_env" ]; then - cueq_new=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") - python3 -m venv --system-site-packages --without-pip "$cueq_new" - ln -s "$cueq_new" "$cueq_env" - python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ + cueq_publish=false + if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_host_python=$(command -v python3) + cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + cueq_publish=true + "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" + printf 'Installing CPU JAX dependencies into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off \ 'cuequivariance==0.13.0' 'cuequivariance-jax==0.13.0' \ 'jax==0.10.2' 'flax==0.12.7' fi - JAX_PLATFORMS=cpu "$cueq_env/bin/python" - <<'PY' + printf 'Checking CPU environment: %s\n' "$cueq_env" + JAX_PLATFORMS=cpu "$cueq_env/bin/python" -u - <<'PY' import importlib.metadata as metadata import sys import jax @@ -43,6 +51,11 @@ harbor: assert callable(nnx.IrrepsLinear) print("CPU environment ready:", sys.executable, sys.version.split()[0], jax.devices()) PY + # Publish only after installation and the primitive compatibility check. + # Preserve failed environments for diagnosis; a retry uses a fresh path. + if [ "$cueq_publish" = true ]; then + ln -sT "$cueq_env" "$cueq_root/.venv" + fi skill_workspace: mode: isolated grading: diff --git a/skills/cuequivariance-torch/evals/config.yml b/skills/cuequivariance-torch/evals/config.yml index 21e9737b..d1861b5e 100644 --- a/skills/cuequivariance-torch/evals/config.yml +++ b/skills/cuequivariance-torch/evals/config.yml @@ -7,18 +7,33 @@ harbor: PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin pre_agent_setup: - | + set -eu cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" cueq_env="$cueq_root/.venv" - if [ ! -e "$cueq_env" ]; then - cueq_new=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") - python3 -m venv --system-site-packages --without-pip "$cueq_new" - ln -s "$cueq_new" "$cueq_env" - python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ - 'torch==2.14.1+cpu' --extra-index-url https://download.pytorch.org/whl/cpu - python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q \ - 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' + cueq_publish=false + if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_host_python=$(command -v python3) + cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + cueq_publish=true + "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" + # Reduce setup time: wheels only, + # no bytecode compilation, and no PyPI lookup for the CPU Torch build. + printf 'Installing CPU PyTorch into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off --no-deps 'torch==2.14.1+cpu' --index-url https://download.pytorch.org/whl/cpu + # Resolve Torch's dependencies on the normal index. Older host pip + # rejects some dependency metadata on the PyTorch index. + printf 'Installing Torch dependencies and cuEquivariance into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off \ + 'torch==2.14.1+cpu' 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' fi - "$cueq_env/bin/python" - <<'PY' + printf 'Checking CPU environment: %s\n' "$cueq_env" + "$cueq_env/bin/python" -u - <<'PY' import importlib.metadata as metadata import sys import torch @@ -38,6 +53,11 @@ harbor: assert x.grad is not None and bool(torch.isfinite(x.grad).all()) print("CPU environment ready:", sys.executable, sys.version.split()[0], torch.__version__) PY + # A failed install or smoke check must not leave .venv pointing at a + # partial environment. Preserve it for diagnosis and retry in a fresh path. + if [ "$cueq_publish" = true ]; then + ln -sT "$cueq_env" "$cueq_root/.venv" + fi skill_workspace: mode: isolated grading: From 08e0bafacea17f4bb41630a8316b88232682d490 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 20 Oct 2026 00:06:32 -0700 Subject: [PATCH 13/19] fix evals for JAX and PyTorch skills Signed-off-by: Ohad Mosafi --- .github/evals/cueq_challenges.py | 485 ++++++++++++++++++ .github/evals/reference_jax_challenges.py | 73 +++ .github/evals/reference_torch_challenges.py | 73 +++ .github/scripts/prepare_skill_challenges.py | 188 +++++++ .github/tests/challenge_checks.py | 105 ++++ .github/tests/test_jax_skill_challenges.py | 33 ++ .../tests/test_skill_challenge_packaging.py | 115 +++++ .github/tests/test_torch_skill_challenges.py | 33 ++ .github/workflows/tests.yml | 4 +- skills/README.md | 89 +++- .../cuequivariance-jax/evals/challenges.json | 39 ++ .../evals/challenges.json | 39 ++ 12 files changed, 1271 insertions(+), 5 deletions(-) create mode 100644 .github/evals/cueq_challenges.py create mode 100644 .github/evals/reference_jax_challenges.py create mode 100644 .github/evals/reference_torch_challenges.py create mode 100644 .github/scripts/prepare_skill_challenges.py create mode 100644 .github/tests/challenge_checks.py create mode 100644 .github/tests/test_jax_skill_challenges.py create mode 100644 .github/tests/test_skill_challenge_packaging.py create mode 100644 .github/tests/test_torch_skill_challenges.py create mode 100644 skills/cuequivariance-jax/evals/challenges.json create mode 100644 skills/cuequivariance-torch/evals/challenges.json diff --git a/.github/evals/cueq_challenges.py b/.github/evals/cueq_challenges.py new file mode 100644 index 00000000..79137c3d --- /dev/null +++ b/.github/evals/cueq_challenges.py @@ -0,0 +1,485 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Execute challenge submissions. This file is a verifier, never a skill resource. + +Standalone: python cueq_challenges.py --backend jax --case layout --solution FILE +Harbor: read entry.json and grade /workspace/solution.py in the prepared CPU venv. +Custom scores supplement ACES; they do not overwrite its metrics or pass rule. +""" + +from __future__ import annotations + +import argparse +import ast +import importlib.util +import json +import os +import subprocess +import sys +import traceback +from pathlib import Path + +PARAMETERS = ("radial_in", "radial_out", "projection") + + +def load_solution(path): + # Match normal execution of a saved Python module, including sibling helpers + # and decorators that resolve annotations through sys.modules. + sys.path.insert(0, str(path.resolve().parent)) + spec = importlib.util.spec_from_file_location("challenge_solution", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def array(value): + import numpy as np + + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + return np.asarray(value) + + +def close(actual, expected, *, exact=False): + import numpy as np + + actual, expected = array(actual), array(expected) + assert actual.shape == expected.shape, (actual.shape, expected.shape) + assert actual.dtype == expected.dtype, (actual.dtype, expected.dtype) + assert np.isfinite(actual).all(), "nonfinite result" + if exact: + np.testing.assert_array_equal(actual, expected) + else: + np.testing.assert_allclose(actual, expected, rtol=2e-8, atol=2e-9) + + +def numpy_blocks(irreps, flat): + """Independent oracle: slice original blocks and concatenate equal irreps.""" + import numpy as np + + parts, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)).swapaxes(-1, -2) + parts.setdefault(ir, []).append(block) + offset += mul * ir.dim + return {ir: np.concatenate(values, axis=-2) for ir, values in parts.items()} + + +def change_layout(flat, irreps, *, source): + import numpy as np + + parts, offset = [], 0 + for mul, ir in irreps: + suffix = (mul, ir.dim) if source == "mul_ir" else (ir.dim, mul) + block = flat[..., offset : offset + mul * ir.dim] + parts.append( + block.reshape(flat.shape[:-1] + suffix) + .swapaxes(-1, -2) + .reshape(flat.shape[:-1] + (mul * ir.dim,)) + ) + offset += mul * ir.dim + return np.concatenate(parts, axis=-1) + + +def check_layout(solution, backend): + import numpy as np + + import cuequivariance as cue + + if backend == "jax": + import jax + import jax.numpy as xp + + jax.config.update("jax_enable_x64", True) + else: + import torch + + for text in ("1o + 2x0e + 2x1o + 1e + 0o", "2x1e + 0o + 1o + 1e + 2x0e + 1o"): + irreps = cue.Irreps("O3", text) + for batch in ((), (4,), (2, 3)): + raw = np.arange(int(np.prod(batch)) * irreps.dim, dtype=np.float64) + raw = (raw.reshape(batch + (irreps.dim,)) + 0.25) / 8 + if backend == "jax": + x = xp.asarray(raw) + else: + # Exercise a noncontiguous tensor with the same values. + x = torch.tensor(np.stack((raw, -raw), -1))[..., 0] + x.requires_grad_() + assert not x.is_contiguous() + expected = numpy_blocks(irreps, raw) + blocks = solution.to_blocks(irreps, x) + assert set(blocks) == set(expected), ( + "keys must include both degree and parity" + ) + for ir, value in expected.items(): + close(blocks[ir], value, exact=True) + restored = solution.from_blocks(irreps, blocks) + close(restored, raw, exact=True) + + # Use a nonuniform linear loss so inverse permutations cannot hide. + cotangent = np.linspace(0.2, 1.3, raw.size).reshape(raw.shape) + if backend == "jax": + + def convert(z): + return solution.to_blocks(irreps, z) + + def round_trip(z): + return solution.from_blocks(irreps, convert(z)) + + close(jax.jit(round_trip)(x), raw, exact=True) + for ir, value in jax.jit(convert)(x).items(): + close(value, expected[ir], exact=True) + gradient = jax.grad( + lambda z: xp.sum(round_trip(z) * xp.asarray(cotangent)) + )(x) + else: + gradient = torch.autograd.grad( + (restored * torch.tensor(cotangent)).sum(), x + )[0] + close(gradient, cotangent) + + rep = cue.IrrepsAndLayout(irreps, cue.ir_mul) + for angles, parity in (([0.21, -0.32, 0.17], [0]), ([0, 0, 0], [1])): + matrix = rep.exp_map(angles, parity) + transformed = raw @ matrix.T + z = ( + xp.asarray(transformed) + if backend == "jax" + else torch.tensor(transformed) + ) + transformed_blocks = solution.to_blocks(irreps, z) + for ir in expected: + close( + transformed_blocks[ir], + expected[ir] @ ir.exp_map(angles, parity).T, + ) + + +def graph_descriptors(): + import cuequivariance as cue + + irreps_in = cue.Irreps("SO3", "2x0 + 2x1") + irreps_sh = cue.Irreps("SO3", "0 + 1 + 2") + irreps_out = cue.Irreps("SO3", "0 + 1") + product = cue.descriptors.channelwise_tensor_product( + irreps_in, irreps_sh, [cue.SO3(0), cue.SO3(1)] + ) + projection = cue.descriptors.linear(product.outputs[0].irreps, irreps_out) + harmonics = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + return irreps_in, irreps_out, product, projection, harmonics + + +def graph_reference(params, x, vectors, senders, receivers, num_dst): + """Core NumPy descriptors plus an explicit edge loop; no frontend forward.""" + import numpy as np + + irreps_in, irreps_out, product, projection, harmonics = graph_descriptors() + lengths = np.linalg.norm(vectors, axis=-1, keepdims=True) + weights = np.tanh(lengths @ params["radial_in"]) @ params["radial_out"] + sh = harmonics(vectors / lengths)[0] + flat = change_layout(x, irreps_in, source="mul_ir") + result = np.zeros((num_dst, irreps_out.dim), dtype=np.float64) + for edge, (src, dst) in enumerate(zip(senders, receivers)): + message = product(weights[edge], flat[src], sh[edge])[0] + projected = projection(params["projection"], message)[0] + result[dst] += change_layout(projected, irreps_out, source="ir_mul") + return result + + +def graph_inputs(seed): + import numpy as np + + rng = np.random.default_rng(seed) + irreps_in, _, product, projection, _ = graph_descriptors() + params = { + "radial_in": rng.normal(scale=0.4, size=(1, 4)), + "radial_out": rng.normal(scale=0.4, size=(4, product.inputs[0].dim)), + "projection": rng.normal(scale=0.4, size=(projection.inputs[0].dim,)), + } + x = rng.normal(size=(5, irreps_in.dim)) + vectors = rng.normal(size=(7, 3)) + np.array([0.4, -0.2, 0.7]) + senders = np.array([0, 1, 3, 0, 1, 4, 2], dtype=np.int64) + receivers = np.array([2, 0, 2, 1, 2, 0, 1], dtype=np.int64) + return params, x, vectors, senders, receivers + + +def check_graph(solution, backend): + import numpy as np + + import cuequivariance as cue + + if backend == "jax": + import jax + import jax.numpy as jnp + + jax.config.update("jax_enable_x64", True) + else: + import torch + + irreps_in, irreps_out, _, _, _ = graph_descriptors() + for seed in (41, 83): + params, x, vectors, senders, receivers = graph_inputs(seed) + expected = graph_reference(params, x, vectors, senders, receivers, 4) + if backend == "jax": + p = {k: jnp.asarray(v) for k, v in params.items()} + a, b, s, r = map(jnp.asarray, (x, vectors, senders, receivers)) + + def apply(pp, xx, vv, ss, rr): + return solution.apply(pp, xx, vv, ss, rr, 4) + + actual = apply(p, a, b, s, r) + close(jax.jit(apply)(p, a, b, s, r), expected) + + def loss(pp, xx, vv): + return jnp.sum(apply(pp, xx, vv, s, r) ** 2) + + derivatives = jax.grad(loss, argnums=(0, 1, 2))(p, a, b) + close( + jax.jit(jax.grad(loss))(p, a, b)["radial_in"], + derivatives[0]["radial_in"], + ) + grad_params, grad_x, grad_vectors = derivatives + else: + model = solution.GraphLayer().double() + assert isinstance(model, torch.nn.Module) + registered = dict(model.named_parameters()) + assert set(registered) == set(PARAMETERS), ( + "register exactly the three specified parameters" + ) + with torch.no_grad(): + for name, value in params.items(): + assert registered[name].shape == value.shape, name + registered[name].copy_(torch.tensor(value)) + a = torch.tensor(x, requires_grad=True) + b = torch.tensor(vectors, requires_grad=True) + s, r = torch.tensor(senders), torch.tensor(receivers) + + def apply(pp, xx, vv, ss, rr): + return model(xx, vv, ss, rr, 4) + + p = registered + actual = apply(p, a, b, s, r) + grads = torch.autograd.grad( + actual.square().sum(), [*(registered[k] for k in PARAMETERS), a, b] + ) + grad_params = dict(zip(PARAMETERS, grads[:3])) + grad_x, grad_vectors = grads[-2:] + close(actual, expected) + close(apply(p, a, b, s, r), expected) + close(actual[3], np.zeros(4), exact=True) + + # Relabel source nodes and reorder edges independently. + order = np.array([3, 0, 4, 1, 2]) + inverse = np.argsort(order) + edge_order = np.array([6, 1, 4, 0, 3, 5, 2]) + if backend == "jax": + new_s, new_r = map( + jnp.asarray, (inverse[senders][edge_order], receivers[edge_order]) + ) + else: + new_s, new_r = map( + torch.tensor, (inverse[senders][edge_order], receivers[edge_order]) + ) + close(apply(p, a[order], b[edge_order], new_s, new_r), expected) + destination_order = np.array([3, 1, 0, 2]) + destination_inverse = np.argsort(destination_order) + cast = jnp.asarray if backend == "jax" else torch.tensor + close( + apply(p, a, b, s, cast(destination_inverse[receivers])), + expected[destination_order], + ) + if backend == "jax": + resized = solution.apply(p, a[:4], b[:4], s[:4], r[:4], 6) + else: + resized = model(a[:4], b[:4], s[:4], r[:4], 6) + close( + resized, + graph_reference(params, x[:4], vectors[:4], senders[:4], receivers[:4], 6), + ) + angles = [0.31, -0.23, 0.41] + din = cue.IrrepsAndLayout(irreps_in, cue.mul_ir).exp_map(angles, []) + dout = cue.IrrepsAndLayout(irreps_out, cue.mul_ir).exp_map(angles, []) + dv = cue.SO3(1).exp_map(angles, []) + close(apply(p, a @ cast(din.T), b @ cast(dv.T), s, r), expected @ dout.T) + + # Check one informative derivative in every parameter/input against NumPy. + gradients = {**grad_params, "features": grad_x, "vectors": grad_vectors} + values = {**params, "features": x, "vectors": vectors} + for name, gradient in gradients.items(): + gradient = array(gradient) + assert np.isfinite(gradient).all() and np.max(np.abs(gradient)) > 1e-9, name + index = np.unravel_index(np.argmax(np.abs(gradient)), gradient.shape) + losses = [] + for sign in (-1, 1): + shifted = {k: v.copy() for k, v in values.items()} + shifted[name][index] += sign * 1e-5 + y = graph_reference( + {k: shifted[k] for k in PARAMETERS}, + shifted["features"], + shifted["vectors"], + senders, + receivers, + 4, + ) + losses.append(np.sum(y**2)) + np.testing.assert_allclose( + gradient[index], (losses[1] - losses[0]) / 2e-5, rtol=2e-5, atol=2e-7 + ) + # A real parameter update must retain the reference semantics. + updated = {k: params[k] - 1e-3 * array(grad_params[k]) for k in PARAMETERS} + if backend == "jax": + after = apply({k: jnp.asarray(v) for k, v in updated.items()}, a, b, s, r) + else: + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer.zero_grad() + for name in PARAMETERS: + p[name].grad = grad_params[name].detach().clone() + optimizer.step() + after = apply(p, a, b, s, r) + assert {k: id(v) for k, v in model.named_parameters()} == { + k: id(v) for k, v in registered.items() + } + close(after, graph_reference(updated, x, vectors, senders, receivers, 4)) + assert not np.allclose(array(after), array(actual)), ( + "update did not affect output" + ) + + +def check_routing(solution, backend, path): + import numpy as np + + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + assert not any(a.name.startswith("cuequivariance") for a in node.names) + if isinstance(node, ast.ImportFrom): + assert not (node.module or "").startswith("cuequivariance") + for count in (1, 3, 7): + predictions = np.linspace(-0.7, 1.1, count).reshape(count, 1) + targets = np.linspace(1.2, -0.1, count) + expected = np.mean((predictions[:, 0] - targets) ** 2) + expected_grad = 2 * (predictions - targets[:, None]) / count + if backend == "jax": + import jax + import jax.numpy as jnp + + jax.config.update("jax_enable_x64", True) + p, t = jnp.asarray(predictions), jnp.asarray(targets) + value = jax.jit(solution.mse_loss)(p, t) + gradient = jax.grad(solution.mse_loss)(p, t) + else: + import torch + + p = torch.tensor(predictions, requires_grad=True) + value = solution.mse_loss(p, torch.tensor(targets)) + gradient = torch.autograd.grad(value, p)[0] + close(value, expected) + close(gradient, expected_grad) + + +def worker(backend, case, solution): + try: + module = load_solution(solution) + if case == "layout": + check_layout(module, backend) + elif case == "graph": + check_graph(module, backend) + else: + check_routing(module, backend, solution) + except Exception as error: + return { + "passed": False, + "error": f"{type(error).__name__}: {error}", + "traceback": traceback.format_exc(), + } + return {"passed": True} + + +def evaluate(backend, case, solution, *, timeout=180, python=None): + command = [ + python or sys.executable, + str(Path(__file__).resolve()), + "--worker", + "--backend", + backend, + "--case", + case, + "--solution", + str(solution.resolve()), + ] + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, "JAX_PLATFORMS": "cpu", "PYTHONDONTWRITEBYTECODE": "1"}, + ) + marker = "CUEQ_RESULT=" + if result.returncode or marker not in result.stdout: + return { + "passed": False, + "error": f"verifier exit {result.returncode}", + "output": (result.stdout + result.stderr)[-6000:], + } + return json.loads(result.stdout.rsplit(marker, 1)[1].splitlines()[0]) + except subprocess.TimeoutExpired: + return {"passed": False, "error": f"submission exceeded {timeout} seconds"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("jax", "torch")) + parser.add_argument("--case", choices=("layout", "graph", "routing")) + parser.add_argument("--solution", type=Path) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--worker", action="store_true") + args = parser.parse_args() + harbor = args.backend is None + if harbor: + entry_path = Path(os.environ.get("HARBOR_ENTRY_JSON", "/tests/entry.json")) + entry = json.loads(entry_path.read_text()) + backend, _, case, version = entry["id"].split("-") + assert version == "v1" and backend in ("jax", "torch") + assert case in ("layout", "graph", "routing") + args.backend, args.case = backend, case + args.solution = Path("/workspace/solution.py") + elif args.case is None or args.solution is None: + parser.error("--backend requires --case and --solution") + if args.worker: + print( + "CUEQ_RESULT=" + json.dumps(worker(args.backend, args.case, args.solution)) + ) + return + interpreter = ( + Path("/workspace/.venv/bin/python") if harbor else Path(sys.executable) + ) + result = evaluate( + args.backend, + args.case, + args.solution, + timeout=args.timeout, + python=str(interpreter), + ) + print(json.dumps(result, indent=2)) + if harbor: + reward = Path( + os.environ.get("HARBOR_REWARD_JSON", "/logs/verifier/reward.json") + ) + reward.parent.mkdir(parents=True, exist_ok=True) + reward.write_text( + json.dumps( + { + "custom_metrics": {"numerical_success": float(result["passed"])}, + "details": {"numerical_success": result}, + }, + indent=2, + ) + ) + elif not result["passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/evals/reference_jax_challenges.py b/.github/evals/reference_jax_challenges.py new file mode 100644 index 00000000..80c1bfcd --- /dev/null +++ b/.github/evals/reference_jax_challenges.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Maintainer reference only. Never copy this file into an agent environment.""" + +import cuequivariance_jax.ir_dict as ir_dict +import jax +import jax.numpy as jnp + +import cuequivariance as cue +import cuequivariance_jax as cuex + +jax.config.update("jax_enable_x64", True) + + +def to_blocks(irreps, flat): + data = cuex.RepArray(cue.IrrepsAndLayout(irreps, cue.ir_mul), flat).regroup() + return ir_dict.flat_to_dict(data.irreps, data.array, layout="ir_mul") + + +def from_blocks(irreps, blocks): + offsets, parts = {}, [] + for mul, ir in irreps: + start = offsets.get(ir, 0) + block = blocks[ir][..., start : start + mul, :] + parts.append(block.swapaxes(-1, -2).reshape(block.shape[:-2] + (mul * ir.dim,))) + offsets[ir] = start + mul + return jnp.concatenate(parts, axis=-1) + + +INPUT = cue.Irreps("SO3", "2x0 + 2x1") +OUTPUT = cue.Irreps("SO3", "0 + 1") +PRODUCT = cue.descriptors.channelwise_tensor_product( + INPUT, cue.Irreps("SO3", "0 + 1 + 2"), [cue.SO3(0), cue.SO3(1)] +) +PROJECTION = cue.descriptors.linear(PRODUCT.outputs[0].irreps, OUTPUT) +HARMONICS = cue.descriptors.spherical_harmonics(cue.SO3(1), [0, 1, 2]) + + +def transpose(flat, irreps, *, source): + parts, offset = [], 0 + for mul, ir in irreps: + suffix = (mul, ir.dim) if source == "mul_ir" else (ir.dim, mul) + block = flat[..., offset : offset + mul * ir.dim] + parts.append( + block.reshape(flat.shape[:-1] + suffix) + .swapaxes(-1, -2) + .reshape(flat.shape[:-1] + (mul * ir.dim,)) + ) + offset += mul * ir.dim + return jnp.concatenate(parts, axis=-1) + + +def run(descriptor, operands, rows): + return cuex.segmented_polynomial( + descriptor.polynomial, + operands, + [jax.ShapeDtypeStruct((rows, descriptor.outputs[0].dim), jnp.float64)], + method="naive", + )[0] + + +def apply(params, features, vectors, senders, receivers, num_dst): + lengths = jnp.linalg.norm(vectors, axis=-1, keepdims=True) + weights = jnp.tanh(lengths @ params["radial_in"]) @ params["radial_out"] + sh = run(HARMONICS, [vectors / lengths], vectors.shape[0]) + inputs = transpose(features, INPUT, source="mul_ir") + messages = run(PRODUCT, [weights, inputs[senders], sh], vectors.shape[0]) + summed = ( + jnp.zeros((num_dst, messages.shape[-1]), messages.dtype) + .at[receivers] + .add(messages) + ) + output = run(PROJECTION, [params["projection"], summed], num_dst) + return transpose(output, OUTPUT, source="ir_mul") diff --git a/.github/evals/reference_torch_challenges.py b/.github/evals/reference_torch_challenges.py new file mode 100644 index 00000000..68444c43 --- /dev/null +++ b/.github/evals/reference_torch_challenges.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Maintainer reference only. Never copy this file into an agent environment.""" + +import torch + +import cuequivariance as cue +import cuequivariance_torch as cuet + + +def to_blocks(irreps, flat): + parts, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)).transpose(-1, -2) + parts.setdefault(ir, []).append(block) + offset += mul * ir.dim + return {ir: torch.cat(values, dim=-2) for ir, values in parts.items()} + + +def from_blocks(irreps, blocks): + offsets, parts = {}, [] + for mul, ir in irreps: + start = offsets.get(ir, 0) + block = blocks[ir][..., start : start + mul, :] + parts.append( + block.transpose(-1, -2).reshape(block.shape[:-2] + (mul * ir.dim,)) + ) + offsets[ir] = start + mul + return torch.cat(parts, dim=-1) + + +class GraphLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.product = cuet.ChannelWiseTensorProduct( + cue.Irreps("SO3", "2x0 + 2x1"), + cue.Irreps("SO3", "0 + 1 + 2"), + [cue.SO3(0), cue.SO3(1)], + shared_weights=False, + internal_weights=False, + layout=cue.mul_ir, + method="naive", + dtype=torch.float64, + ) + self.readout = cuet.Linear( + self.product.irreps_out, + cue.Irreps("SO3", "0 + 1"), + internal_weights=False, + layout=cue.mul_ir, + method="naive", + dtype=torch.float64, + ) + self.sh = cuet.SphericalHarmonics([0, 1, 2], normalize=True, method="naive") + self.radial_in = torch.nn.Parameter(torch.zeros(1, 4, dtype=torch.float64)) + self.radial_out = torch.nn.Parameter( + torch.zeros(4, self.product.weight_numel, dtype=torch.float64) + ) + self.projection = torch.nn.Parameter( + torch.zeros(self.readout.weight_numel, dtype=torch.float64) + ) + + def forward(self, features, vectors, senders, receivers, num_dst): + lengths = torch.linalg.vector_norm(vectors, dim=-1, keepdim=True) + weights = torch.tanh(lengths @ self.radial_in) @ self.radial_out + messages = self.product( + features, + self.sh(vectors), + weight=weights, + indices_1=senders, + indices_out=receivers, + size_out=num_dst, + ) + return self.readout(messages, self.projection[None, :]) diff --git a/.github/scripts/prepare_skill_challenges.py b/.github/scripts/prepare_skill_challenges.py new file mode 100644 index 00000000..8324acb3 --- /dev/null +++ b/.github/scripts/prepare_skill_challenges.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Stage challenge-v1 pilots without changing canonical skills or regression evals. + +Run with Python + PyYAML. --export-harbor additionally needs SkillEvaluator 1.5.6 +or a compatible builder that separates verifier material from runtime skills. +All output is created in a fresh directory; this command never launches agents. +""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import shlex +import shutil +import tempfile +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +GRADER = ROOT / ".github/evals/cueq_challenges.py" +# Match the models resolved in the user's 2026-10-20 comparison. +MODELS = { + "claude-code": "aws/anthropic/bedrock-claude-opus-4-8", + "codex": "openai/openai/gpt-5.5", +} + + +def digest(path): + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify_environment(task, backend, with_skill): + """Fail the export if evaluator or reference material reaches the agent.""" + environment = task / "environment" + for path in environment.rglob("*"): + relative = path.relative_to(environment) + assert "evals" not in relative.parts, relative + assert path.name != "grader.py", relative + assert not path.name.startswith("reference_"), relative + assert path.name not in ("entry.json", "challenges.json"), relative + skill = environment / "skills" / f"cuequivariance-{backend}" / "SKILL.md" + assert skill.exists() == with_skill + if with_skill: + assert digest(skill) == digest(ROOT / "skills" / skill.parent.name / "SKILL.md") + assert (task / "tests/grader.py").read_bytes() == GRADER.read_bytes() + entry = json.loads((task / "tests/entry.json").read_text()) + assert entry["has_skill"] == with_skill + + +def prepare(destination, backends, *, export_harbor=False): + destination = Path(destination).resolve() + destination.mkdir(parents=True, exist_ok=False) + if export_harbor: + from skillevaluator.tier3.harbor.adapter import generate_harbor_tasks + + if ( + "evaluator_skill_path" + not in inspect.signature(generate_harbor_tasks).parameters + ): + raise RuntimeError( + "Use the SkillEvaluator 1.5.6 builder with private evaluator-source support" + ) + manifest = { + "benchmark": "cuequivariance-challenges-v1", + "attempts_per_case_per_arm": 3, + "stop_on_pass": False, + "agents": ["claude-code", "codex"], + "agent_models": MODELS, + "grading": "aces_plus_custom", + "custom_metric": "numerical_success", + "custom_metric_changes_aces_gate": False, + "skills": {}, + } + for backend in backends: + name = f"cuequivariance-{backend}" + source, staged = ROOT / "skills" / name, destination / "skills" / name + staged.mkdir(parents=True) + shutil.copy2(source / "SKILL.md", staged / "SKILL.md") + # Only runtime resources and the challenge evaluator are staged. + for directory in ("scripts", "references", "assets", "agents"): + if (source / directory).is_dir(): + shutil.copytree(source / directory, staged / directory) + evals = staged / "evals" + evals.mkdir() + shutil.copy2(source / "evals/challenges.json", evals / "evals.json") + shutil.copy2(GRADER, evals / "grader.py") + config = yaml.safe_load((source / "evals/config.yml").read_text()) + config["grading"]["mode"] = "aces_plus_custom" + (evals / "config.yml").write_text(yaml.safe_dump(config, sort_keys=False)) + command = [ + "skillevaluator", + "tier3", + "evaluate", + str(staged), + "--env-mode", + "k8s-sandbox", + "--agents", + "claude-code,codex", + "--agent-model", + f"claude-code={MODELS['claude-code']}", + "--agent-model", + f"codex={MODELS['codex']}", + "--n-attempts", + "3", + "--no-stop-on-pass", + "--n-concurrent", + "1", + "--max-agents", + "1", + "--harbor-keep-jobs", + "--results-dir", + str(destination / "results" / name), + ] + record = { + "path": str(staged), + "skill_digest": digest(staged / "SKILL.md"), + "dataset_digest": digest(evals / "evals.json"), + "config_digest": digest(evals / "config.yml"), + "grader_digest": digest(evals / "grader.py"), + "case_ids": [ + e["id"] for e in json.loads((evals / "evals.json").read_text())["evals"] + ], + "command": command, + } + if export_harbor: + tasks_by_arm = {} + for arm, with_skill in (("with_skill", True), ("baseline", False)): + task_root = destination / "harbor" / backend / arm + tasks = generate_harbor_tasks( + staged, + task_root, + evaluator_skill_path=staged, + with_skill=with_skill, + workspace_mode="isolated", + grading_mode="aces_plus_custom", + env_mode="astra-sandbox", + runtime_env=config["harbor"]["runtime_env"], + pre_agent_setup=config["harbor"]["pre_agent_setup"], + agent_workdir="/workspace", + copy_repo=False, + repo_context_exclude_paths=[ROOT / ".github/evals"], + ) + for task in tasks: + verify_environment(task, backend, with_skill) + tasks_by_arm[arm] = tasks + # Inputs and the verifier must be identical in the two conditions. + for with_task, without_task in zip( + tasks_by_arm["with_skill"], tasks_by_arm["baseline"], strict=True + ): + assert (with_task / "instruction.md").read_bytes() == ( + without_task / "instruction.md" + ).read_bytes() + assert (with_task / "tests/grader.py").read_bytes() == ( + without_task / "tests/grader.py" + ).read_bytes() + record["verified_harbor_tasks"] = sum(map(len, tasks_by_arm.values())) + manifest["skills"][name] = record + (destination / "pilot.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("jax", "torch", "both"), default="both") + parser.add_argument( + "--output", type=Path, help="New directory; existing paths are refused." + ) + parser.add_argument("--export-harbor", action="store_true") + args = parser.parse_args() + if args.output is None: + root = ROOT / ".venv" + root.mkdir(exist_ok=True) + # The parent is unique; prepare owns a new, initially absent child. + args.output = ( + Path(tempfile.mkdtemp(prefix="cueq-challenges-v1-", dir=root)) / "pilot" + ) + backends = ("jax", "torch") if args.backend == "both" else (args.backend,) + manifest = prepare(args.output, backends, export_harbor=args.export_harbor) + print(f"Pilot manifest: {args.output.resolve() / 'pilot.json'}") + for record in manifest["skills"].values(): + print(shlex.join(record["command"])) + + +if __name__ == "__main__": + main() diff --git a/.github/tests/challenge_checks.py b/.github/tests/challenge_checks.py new file mode 100644 index 00000000..4d931a10 --- /dev/null +++ b/.github/tests/challenge_checks.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared checks of the challenge verifier, references, and deliberate defects.""" + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +EVALS = Path(__file__).resolve().parents[1] / "evals" +spec = importlib.util.spec_from_file_location( + "cueq_challenge_verifier", EVALS / "cueq_challenges.py" +) +grader = importlib.util.module_from_spec(spec) +spec.loader.exec_module(grader) + + +def reference(backend): + return grader.load_solution(EVALS / f"reference_{backend}_challenges.py") + + +def run_mutation(backend, defect): + original = reference(backend) + if defect in ("overwrite_duplicate", "reshape_only", "merge_parity"): + + def broken(irreps, flat): + if defect == "merge_parity": + return { + ir.l: value + for ir, value in original.to_blocks(irreps, flat).items() + } + if defect == "reshape_only": + return { + ir: value.swapaxes(-1, -2).reshape(value.shape) + for ir, value in original.to_blocks(irreps, flat).items() + } + blocks, offset = {}, 0 + for mul, ir in irreps: + block = flat[..., offset : offset + mul * ir.dim] + block = block.reshape(flat.shape[:-1] + (ir.dim, mul)) + block = block.swapaxes(-1, -2) + blocks[ir] = block + offset += mul * ir.dim + return blocks + + mutant = SimpleNamespace(to_blocks=broken, from_blocks=original.from_blocks) + with pytest.raises((AssertionError, KeyError, ValueError)): + grader.check_layout(mutant, backend) + return + if backend == "jax": + import jax + import jax.numpy as jnp + + def broken(params, x, v, s, r, n): + if defect == "shared_radial": + v = jnp.broadcast_to(v[0], v.shape) + if defect == "detached_vectors": + v = jax.lax.stop_gradient(v) + output = original.apply(params, x, v, s, r, n) + if defect == "mean_reduction": + # Average repeated destinations instead of summing them. + counts = jnp.bincount(r, length=n).clip(1)[:, None] + output = output / counts + if defect == "wrong_layout": + output = original.apply(params, x[:, ::-1], v, s, r, n) + return output + + mutant = SimpleNamespace(apply=broken) + else: + import torch + + class Broken(original.GraphLayer): + def forward(self, x, v, s, r, n): + if defect == "shared_radial": + v = v[0].expand_as(v) + if defect == "detached_vectors": + v = v.detach() + output = super().forward(x, v, s, r, n) + if defect == "mean_reduction": + output = ( + output / torch.bincount(r, minlength=n).clamp_min(1)[:, None] + ) + if defect == "wrong_layout": + output = super().forward(x.flip(-1), v, s, r, n) + return output + + mutant = SimpleNamespace(GraphLayer=Broken) + with pytest.raises((AssertionError, RuntimeError)): + grader.check_graph(mutant, backend) + + +def check_routing(backend, tmp_path): + path = tmp_path / "solution.py" + path.write_text( + "def mse_loss(predictions, targets):\n" + " return ((predictions[:, 0] - targets) ** 2).mean()\n" + ) + grader.check_routing(grader.load_solution(path), backend, path) + # Shape-valid broadcasting is precisely the bug this negative case asks to fix. + path.write_text( + "def mse_loss(predictions, targets):\n" + " return ((predictions - targets) ** 2).mean()\n" + ) + with pytest.raises(AssertionError): + grader.check_routing(grader.load_solution(path), backend, path) diff --git a/.github/tests/test_jax_skill_challenges.py b/.github/tests/test_jax_skill_challenges.py new file mode 100644 index 00000000..4e3d6b14 --- /dev/null +++ b/.github/tests/test_jax_skill_challenges.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical acceptance and defect rejection for challenge-v1 JAX submissions.""" + +import pytest +from challenge_checks import check_routing, grader, reference, run_mutation + + +def test_layout_reference(): + grader.check_layout(reference("jax"), "jax") + + +def test_graph_reference(): + grader.check_graph(reference("jax"), "jax") + + +@pytest.mark.parametrize( + "defect", + [ + "overwrite_duplicate", + "reshape_only", + "merge_parity", + "shared_radial", + "detached_vectors", + "mean_reduction", + "wrong_layout", + ], +) +def test_rejects_wrong_solution(defect): + run_mutation("jax", defect) + + +def test_nearby_negative_and_broadcast_defect(tmp_path): + check_routing("jax", tmp_path) diff --git a/.github/tests/test_skill_challenge_packaging.py b/.github/tests/test_skill_challenge_packaging.py new file mode 100644 index 00000000..5ac3b788 --- /dev/null +++ b/.github/tests/test_skill_challenge_packaging.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Preserve regression inputs, isolate reference answers, and bound verification.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest +import yaml +from challenge_checks import EVALS, grader + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "prepare_skill_challenges", ROOT / ".github/scripts/prepare_skill_challenges.py" +) +preparer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preparer) + + +def test_pilot_keeps_skill_and_environment_fixed_and_excludes_answers(tmp_path): + original = {} + for backend in ("jax", "torch"): + source = ROOT / "skills" / f"cuequivariance-{backend}" + for relative in ("SKILL.md", "evals/config.yml", "evals/evals.json"): + original[source / relative] = (source / relative).read_bytes() + manifest = preparer.prepare(tmp_path / "pilot", ("jax", "torch")) + assert manifest["attempts_per_case_per_arm"] == 3 + assert manifest["stop_on_pass"] is False + for backend in ("jax", "torch"): + name = f"cuequivariance-{backend}" + record = manifest["skills"][name] + source, staged = ROOT / "skills" / name, Path(record["path"]) + assert (staged / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes() + config = yaml.safe_load((staged / "evals/config.yml").read_text()) + original_config = yaml.safe_load((source / "evals/config.yml").read_text()) + assert config["harbor"] == original_config["harbor"] + assert config["grading"]["mode"] == "aces_plus_custom" + entries = json.loads((staged / "evals/evals.json").read_text())["evals"] + assert len(entries) == 3 + assert len({e["id"] for e in entries}) == len(entries) + assert sum(e["expected_skill"] == name for e in entries) == 2 + assert sum(e["expected_skill"] is None for e in entries) == 1 + assert all("v1" in e["id"] for e in entries) + assert not list(staged.rglob("reference_*")) + assert preparer.digest(staged / "evals/grader.py") == record["grader_digest"] + assert record["command"][record["command"].index("--n-attempts") + 1] == "3" + assert "--skip-baseline" not in record["command"] + for agent, model in manifest["agent_models"].items(): + assert f"{agent}={model}" in record["command"] + for path, content in original.items(): + assert path.read_bytes() == content + with pytest.raises(FileExistsError): + preparer.prepare(tmp_path / "pilot", ("jax",)) + + +def test_missing_submission_and_timeout_are_failures(tmp_path): + missing = grader.evaluate( + "jax", "layout", tmp_path / "missing.py", python=sys.executable + ) + assert not missing["passed"] and "FileNotFoundError" in missing["error"] + hanging = tmp_path / "solution.py" + hanging.write_text("while True: pass\n") + timed_out = grader.evaluate("jax", "layout", hanging, timeout=0.2) + assert not timed_out["passed"] and "exceeded" in timed_out["error"] + + +def test_submission_import_supports_sibling_helpers_and_dataclasses( + tmp_path, monkeypatch +): + monkeypatch.setattr(sys, "path", sys.path.copy()) + (tmp_path / "challenge_import_helper.py").write_text("VALUE = 7\n") + solution = tmp_path / "solution.py" + solution.write_text( + "from __future__ import annotations\n" + "from dataclasses import dataclass\n" + "from challenge_import_helper import VALUE\n" + "@dataclass\n" + "class Result:\n" + " value: int = VALUE\n" + ) + assert grader.load_solution(solution).Result().value == 7 + + +@pytest.mark.parametrize("leak", ("evals/evals.json", "grader.py", "reference_jax.py")) +def test_export_rejects_evaluation_material_in_agent_bundle(tmp_path, leak): + task = tmp_path / "task" + path = task / "environment" / leak + path.parent.mkdir(parents=True) + path.write_text("must not be visible to the agent") + with pytest.raises(AssertionError): + preparer.verify_environment(task, "jax", False) + + +def test_harbor_dispatch_writes_a_real_failure_reward(tmp_path, monkeypatch): + entry = tmp_path / "entry.json" + entry.write_text(json.dumps({"id": "jax-challenge-layout-v1"})) + reward = tmp_path / "reward.json" + monkeypatch.setenv("HARBOR_ENTRY_JSON", str(entry)) + monkeypatch.setenv("HARBOR_REWARD_JSON", str(reward)) + monkeypatch.setattr(sys, "argv", [str(EVALS / "cueq_challenges.py")]) + observed = {} + + def failed(backend, case, solution, **kwargs): + observed.update(backend=backend, case=case, solution=str(solution), **kwargs) + return {"passed": False, "error": "numerical mismatch"} + + monkeypatch.setattr(grader, "evaluate", failed) + grader.main() + assert observed["backend"] == "jax" and observed["case"] == "layout" + assert observed["solution"] == "/workspace/solution.py" + assert observed["python"] == "/workspace/.venv/bin/python" + result = json.loads(reward.read_text()) + assert result["custom_metrics"]["numerical_success"] == 0 + assert "overall" not in result and "goal_accuracy" not in result diff --git a/.github/tests/test_torch_skill_challenges.py b/.github/tests/test_torch_skill_challenges.py new file mode 100644 index 00000000..bb92011b --- /dev/null +++ b/.github/tests/test_torch_skill_challenges.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical acceptance and defect rejection for challenge-v1 Torch submissions.""" + +import pytest +from challenge_checks import check_routing, grader, reference, run_mutation + + +def test_layout_reference(): + grader.check_layout(reference("torch"), "torch") + + +def test_graph_reference(): + grader.check_graph(reference("torch"), "torch") + + +@pytest.mark.parametrize( + "defect", + [ + "overwrite_duplicate", + "reshape_only", + "merge_parity", + "shared_radial", + "detached_vectors", + "mean_reduction", + "wrong_layout", + ], +) +def test_rejects_wrong_solution(defect): + run_mutation("torch", defect) + + +def test_nearby_negative_and_broadcast_defect(tmp_path): + check_routing("torch", tmp_path) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7adb2d43..1c0b727a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,7 +72,7 @@ jobs: env: JAX_PLATFORMS: cpu run: | - pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_skill_execution.py .github/tests/test_skill_eval_setup.py + pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_jax_skill_challenges.py .github/tests/test_skill_challenge_packaging.py .github/tests/test_skill_execution.py .github/tests/test_skill_eval_setup.py uv pip install hatchling python .github/scripts/check_skill_packaging.py --package cuequivariance_jax @@ -122,6 +122,6 @@ jobs: - name: Check PyTorch skill evaluations and packaging run: | - pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py + pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py .github/tests/test_torch_skill_challenges.py uv pip install hatchling python .github/scripts/check_skill_packaging.py --package cuequivariance_torch diff --git a/skills/README.md b/skills/README.md index c4365851..a29d9a10 100644 --- a/skills/README.md +++ b/skills/README.md @@ -28,7 +28,9 @@ how to find code in a checkout or an installed package. ## Evaluation coverage -Each dataset contains five positive cases and one unrelated negative case. +Each regression dataset (`evals/evals.json`) contains five positive cases and one +unrelated negative case. The JAX and PyTorch skills also have a separate +`evals/challenges.json` capability dataset; the original six cases are unchanged. The core cases were adapted from the BioNeMo agent toolkit's cuEquivariance evaluation ideas; they use this repository's APIs and contain no references to BioNeMo-only skill resources. The JAX and PyTorch cases exercise their respective @@ -84,8 +86,8 @@ a development checkout with the relevant frontend dependencies installed: ```bash pytest -q .github/tests/test_core_skill_evals.py -JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py -pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py +JAX_PLATFORMS=cpu pytest -q .github/tests/test_jax_skill_evals.py .github/tests/test_jax_skill_examples.py .github/tests/test_jax_skill_challenges.py .github/tests/test_skill_challenge_packaging.py +pytest -q .github/tests/test_torch_skill_evals.py .github/tests/test_torch_skill_examples.py .github/tests/test_torch_skill_challenges.py ``` To verify the pinned evaluation environment in a separate Python 3.12 environment: @@ -107,6 +109,87 @@ skillevaluator tier3 validate skills/cuequivariance-jax skillevaluator tier3 validate skills/cuequivariance-torch ``` +### Challenge-v1 capability pilot + +The challenge datasets contain two positive integration tasks and one nearby +negative task per frontend. Their IDs deliberately differ from the regression +IDs so results from the two datasets cannot be paired accidentally. + +| Case suffix | Required outcome | +| --- | --- | +| `challenge-layout-v1` | Migrate original O3 `ir_mul` buffers with separated duplicate blocks and both parities into dictionaries, reconstruct the original order, and retain gradients, batch axes and transformation behavior. JAX also requires JIT; Torch includes noncontiguous tensors. | +| `challenge-graph-v1` | Compose radial weights, normalized harmonics, a channelwise product, destination accumulation and a projection. Match an independent NumPy edge-loop reference before and after a training update, retain gradients through every parameter and both floating inputs, and handle relabeling, edge permutations and isolated destinations. | +| `challenge-routing-v1` | Repair an ordinary JAX/Torch regression-loss broadcasting bug without invoking cuEquivariance or its skill. | + +Prompts specify the public interface, mathematics, constraints and observable +outcomes. They leave the frontend APIs and implementation strategy to the agent. +Every task writes an importable `/workspace/solution.py`; execution claims remain +subject to the ordinary trajectory checks. + +`.github/evals/cueq_challenges.py` independently executes the submitted artifact +in a bounded subprocess using the prepared CPU interpreter. The positive +oracles use original block slicing and core NumPy descriptors rather than the +submitted frontend forward. Checks include multiple data seeds, independent +forward values, finite differences, rotations, O3 inversion where applicable, +graph resizing and permutations, and a real parameter update. +It emits a binary `numerical_success` custom metric: all required numerical +checks must pass. A missing file, exception, detached gradient or timeout fails. +The nearby negative additionally checks the loss values and prediction gradients; +the ACES routing check detects skill activation beyond imports in the submission. + +The maintainer references (`.github/evals/reference_*_challenges.py`) are outside +all skill directories and are never included in the pilot. Regression tests run +the verifier against those references and deliberate defects: overwritten +duplicate irreps, wrong layouts, parity merging, incorrectly shared edge data, +detached vectors, mean instead of sum aggregation, and the original MSE +broadcasting bug. These tests validate the benchmark and verifier, not agent lift. + +Prepare a fresh pilot with Python, PyYAML and SkillEvaluator 1.5.6 available: + +```bash +python .github/scripts/prepare_skill_challenges.py --export-harbor +``` + +This creates a unique directory under `.venv/` and prints a manifest location +and two evaluation commands. `--backend jax` or `--backend torch` selects one +frontend. `--output PATH` selects a new output directory; existing directories +are refused. Without `--export-harbor`, preparation needs only Python and PyYAML. + +The staged skills retain the exact canonical `SKILL.md`, requirements and +environment setup. Only the selected dataset and grading mode change: +`aces_plus_custom` adds artifact verification while retaining the default ACES +metrics and quality gate. **`numerical_success` does not replace ACES's overall +score or change its gate.** Read both the numerical result and the normal +trajectory assessment; neither should be inferred from the other. + +The export validates both conditions, identical prompts/verifiers, and the +absence of eval metadata, grader code and reference solutions in the agent's +runtime bundle. This requires the builder's private evaluator-source support; +older builders are rejected. Do not use `--copy-repo`: the maintenance checkout +contains the reference answers. + +The manifest records skill, dataset, configuration and verifier digests. Its +commands pin the models from the 2026-10-20 comparison: +`claude-code=aws/anthropic/bedrock-claude-opus-4-8` and +`codex=openai/openai/gpt-5.5`. They request baseline plus skill, three attempts +per case and no early stop on success: 36 executions per frontend, 72 for both. +Confirm those models are still available in the authenticated environment; +changing a model requires a new baseline with that same model. +The preparation command only stages and validates; it does not launch agents. +Before running the printed commands in the authenticated CI/runtime environment: + +```bash +skillevaluator tier3 doctor --env-mode k8s-sandbox --agents claude-code,codex +``` + +Keep all attempts, infrastructure failures and trial artifacts. Fix the skill +text only after this first paired pilot, then measure both revisions on the +same frozen challenge dataset and environment. Reserve additional numerical +variants for later validation. Do not compare challenge lift directly with the +earlier six-case regression lift, or interpret local reference passes as evidence +of agent improvement. The ordinary NVSkills workflow continues to discover +`evals/evals.json`; promotion of the new benchmark follows the pilot. + ### Interpreting Tier 3 comparisons Use the exported JSON and per-trial evidence to compare revisions. The pipeline diff --git a/skills/cuequivariance-jax/evals/challenges.json b/skills/cuequivariance-jax/evals/challenges.json new file mode 100644 index 00000000..b0cc9ba9 --- /dev/null +++ b/skills/cuequivariance-jax/evals/challenges.json @@ -0,0 +1,39 @@ +{ + "skill_name": "cuequivariance-jax", + "evals": [ + { + "id": "jax-challenge-layout-v1", + "expected_skill": "cuequivariance-jax", + "prompt": "Our JAX model receives O3 features in an older storage format: the original irreps can contain separated copies of the same irrep, and each original block is flattened in ir_mul order. We need a differentiable adapter to dictionaries keyed by cuequivariance.Irrep, with blocks shaped (..., multiplicity, irrep_dimension), plus an inverse that recovers the ORIGINAL block and channel order. Write /workspace/solution.py exposing to_blocks(irreps, flat) and from_blocks(irreps, blocks). Equal irreps must concatenate channels in their order of appearance; opposite parities must remain distinct. Support arbitrary leading batch axes and these representations: '1o + 2x0e + 2x1o + 1e + 0o' and '2x1e + 0o + 1o + 1e + 2x0e + 1o'. Use the installed cuEquivariance JAX integration on CPU and float64. Write and run checks with distinct values, both unbatched data and batch shape (2,3), exact dictionary block values and inverse values, eager/JIT agreement, and gradients through the round trip. Also check that rotation and inversion commute with the conversion. Keep these functions importable without running a demo at import time. Report the actual executed checks.", + "expected_output": "An executed, importable JAX adapter that preserves data and original ordering despite duplicate O3 irreps, keeps opposite parities distinct, and passes block-value, inverse, JIT, gradient, rotation, and inversion checks.", + "assertions": [ + "The submitted functions use full irrep identities, preserve channel order among repeated irreps, and give correct block values; changing only irreps metadata or using a reshape-only conversion is insufficient.", + "The inverse reconstructs the original ungrouped flat buffer with all leading batch axes preserved, including an unbatched input.", + "Both conversion functions remain differentiable and usable with JIT for fixed irreps; the executed round-trip gradient agrees with a nonuniform cotangent.", + "The script actually runs exact value checks and nontrivial rotation and inversion checks on CPU float64, and the final claims match captured results." + ] + }, + { + "id": "jax-challenge-graph-v1", + "expected_skill": "cuequivariance-jax", + "prompt": "Implement a small trainable equivariant graph layer for a CPU JAX application, using the installed cuEquivariance integration. Write /workspace/solution.py exposing apply(params, features, vectors, senders, receivers, num_dst). Input node features have SO3 irreps '2x0 + 2x1' in mul_ir layout; output destination features have '0 + 1' in mul_ir layout. Edges carry nonzero displacement vectors. The layer computes normalized spherical harmonics of degrees 0,1,2; a channelwise tensor product of each source feature and its edge harmonics, keeping output types l=0,1; a sum of messages at each destination; and a bias-free equivariant linear projection to the requested output. Use cuEquivariance's default descriptor coefficients and emitted tensor-product channel order. Parameters are a dict of float64 arrays: radial_in has shape (1,4), radial_out has shape (4,n_tp), and projection has shape (n_linear,). Here n_tp is the channelwise weight count and n_linear is the projection weight count. For each edge, the weight vector is tanh(norm(vector) @ radial_in) @ radial_out, where norm(vector) is shaped (1,). There are no other learned weights or biases. Do not initialize parameters inside apply. Support varying source/destination counts and repeated receivers, including destinations with no incoming edges. Demonstrate five source nodes, four destinations, senders=[0,1,3,0,1,4,2], receivers=[2,0,2,1,2,0,1], and deterministic nonconstant features, vectors, and parameters. Run an independent NumPy edge-loop comparison, rotation and permutation checks, eager/JIT agreement, and gradients of sum(output**2) for every parameter and both floating inputs, checked against finite differences. Execute one SGD update with learning rate 0.001 and show that the output changes while retaining reference agreement. Keep apply importable without running a demo at import time; report observed results.", + "expected_output": "A CPU graph layer whose values match independent NumPy descriptors and edge accumulation, with correct layouts, zero isolated destinations, stable parameters, equivariance, JIT, nonzero checked gradients, and a verified optimization update.", + "assertions": [ + "The layer combines edge-dependent radial weights, normalized harmonics, the prescribed channelwise product and projection without losing descriptor output ordering or changing normalization.", + "Repeated receivers accumulate and isolated destinations remain zero; source relabeling and edge permutation preserve the corresponding output.", + "The saved implementation remains differentiable through features, edge vectors, and all three parameter arrays, and eager and compiled results agree with the independent reference.", + "The agent executes the numerical, rotation, gradient and update checks, preserves parameters across repeated calls, and supports the stated CPU environment without GPU dependencies." + ] + }, + { + "id": "jax-challenge-routing-v1", + "expected_skill": null, + "prompt": "Fix this loss in my ordinary JAX regression model: def mse_loss(predictions, targets): return ((predictions - targets) ** 2).mean(). Predictions have shape (N,1), targets have shape (N,), and each prediction should be compared only with its own target. Write /workspace/solution.py with the corrected, importable mse_loss function. Run a small example with distinct values and check its prediction gradient and JIT behavior. Preserve the input floating dtype. Keep the change limited to the regression loss and use the existing JAX installation.", + "expected_output": "A standard JAX loss with correct per-example alignment and gradient, without invoking cuEquivariance or its skill.", + "assertions": [ + "The scalar loss is the mean of N paired squared errors, and its prediction gradient has shape (N,1) with values 2*(prediction-target)/N.", + "The agent executes the loss, gradient and JIT checks, and does not import, install, or invoke cuEquivariance or load its skill." + ] + } + ] +} diff --git a/skills/cuequivariance-torch/evals/challenges.json b/skills/cuequivariance-torch/evals/challenges.json new file mode 100644 index 00000000..b94ab69a --- /dev/null +++ b/skills/cuequivariance-torch/evals/challenges.json @@ -0,0 +1,39 @@ +{ + "skill_name": "cuequivariance-torch", + "evals": [ + { + "id": "torch-challenge-layout-v1", + "expected_skill": "cuequivariance-torch", + "prompt": "Our PyTorch equivariant model receives O3 features in an older storage format: the original irreps can contain separated copies of the same irrep, and each original block is flattened in ir_mul order. We need a differentiable adapter to dictionaries keyed by cuequivariance.Irrep, with blocks shaped (..., multiplicity, irrep_dimension), plus an inverse that recovers the ORIGINAL block and channel order. Write /workspace/solution.py exposing to_blocks(irreps, flat) and from_blocks(irreps, blocks). Equal irreps must concatenate channels in their order of appearance; opposite parities must remain distinct. Support arbitrary leading batch axes and these representations: '1o + 2x0e + 2x1o + 1e + 0o' and '2x1e + 0o + 1o + 1e + 2x0e + 1o'. Use the installed cuEquivariance representation metadata and PyTorch on CPU with float64, including noncontiguous tensors. Write and run checks with distinct values, unbatched data and batch shape (2,3), exact dictionary block values and inverse values, and autograd through the round trip with a nonuniform cotangent. Also check that rotation and inversion commute with conversion. Keep these functions importable without running a demo at import time. Report the actual executed checks.", + "expected_output": "An executed, importable PyTorch adapter that preserves original data and ordering, combines only identical O3 irreps, supports noncontiguous tensors, and passes value, inverse, gradient, rotation and inversion checks.", + "assertions": [ + "The functions concatenate repeated full irrep identities without overwriting earlier blocks or merging opposite parities, and preserve the original per-irrep channel order.", + "The inverse recovers the original ungrouped buffer exactly for unbatched and batched inputs, including noncontiguous tensors; layout conversion handles multiplicity and representation axes correctly.", + "Tensor operations retain autograd, and the executed round-trip derivative matches a nonuniform cotangent.", + "The agent actually runs block-value, inverse, rotation and inversion comparisons on CPU float64 and reports observed results." + ] + }, + { + "id": "torch-challenge-graph-v1", + "expected_skill": "cuequivariance-torch", + "prompt": "Implement a small trainable equivariant graph layer for a CPU PyTorch application, using the installed cuEquivariance integration. Write /workspace/solution.py exposing GraphLayer, a torch.nn.Module with a no-argument constructor and forward(features, vectors, senders, receivers, num_dst). Input node features have SO3 irreps '2x0 + 2x1' in mul_ir layout; output destination features have '0 + 1' in mul_ir layout. Edges carry nonzero displacement vectors. The layer computes normalized spherical harmonics of degrees 0,1,2; a channelwise tensor product of each source feature and its edge harmonics, keeping output types l=0,1; a sum of messages at each destination; and a bias-free equivariant linear projection to the requested output. Use cuEquivariance's default descriptor coefficients and emitted tensor-product channel order. For reproducible parameter injection, expose exactly three trainable parameters as attributes: radial_in of shape (1,4), radial_out of shape (4,n_tp), and projection of shape (n_linear,). Here n_tp is the channelwise weight count and n_linear is the projection weight count. For each edge, its weight vector is tanh(norm(vector) @ radial_in) @ radial_out, with norm(vector) shaped (1,). There are no other learned weights or biases. Parameters must persist across forwards and be visible to a normal optimizer; support model.double(). Support varying source/destination counts and repeated receivers, including destinations with no incoming edges. Demonstrate five source nodes, four destinations, senders=[0,1,3,0,1,4,2], receivers=[2,0,2,1,2,0,1], and deterministic nonconstant float64 features, vectors and parameters. Run an independent NumPy edge-loop comparison, rotation and permutation checks, and gradients of sum(output**2) for every parameter and both floating inputs, checked against finite differences. Execute one torch.optim.SGD update with learning rate 0.001 and show that the output changes while retaining reference agreement. Keep GraphLayer importable without running a demo at import time; report observed results.", + "expected_output": "A CPU nn.Module with persistent registered parameters, correct edge-dependent tensor products and destination sums, passing independent values/equivariance/gradient checks and a verified optimizer update.", + "assertions": [ + "The module combines the prescribed radial weights, normalized harmonics, channelwise product and linear projection with consistent layouts, descriptor ordering and normalization.", + "Repeated receivers accumulate, isolated destinations remain zero, and source relabeling and edge permutation preserve the corresponding output.", + "All three parameters are registered once and reused; autograd reaches every parameter, node features and edge vectors with finite nonzero derivatives consistent with the independent reference.", + "The agent executes the reference, rotation, gradient and optimizer-step checks on CPU float64 without replacing the CPU installation or claiming CUDA execution." + ] + }, + { + "id": "torch-challenge-routing-v1", + "expected_skill": null, + "prompt": "Fix this loss in my ordinary PyTorch regression model: def mse_loss(predictions, targets): return ((predictions - targets) ** 2).mean(). Predictions have shape (N,1), targets have shape (N,), and each prediction should be compared only with its own target. Write /workspace/solution.py with the corrected, importable mse_loss function. Run a small example with distinct values and check its prediction gradient. Preserve the input floating dtype. Keep the change limited to the regression loss and use the existing PyTorch installation.", + "expected_output": "A standard PyTorch loss with correct per-example alignment and gradient, without invoking cuEquivariance or its skill.", + "assertions": [ + "The scalar loss is the mean of N paired squared errors, and its prediction gradient has shape (N,1) with values 2*(prediction-target)/N.", + "The agent executes the loss and gradient checks, and does not import, install, or invoke cuEquivariance or load its skill." + ] + } + ] +} From 38624bef92f8bdb5a9048770ce0a287d8ce6bffc Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 20 Oct 2026 10:12:54 -0700 Subject: [PATCH 14/19] Extend Torch Tier 3 setup timeout with native Harbor tasks Signed-off-by: Ohad Mosafi --- .github/scripts/prepare_skill_challenges.py | 26 +++- .github/scripts/prepare_skill_eval_tasks.py | 130 ++++++++++++++++++ .../tests/test_skill_challenge_packaging.py | 22 +++ .github/tests/test_skill_eval_setup.py | 69 +++++++++- skills/README.md | 38 +++-- skills/cuequivariance-torch/evals/config.yml | 55 +------- .../evals/harbor/dataset.toml | 23 ++++ .../harbor/torch-001/environment/Dockerfile | 3 + .../evals/harbor/torch-001/instruction.md | 1 + .../evals/harbor/torch-001/task.toml | 31 +++++ .../harbor/torch-002/environment/Dockerfile | 3 + .../evals/harbor/torch-002/instruction.md | 1 + .../evals/harbor/torch-002/task.toml | 31 +++++ .../harbor/torch-003/environment/Dockerfile | 3 + .../evals/harbor/torch-003/instruction.md | 1 + .../evals/harbor/torch-003/task.toml | 31 +++++ .../harbor/torch-004/environment/Dockerfile | 3 + .../evals/harbor/torch-004/instruction.md | 1 + .../evals/harbor/torch-004/task.toml | 31 +++++ .../harbor/torch-005/environment/Dockerfile | 3 + .../evals/harbor/torch-005/instruction.md | 1 + .../evals/harbor/torch-005/task.toml | 31 +++++ .../harbor/torch-006/environment/Dockerfile | 3 + .../evals/harbor/torch-006/instruction.md | 1 + .../evals/harbor/torch-006/task.toml | 31 +++++ .../cuequivariance-torch/evals/setup_cpu.sh | 52 +++++++ 26 files changed, 556 insertions(+), 69 deletions(-) create mode 100644 .github/scripts/prepare_skill_eval_tasks.py create mode 100644 skills/cuequivariance-torch/evals/harbor/dataset.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-001/task.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-002/task.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-003/task.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-004/task.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-005/task.toml create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md create mode 100644 skills/cuequivariance-torch/evals/harbor/torch-006/task.toml create mode 100644 skills/cuequivariance-torch/evals/setup_cpu.sh diff --git a/.github/scripts/prepare_skill_challenges.py b/.github/scripts/prepare_skill_challenges.py index 8324acb3..b583fe77 100644 --- a/.github/scripts/prepare_skill_challenges.py +++ b/.github/scripts/prepare_skill_challenges.py @@ -12,6 +12,7 @@ import hashlib import inspect import json +import runpy import shlex import shutil import tempfile @@ -54,7 +55,10 @@ def prepare(destination, backends, *, export_harbor=False): destination = Path(destination).resolve() destination.mkdir(parents=True, exist_ok=False) if export_harbor: - from skillevaluator.tier3.harbor.adapter import generate_harbor_tasks + from skillevaluator.tier3.harbor.adapter import ( + generate_harbor_tasks, + stage_native_harbor_tasks, + ) if ( "evaluator_skill_path" @@ -90,6 +94,14 @@ def prepare(destination, backends, *, export_harbor=False): config = yaml.safe_load((source / "evals/config.yml").read_text()) config["grading"]["mode"] = "aces_plus_custom" (evals / "config.yml").write_text(yaml.safe_dump(config, sort_keys=False)) + native = config["harbor"].get("task_source") == "native_harbor" + if native: + shutil.copy2(source / "evals/setup_cpu.sh", evals / "setup_cpu.sh") + # Render the challenge IDs/prompts, never copy regression tasks. + builder = runpy.run_path( + str(ROOT / ".github/scripts/prepare_skill_eval_tasks.py") + ) + builder["write_tasks"](staged) command = [ "skillevaluator", "tier3", @@ -125,11 +137,19 @@ def prepare(destination, backends, *, export_harbor=False): ], "command": command, } + if native: + record["setup_digest"] = digest(evals / "setup_cpu.sh") + record["native_task_digests"] = { + str(p.relative_to(evals / "harbor")): digest(p) + for p in sorted((evals / "harbor").rglob("*")) + if p.is_file() + } if export_harbor: tasks_by_arm = {} + emit_tasks = stage_native_harbor_tasks if native else generate_harbor_tasks for arm, with_skill in (("with_skill", True), ("baseline", False)): task_root = destination / "harbor" / backend / arm - tasks = generate_harbor_tasks( + tasks = emit_tasks( staged, task_root, evaluator_skill_path=staged, @@ -138,7 +158,7 @@ def prepare(destination, backends, *, export_harbor=False): grading_mode="aces_plus_custom", env_mode="astra-sandbox", runtime_env=config["harbor"]["runtime_env"], - pre_agent_setup=config["harbor"]["pre_agent_setup"], + pre_agent_setup=config["harbor"].get("pre_agent_setup"), agent_workdir="/workspace", copy_repo=False, repo_context_exclude_paths=[ROOT / ".github/evals"], diff --git a/.github/scripts/prepare_skill_eval_tasks.py b/.github/scripts/prepare_skill_eval_tasks.py new file mode 100644 index 00000000..cf1d2d0a --- /dev/null +++ b/.github/scripts/prepare_skill_eval_tasks.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Render repo-owned Harbor tasks from evals.json and setup_cpu.sh. + +SkillEvaluator 1.5.6 preserves native task healthcheck budgets. Its generated +pre_agent_setup checks instead impose a fixed 120-second installation timeout. +Only the environment setup budget differs here; ACES still supplies the grader. +""" + +import argparse +import json +import re +import shlex +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SETUP_TIMEOUT_SECONDS = 600 + + +def task_files(skill_path): + """Return reproducible native task files, without answers or grader code.""" + skill_path = Path(skill_path) + evals = skill_path / "evals" + entries = json.loads((evals / "evals.json").read_text())["evals"] + setup = (evals / "setup_cpu.sh").read_text() + # Keep stdout/stderr in a task artifact, including when installation fails. + # pipefail preserves the setup's exit status through tee. + command = "bash -c " + shlex.quote( + "set -euo pipefail\n" + f"bash -lc {shlex.quote(setup)} 2>&1 " + '| tee "${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log"' + ) + files = {} + seen = set() + for entry in entries: + case_id = entry["id"] + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]*", case_id): + raise ValueError(f"Invalid case ID: {case_id!r}") + if case_id in seen: + raise ValueError(f"Duplicate case ID: {case_id}") + seen.add(case_id) + case = Path(case_id) + files[case / "instruction.md"] = entry["prompt"] + "\n" + files[case / "environment/Dockerfile"] = ( + "# The evaluator rebases this image for its selected agent runtime.\n" + "FROM python:3.11-slim\n" + "WORKDIR /workspace\n" + ) + files[ + case / "task.toml" + ] = f"""# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-{case_id}" +description = "Skill evaluation task for {entry.get("expected_skill") or "none"}" + +[metadata] +entry_id = "{case_id}" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = {json.dumps(command)} +interval_sec = 5.0 +timeout_sec = {float(SETUP_TIMEOUT_SECONDS)} +retries = 1 +""" + files[Path("dataset.toml")] = ( + "# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit.\n" + '[dataset]\nname = "nvidia/skillevaluator"\n' + 'description = "SkillEvaluator skill evaluation dataset"\n\n' + + "".join( + f'[[tasks]]\nname = "nvidia/{case_id}"\n\n' for case_id in sorted(seen) + ) + ) + return files + + +def write_tasks(skill_path, *, check=False): + """Refresh known task files; refuse stale cases rather than deleting them.""" + skill_path = Path(skill_path) + target = skill_path / "evals/harbor" + expected = task_files(skill_path) + expected_cases = {p.parts[0] for p in expected if p.name == "task.toml"} + existing_cases = {p.parent.name for p in target.glob("*/task.toml")} + if stale := existing_cases - expected_cases: + raise ValueError(f"Remove stale native cases explicitly: {sorted(stale)}") + differing = [] + for relative, content in expected.items(): + path = target / relative + if path.is_file() and path.read_text() == content: + continue + differing.append(str(relative)) + if not check: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + if check and differing: + raise ValueError(f"Regenerate native eval tasks: {', '.join(differing)}") + return target + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "skill", + type=Path, + nargs="?", + default=ROOT / "skills/cuequivariance-torch", + ) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + print(write_tasks(args.skill, check=args.check)) + + +if __name__ == "__main__": + main() diff --git a/.github/tests/test_skill_challenge_packaging.py b/.github/tests/test_skill_challenge_packaging.py index 5ac3b788..a8b753fd 100644 --- a/.github/tests/test_skill_challenge_packaging.py +++ b/.github/tests/test_skill_challenge_packaging.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +import tomllib import yaml from challenge_checks import EVALS, grader @@ -48,6 +49,27 @@ def test_pilot_keeps_skill_and_environment_fixed_and_excludes_answers(tmp_path): assert "--skip-baseline" not in record["command"] for agent, model in manifest["agent_models"].items(): assert f"{agent}={model}" in record["command"] + if backend == "torch": + assert (staged / "evals/setup_cpu.sh").read_bytes() == ( + source / "evals/setup_cpu.sh" + ).read_bytes() + tasks = staged / "evals/harbor" + assert {p.name for p in tasks.iterdir() if p.is_dir()} == { + e["id"] for e in entries + } + dataset = tomllib.loads((tasks / "dataset.toml").read_text()) + assert {t["name"] for t in dataset["tasks"]} == { + f"nvidia/{e['id']}" for e in entries + } + for entry in entries: + task = tasks / entry["id"] + assert (task / "instruction.md").read_text() == entry["prompt"] + "\n" + data = tomllib.loads((task / "task.toml").read_text()) + assert data["environment"]["healthcheck"]["timeout_sec"] == 600 + assert record["setup_digest"] == preparer.digest( + staged / "evals/setup_cpu.sh" + ) + assert record["native_task_digests"] for path, content in original.items(): assert path.read_bytes() == content with pytest.raises(FileExistsError): diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py index f4eac971..4abe4e3c 100644 --- a/.github/tests/test_skill_eval_setup.py +++ b/.github/tests/test_skill_eval_setup.py @@ -3,23 +3,27 @@ import json import os +import runpy +import shlex import subprocess import sys from pathlib import Path import pytest +import tomllib import yaml +ROOT = Path(__file__).resolve().parents[2] +BUILDER = runpy.run_path(str(ROOT / ".github/scripts/prepare_skill_eval_tasks.py")) + def setup_command(frontend): config_path = ( - Path(__file__).resolve().parents[2] - / "skills" - / f"cuequivariance-{frontend}" - / "evals" - / "config.yml" + ROOT / "skills" / f"cuequivariance-{frontend}" / "evals" / "config.yml" ) config = yaml.safe_load(config_path.read_text()) + if config["harbor"].get("task_source") == "native_harbor": + return (config_path.parent / "setup_cpu.sh").read_text() return "set -euo pipefail\n" + "\n".join(config["harbor"]["pre_agent_setup"]) @@ -160,3 +164,58 @@ def test_pre_agent_setup_preserves_existing_incompatible_environment( assert sentinel.read_text() == "existing environment" assert (env_dir / "pyvenv.cfg").read_bytes() == original_config assert not list(tmp_path.glob(".cueq-env.*")) + + +def test_native_torch_tasks_keep_regression_prompts_and_setup_budget(): + skill = ROOT / "skills/cuequivariance-torch" + BUILDER["write_tasks"](skill, check=True) + config = yaml.safe_load((skill / "evals/config.yml").read_text()) + assert config["harbor"]["task_source"] == "native_harbor" + # Otherwise 1.5.6 rejects the existing native healthcheck as a collision. + assert "pre_agent_setup" not in config["harbor"] + assert config["grading"]["mode"] == "aces_default" + entries = json.loads((skill / "evals/evals.json").read_text())["evals"] + tasks = skill / "evals/harbor" + assert {p.name for p in tasks.iterdir() if p.is_dir()} == {e["id"] for e in entries} + manifest = tomllib.loads((tasks / "dataset.toml").read_text()) + assert [t["name"] for t in manifest["tasks"]] == [ + f"nvidia/{case_id}" for case_id in sorted(e["id"] for e in entries) + ] + healthchecks = [] + for entry in entries: + task = tasks / entry["id"] + assert (task / "instruction.md").read_text() == entry["prompt"] + "\n" + data = tomllib.loads((task / "task.toml").read_text()) + assert data["metadata"]["entry_id"] == entry["id"] + assert data["agent"]["timeout_sec"] == 300 + assert data["verifier"]["timeout_sec"] == 180 + healthchecks.append(data["environment"]["healthcheck"]) + assert healthchecks[-1]["timeout_sec"] == 600 + assert healthchecks[-1]["retries"] == 1 + assert all(h == healthchecks[0] for h in healthchecks) + + +@pytest.mark.parametrize("status", [0, 23]) +def test_native_setup_retains_output_and_exit_status(tmp_path, status): + evals = tmp_path / "evals" + evals.mkdir() + (evals / "evals.json").write_text( + json.dumps({"evals": [{"id": "smoke", "prompt": "Exercise setup"}]}) + ) + (evals / "setup_cpu.sh").write_text( + f"printf 'installer output\\n'\nprintf 'installer diagnostic\\n' >&2\nexit {status}\n" + ) + files = BUILDER["task_files"](tmp_path) + data = tomllib.loads(files[Path("smoke/task.toml")]) + command = data["environment"]["healthcheck"]["command"] + result = subprocess.run( + shlex.split(command), + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == status + assert "installer output" in result.stdout + assert "installer diagnostic" in result.stdout + assert (tmp_path / "cueq-setup.log").read_text() == result.stdout diff --git a/skills/README.md b/skills/README.md index a29d9a10..cea819d8 100644 --- a/skills/README.md +++ b/skills/README.md @@ -47,10 +47,10 @@ environment. In SkillEvaluator 1.5.6, Docker task generation installs `scripts/requirements.txt`. The `k8s-sandbox` path instead stages skill files into an existing sandbox; `base_image_mode: rebuild` alone does **not** install these dependencies there. Verify the actual interpreter and package versions in each -arm before treating a run as an evaluation of the pinned environment. A shared -custom evaluation image or common `harbor.pre_agent_setup` is needed when the -sandbox does not already provide them. Provision it equally for both arms and -establish a new baseline when changing the environment. +arm before treating a run as an evaluation of the pinned environment. Provision +dependencies equally for both arms and establish a new baseline when changing +the environment. JAX uses `harbor.pre_agent_setup`; Torch uses the native Harbor +environment healthcheck described below. The JAX and PyTorch setup commands publish `/workspace/.venv` only after the pinned packages pass a CPU smoke check. Failed environments remain at their @@ -62,10 +62,29 @@ install resolves its dependencies and cuEquivariance from the normal index, avoiding dependency-metadata incompatibilities with older host pip versions. Both arms use the same setup and package pins. -These are repository-side setup improvements, not a change to SkillEvaluator's -healthcheck timeout. A successful local setup does not establish that the shared -Astra sandbox will finish within its budget. Inspect the live preflight result -before interpreting a missing Torch score as a skill-quality failure. +SkillEvaluator 1.5.6 hardcodes 120 seconds for generated `pre_agent_setup` +healthchecks. Torch selects `harbor.task_source: native_harbor` so its repo-owned +`evals/harbor/*/task.toml` files can give installation and the smoke check +600 seconds. This uses the unmodified evaluator's supported native-task path. +The agent and verifier budgets remain 300 and 180 seconds, respectively, and +grading remains `aces_default`. Both arms still run the same six prompts from +`evals/evals.json`. Setup failures remain failures, and Harbor collects +`/workspace/cueq-setup.log` as a task artifact for diagnosis. + +`evals/setup_cpu.sh` is the canonical Torch bootstrap script. Its contents and +the prompts are embedded in the native tasks so setup also runs in the baseline, +where the skill itself is absent. After changing the script or dataset, refresh +and check the generated files: + +```bash +python .github/scripts/prepare_skill_eval_tasks.py +python .github/scripts/prepare_skill_eval_tasks.py --check +``` + +The setup regression tests check synchronization and that capturing logs does +not hide a failing exit status. A successful local setup does not establish that +the shared Astra sandbox is healthy; inspect the live preflight result before +interpreting a missing Torch score as a skill-quality failure. The requirements pin the cuEquivariance release or source revision whose APIs were checked, and pin the JAX/Flax or CPU PyTorch versions. @@ -161,6 +180,9 @@ environment setup. Only the selected dataset and grading mode change: metrics and quality gate. **`numerical_success` does not replace ACES's overall score or change its gate.** Read both the numerical result and the normal trajectory assessment; neither should be inferred from the other. +The Torch pilot renders native tasks from its challenge IDs and prompts and +uses the same 600-second setup budget. Its manifest also records the bootstrap +script and native task file digests. The export validates both conditions, identical prompts/verifiers, and the absence of eval metadata, grader code and reference solutions in the agent's diff --git a/skills/cuequivariance-torch/evals/config.yml b/skills/cuequivariance-torch/evals/config.yml index d1861b5e..dceb2680 100644 --- a/skills/cuequivariance-torch/evals/config.yml +++ b/skills/cuequivariance-torch/evals/config.yml @@ -1,63 +1,12 @@ # Prepare identical CPU dependencies for both arms before agent execution. schema_version: 1 harbor: + # Repo-owned tasks give dependency installation its own 10-minute budget. + task_source: native_harbor base_image_mode: rebuild runtime_env: VIRTUAL_ENV: /workspace/.venv PATH: /workspace/.venv/bin:/tmp/agent-home/.local/bin:/home/agent/.local/bin:/usr/local/share/npm-global/bin:/opt/openhands-venv/bin:/usr/local/bin:/usr/bin:/bin - pre_agent_setup: - - | - set -eu - cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" - cueq_env="$cueq_root/.venv" - cueq_publish=false - if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then - cueq_host_python=$(command -v python3) - cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") - cueq_publish=true - "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" - # Reduce setup time: wheels only, - # no bytecode compilation, and no PyPI lookup for the CPU Torch build. - printf 'Installing CPU PyTorch into %s\n' "$cueq_env" - "$cueq_host_python" -m pip --python "$cueq_env" install \ - --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ - --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ - --progress-bar off --no-deps 'torch==2.14.1+cpu' --index-url https://download.pytorch.org/whl/cpu - # Resolve Torch's dependencies on the normal index. Older host pip - # rejects some dependency metadata on the PyTorch index. - printf 'Installing Torch dependencies and cuEquivariance into %s\n' "$cueq_env" - "$cueq_host_python" -m pip --python "$cueq_env" install \ - --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ - --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ - --progress-bar off \ - 'torch==2.14.1+cpu' 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' - fi - printf 'Checking CPU environment: %s\n' "$cueq_env" - "$cueq_env/bin/python" -u - <<'PY' - import importlib.metadata as metadata - import sys - import torch - import cuequivariance_torch as cuet - - for package, expected in { - "cuequivariance": "0.13.0", "cuequivariance-torch": "0.13.0", - "torch": "2.14.1+cpu", - }.items(): - assert metadata.version(package) == expected, (package, metadata.version(package)) - assert torch.version.cuda is None, torch.__version__ - layer = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") - x = torch.ones((1, 3), dtype=torch.float64, requires_grad=True) - y = layer(x) - assert y.shape == (1, 4) and bool(torch.isfinite(y).all()) - y.square().sum().backward() - assert x.grad is not None and bool(torch.isfinite(x.grad).all()) - print("CPU environment ready:", sys.executable, sys.version.split()[0], torch.__version__) - PY - # A failed install or smoke check must not leave .venv pointing at a - # partial environment. Preserve it for diagnosis and retry in a fresh path. - if [ "$cueq_publish" = true ]; then - ln -sT "$cueq_env" "$cueq_root/.venv" - fi skill_workspace: mode: isolated grading: diff --git a/skills/cuequivariance-torch/evals/harbor/dataset.toml b/skills/cuequivariance-torch/evals/harbor/dataset.toml new file mode 100644 index 00000000..c7533a3c --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/dataset.toml @@ -0,0 +1,23 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +[dataset] +name = "nvidia/skillevaluator" +description = "SkillEvaluator skill evaluation dataset" + +[[tasks]] +name = "nvidia/torch-001" + +[[tasks]] +name = "nvidia/torch-002" + +[[tasks]] +name = "nvidia/torch-003" + +[[tasks]] +name = "nvidia/torch-004" + +[[tasks]] +name = "nvidia/torch-005" + +[[tasks]] +name = "nvidia/torch-006" + diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md new file mode 100644 index 00000000..66b5e4f8 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/instruction.md @@ -0,0 +1 @@ +On CPU, use cuequivariance_torch.SphericalHarmonics for l=0,1,2 with normalize=False. Write and run a deterministic float64 example comparing it to the core NumPy descriptor, then run torch.autograd.gradcheck on nonzero input vectors. Do not install CUDA kernels. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml new file mode 100644 index 00000000..0ec4e249 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-001" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-001" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md new file mode 100644 index 00000000..83e5ee11 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/instruction.md @@ -0,0 +1 @@ +Write and run a CPU float64 example for cuequivariance_torch.Linear mapping SO3 '2x0 + 3x1' to '3x0 + 2x1'. Compare mul_ir and ir_mul layouts using identical explicit weights and nonconstant features. Verify equivalence after correct per-block transposes, then check a nonzero rotation and gradients with respect to features and weights. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml new file mode 100644 index 00000000..7db25aab --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-002" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-002" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md new file mode 100644 index 00000000..00433b64 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/instruction.md @@ -0,0 +1 @@ +Use cuequivariance_torch.ChannelWiseTensorProduct on CPU for SO3 '2x0 + 2x1' and '0 + 1', filtered to l=0,1. I need different externally supplied weights for each batch item. Write and run a float64 example against the equivalent dense NumPy descriptor and demonstrate gradients through both features and weights. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml new file mode 100644 index 00000000..f8614568 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-003" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-003" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md new file mode 100644 index 00000000..6c51859a --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/instruction.md @@ -0,0 +1 @@ +On CPU, use cuequivariance_torch.SegmentedPolynomial for a linear descriptor from SO3 '2x0' to '3x0'. With four source nodes and five edges, gather features using senders=[0,1,0,3,1] and scatter outputs to receivers=[2,0,2,1,2] into four destination nodes. Write and run a float64 example, compare against explicit gather plus index_add_, and compare feature and weight gradients. A destination with no edges must stay zero. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml new file mode 100644 index 00000000..aad45293 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-004" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-004" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md new file mode 100644 index 00000000..4bd5943a --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/instruction.md @@ -0,0 +1 @@ +I copied a cuEquivariance PyTorch example using FullyConnectedTensorProduct and SphericalHarmonics onto a CPU-only laptop and it tries to load CUDA kernels. Explain the backend choice and write and run a small supported CPU tensor-product example. Keep my CPU PyTorch installation. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml new file mode 100644 index 00000000..80a8c11b --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-005" +description = "Skill evaluation task for cuequivariance-torch" + +[metadata] +entry_id = "torch-005" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile b/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile new file mode 100644 index 00000000..3def1061 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/environment/Dockerfile @@ -0,0 +1,3 @@ +# The evaluator rebases this image for its selected agent runtime. +FROM python:3.11-slim +WORKDIR /workspace diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md b/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md new file mode 100644 index 00000000..db13cce7 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/instruction.md @@ -0,0 +1 @@ +Explain the difference between git merge and git rebase for a feature branch. I am not asking about numerical code. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml new file mode 100644 index 00000000..edccac94 --- /dev/null +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml @@ -0,0 +1,31 @@ +# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. +schema_version = "1.3" +artifacts = ["/workspace/cueq-setup.log"] + +[task] +name = "nvidia/skillevaluator-torch-006" +description = "Skill evaluation task for none" + +[metadata] +entry_id = "torch-006" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 180.0 + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 2048 +workdir = "/workspace" +network_mode = "public" +skills_dir = "/workspace/skills" +build_timeout_sec = 2400.0 + +[environment.healthcheck] +command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +interval_sec = 5.0 +timeout_sec = 600.0 +retries = 1 diff --git a/skills/cuequivariance-torch/evals/setup_cpu.sh b/skills/cuequivariance-torch/evals/setup_cpu.sh new file mode 100644 index 00000000..95d150d7 --- /dev/null +++ b/skills/cuequivariance-torch/evals/setup_cpu.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail +cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" +cueq_env="$cueq_root/.venv" +cueq_publish=false +if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_host_python=$(command -v python3) + cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") + cueq_publish=true + "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" + # Reduce setup time: wheels only, + # no bytecode compilation, and no PyPI lookup for the CPU Torch build. + printf 'Installing CPU PyTorch into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off --no-deps 'torch==2.14.1+cpu' --index-url https://download.pytorch.org/whl/cpu + # Resolve Torch's dependencies on the normal index. Older host pip + # rejects some dependency metadata on the PyTorch index. + printf 'Installing Torch dependencies and cuEquivariance into %s\n' "$cueq_env" + "$cueq_host_python" -m pip --python "$cueq_env" install \ + --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ + --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ + --progress-bar off \ + 'torch==2.14.1+cpu' 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' +fi +printf 'Checking CPU environment: %s\n' "$cueq_env" +"$cueq_env/bin/python" -u - <<'PY' +import importlib.metadata as metadata +import sys +import torch +import cuequivariance_torch as cuet + +for package, expected in { + "cuequivariance": "0.13.0", "cuequivariance-torch": "0.13.0", + "torch": "2.14.1+cpu", +}.items(): + assert metadata.version(package) == expected, (package, metadata.version(package)) +assert torch.version.cuda is None, torch.__version__ +layer = cuet.SphericalHarmonics([0, 1], normalize=False, method="naive") +x = torch.ones((1, 3), dtype=torch.float64, requires_grad=True) +y = layer(x) +assert y.shape == (1, 4) and bool(torch.isfinite(y).all()) +y.square().sum().backward() +assert x.grad is not None and bool(torch.isfinite(x.grad).all()) +print("CPU environment ready:", sys.executable, sys.version.split()[0], torch.__version__) +PY +# A failed install or smoke check must not leave .venv pointing at a +# partial environment. Preserve it for diagnosis and retry in a fresh path. +if [ "$cueq_publish" = true ]; then + ln -sT "$cueq_env" "$cueq_root/.venv" +fi From d6ecb8f83b89c69cc4dbe00c119bfbda38451470 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 20 Oct 2026 12:49:44 -0700 Subject: [PATCH 15/19] Surface Torch Tier 3 setup failures in CI logs Signed-off-by: Ohad Mosafi --- .github/scripts/prepare_skill_eval_tasks.py | 52 +++++++- .github/tests/test_skill_eval_setup.py | 111 +++++++++++++++++- .../evals/harbor/torch-001/task.toml | 8 +- .../evals/harbor/torch-002/task.toml | 8 +- .../evals/harbor/torch-003/task.toml | 8 +- .../evals/harbor/torch-004/task.toml | 8 +- .../evals/harbor/torch-005/task.toml | 8 +- .../evals/harbor/torch-006/task.toml | 8 +- .../cuequivariance-torch/evals/setup_cpu.sh | 18 +++ 9 files changed, 207 insertions(+), 22 deletions(-) diff --git a/.github/scripts/prepare_skill_eval_tasks.py b/.github/scripts/prepare_skill_eval_tasks.py index cf1d2d0a..dc45a517 100644 --- a/.github/scripts/prepare_skill_eval_tasks.py +++ b/.github/scripts/prepare_skill_eval_tasks.py @@ -3,7 +3,8 @@ SkillEvaluator 1.5.6 preserves native task healthcheck budgets. Its generated pre_agent_setup checks instead impose a fixed 120-second installation timeout. -Only the environment setup budget differs here; ACES still supplies the grader. +Native tasks also retain setup diagnostics and expose failures in Harbor's job +log, which SkillEvaluator includes in its Tier 3 report. ACES supplies the grader. """ import argparse @@ -15,6 +16,35 @@ ROOT = Path(__file__).resolve().parents[2] SETUP_TIMEOUT_SECONDS = 600 +# Harbor 0.20 omits healthcheck stdout/stderr from HealthcheckError, but records +# failed collection hooks' output in job.log, even when preflight fails before +# an agent starts. Replay the real setup status here; this hook never grades. +# Keep the excerpt short enough for SkillEvaluator's preflight log tail. +SETUP_DIAGNOSTICS_COMMAND = "bash -c " + shlex.quote( + """set -euo pipefail +cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" +cueq_status=125 +if [ -r "$cueq_root/cueq-setup.status" ]; then + read -r cueq_status < "$cueq_root/cueq-setup.status" || cueq_status=125 +fi +if [[ "$cueq_status" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then + cueq_status=$((10#$cueq_status)) +else + cueq_status=125 +fi +if [ "$cueq_status" -eq 0 ]; then + exit 0 +fi +printf 'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n' "$cueq_status" >&2 +if [ -f "$cueq_root/cueq-setup.log" ] && [ -r "$cueq_root/cueq-setup.log" ]; then + tail -n 12 "$cueq_root/cueq-setup.log" >&2 || printf 'Could not read the setup log.\\n' >&2 +else + printf 'No setup log was produced; setup may have failed before logging started.\\n' >&2 +fi +exit "$cueq_status" +""" +) + def task_files(skill_path): """Return reproducible native task files, without answers or grader code.""" @@ -23,11 +53,21 @@ def task_files(skill_path): entries = json.loads((evals / "evals.json").read_text())["evals"] setup = (evals / "setup_cpu.sh").read_text() # Keep stdout/stderr in a task artifact, including when installation fails. - # pipefail preserves the setup's exit status through tee. + # Reset any previous status before starting. Save the setup exit code even + # through tee; a logging failure must also fail an otherwise successful run. command = "bash -c " + shlex.quote( "set -euo pipefail\n" + 'cueq_root="${CUEQ_EVAL_ROOT:-/workspace}"\n' + 'printf "125\\n" > "$cueq_root/cueq-setup.status"\n' + "set +e\n" f"bash -lc {shlex.quote(setup)} 2>&1 " - '| tee "${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log"' + '| tee "$cueq_root/cueq-setup.log"\n' + 'cueq_codes=("${PIPESTATUS[@]}")\n' + "set -e\n" + 'cueq_status="${cueq_codes[0]}"\n' + 'if [ "$cueq_status" -eq 0 ]; then cueq_status="${cueq_codes[1]}"; fi\n' + 'printf "%s\\n" "$cueq_status" > "$cueq_root/cueq-setup.status"\n' + 'exit "$cueq_status"\n' ) files = {} seen = set() @@ -49,7 +89,7 @@ def task_files(skill_path): case / "task.toml" ] = f"""# Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-{case_id}" @@ -64,6 +104,10 @@ def task_files(skill_path): [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = {json.dumps(SETUP_DIAGNOSTICS_COMMAND)} +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py index 4abe4e3c..32c6679b 100644 --- a/.github/tests/test_skill_eval_setup.py +++ b/.github/tests/test_skill_eval_setup.py @@ -58,8 +58,11 @@ def setup_sandbox(tmp_path): log.write(json.dumps({ "stage": stage, "args": args, "published": (root / ".venv").is_symlink() }) + "\\n") -if os.environ.get("CUEQ_FAIL_STAGE") == stage: - print("injected " + stage + " failure", file=sys.stderr) +failure = os.environ.get("CUEQ_FAIL_STAGE") +if failure == stage or ( + failure == "dependencies" and stage == "install" and "--no-deps" not in args +): + print("injected " + failure + " failure", file=sys.stderr) sys.exit(23) if stage == "smoke": print("CPU environment ready") @@ -166,6 +169,33 @@ def test_pre_agent_setup_preserves_existing_incompatible_environment( assert not list(tmp_path.glob(".cueq-env.*")) +@pytest.mark.parametrize( + ("failure", "reported_stage"), + [ + ("venv", "create_environment"), + ("install", "install_cpu_torch"), + ("dependencies", "install_dependencies"), + ("smoke", "smoke_check"), + ], +) +def test_torch_setup_identifies_the_failed_stage( + tmp_path, setup_sandbox, failure, reported_stage +): + result = subprocess.run( + ["bash", "-c", setup_command("torch")], + env={**setup_sandbox, "CUEQ_FAIL_STAGE": failure}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 23 + assert f"injected {failure} failure" in result.stderr + assert result.stderr.rstrip().endswith( + f"CUEQ_SETUP_FAILED stage={reported_stage} exit_code=23" + ) + assert not os.path.lexists(tmp_path / ".venv") + + def test_native_torch_tasks_keep_regression_prompts_and_setup_budget(): skill = ROOT / "skills/cuequivariance-torch" BUILDER["write_tasks"](skill, check=True) @@ -196,7 +226,8 @@ def test_native_torch_tasks_keep_regression_prompts_and_setup_budget(): @pytest.mark.parametrize("status", [0, 23]) -def test_native_setup_retains_output_and_exit_status(tmp_path, status): +@pytest.mark.parametrize("logging_fails", [False, True]) +def test_native_setup_retains_output_and_exit_status(tmp_path, status, logging_fails): evals = tmp_path / "evals" evals.mkdir() (evals / "evals.json").write_text( @@ -208,14 +239,82 @@ def test_native_setup_retains_output_and_exit_status(tmp_path, status): files = BUILDER["task_files"](tmp_path) data = tomllib.loads(files[Path("smoke/task.toml")]) command = data["environment"]["healthcheck"]["command"] + # A previous successful attempt must not hide the next attempt's failure. + status_file = tmp_path / "cueq-setup.status" + status_file.write_text("0\n") + log_file = tmp_path / "cueq-setup.log" + if logging_fails: + log_file.mkdir() + env = {**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)} result = subprocess.run( shlex.split(command), - env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + env=env, capture_output=True, text=True, timeout=10, ) - assert result.returncode == status + expected_status = status or (1 if logging_fails else 0) + assert result.returncode == expected_status assert "installer output" in result.stdout assert "installer diagnostic" in result.stdout - assert (tmp_path / "cueq-setup.log").read_text() == result.stdout + if not logging_fails: + assert log_file.read_text() == result.stdout + assert status_file.read_text() == f"{expected_status}\n" + + (hook,) = data["verifier"]["collect"] + diagnostics = subprocess.run( + shlex.split(hook["command"]), + env=env, + capture_output=True, + text=True, + timeout=10, + ) + # Harbor logs the stdout/stderr of a nonzero collection hook even when the + # healthcheck failed before agent execution. Success produces no warning. + assert diagnostics.returncode == expected_status + if expected_status: + assert ( + f"CUEQ_SETUP_DIAGNOSTICS exit_code={expected_status}" in diagnostics.stderr + ) + if logging_fails: + assert "No setup log was produced" in diagnostics.stderr + else: + assert "installer diagnostic" in diagnostics.stderr + else: + assert not diagnostics.stdout and not diagnostics.stderr + + +@pytest.mark.parametrize("saved_status", [None, "125\n", "invalid\n"]) +def test_setup_diagnostics_handle_missing_or_incomplete_status(tmp_path, saved_status): + if saved_status is not None: + (tmp_path / "cueq-setup.status").write_text(saved_status) + result = subprocess.run( + shlex.split(BUILDER["SETUP_DIAGNOSTICS_COMMAND"]), + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 125 + assert "CUEQ_SETUP_DIAGNOSTICS exit_code=125" in result.stderr + assert "No setup log was produced" in result.stderr + + +def test_setup_diagnostics_keep_the_error_at_the_end_of_long_logs(tmp_path): + (tmp_path / "cueq-setup.status").write_text("23\n") + (tmp_path / "cueq-setup.log").write_text( + "irrelevant download progress\n" * 1000 + + "ERROR: injected dependency failure\n" + + "CUEQ_SETUP_FAILED stage=install_dependencies exit_code=23\n" + ) + result = subprocess.run( + shlex.split(BUILDER["SETUP_DIAGNOSTICS_COMMAND"]), + env={**os.environ, "CUEQ_EVAL_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 23 + assert "ERROR: injected dependency failure" in result.stderr + assert "CUEQ_SETUP_FAILED stage=install_dependencies exit_code=23" in result.stderr + assert len(result.stderr.splitlines()) <= 13 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml index 0ec4e249..9ef0a1b8 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-001" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml index 7db25aab..480ad9f2 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-002" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml index f8614568..df5b1aa8 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-003" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml index aad45293..adcea012 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-004" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml index 80a8c11b..d679a2bd 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-005" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml index edccac94..c22bafe7 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml @@ -1,6 +1,6 @@ # Generated by .github/scripts/prepare_skill_eval_tasks.py; do not edit. schema_version = "1.3" -artifacts = ["/workspace/cueq-setup.log"] +artifacts = ["/workspace/cueq-setup.log", "/workspace/cueq-setup.status"] [task] name = "nvidia/skillevaluator-torch-006" @@ -15,6 +15,10 @@ timeout_sec = 300.0 [verifier] timeout_sec = 180.0 +[[verifier.collect]] +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_status=125\nif [ -r \"$cueq_root/cueq-setup.status\" ]; then\n read -r cueq_status < \"$cueq_root/cueq-setup.status\" || cueq_status=125\nfi\nif [[ \"$cueq_status\" =~ ^[0-9]{1,3}$ ]] && (( 10#$cueq_status <= 255 )); then\n cueq_status=$((10#$cueq_status))\nelse\n cueq_status=125\nfi\nif [ \"$cueq_status\" -eq 0 ]; then\n exit 0\nfi\nprintf '\"'\"'CUEQ_SETUP_DIAGNOSTICS exit_code=%s (125 means incomplete if no final status was recorded)\\n'\"'\"' \"$cueq_status\" >&2\nif [ -f \"$cueq_root/cueq-setup.log\" ] && [ -r \"$cueq_root/cueq-setup.log\" ]; then\n tail -n 12 \"$cueq_root/cueq-setup.log\" >&2 || printf '\"'\"'Could not read the setup log.\\n'\"'\"' >&2\nelse\n printf '\"'\"'No setup log was produced; setup may have failed before logging started.\\n'\"'\"' >&2\nfi\nexit \"$cueq_status\"\n'" +timeout_sec = 10.0 + [environment] cpus = 2 memory_mb = 4096 @@ -25,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_host_python=$(command -v python3)\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"${CUEQ_EVAL_ROOT:-/workspace}/cueq-setup.log\"'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/setup_cpu.sh b/skills/cuequivariance-torch/evals/setup_cpu.sh index 95d150d7..f4102276 100644 --- a/skills/cuequivariance-torch/evals/setup_cpu.sh +++ b/skills/cuequivariance-torch/evals/setup_cpu.sh @@ -1,15 +1,28 @@ #!/usr/bin/env bash set -euo pipefail +cueq_stage=initialize +cueq_report_failure() { + cueq_exit=$? + if [ "$cueq_exit" -ne 0 ]; then + printf 'CUEQ_SETUP_FAILED stage=%s exit_code=%s\n' "$cueq_stage" "$cueq_exit" >&2 + fi +} +trap cueq_report_failure EXIT + cueq_root="${CUEQ_EVAL_ROOT:-/workspace}" cueq_env="$cueq_root/.venv" cueq_publish=false if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then + cueq_stage=select_interpreter cueq_host_python=$(command -v python3) + cueq_stage=create_environment cueq_env=$(mktemp -d "$cueq_root/.cueq-env.XXXXXX") cueq_publish=true + printf 'Creating CPU environment: %s (host Python: %s)\n' "$cueq_env" "$cueq_host_python" "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" # Reduce setup time: wheels only, # no bytecode compilation, and no PyPI lookup for the CPU Torch build. + cueq_stage=install_cpu_torch printf 'Installing CPU PyTorch into %s\n' "$cueq_env" "$cueq_host_python" -m pip --python "$cueq_env" install \ --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ @@ -17,6 +30,7 @@ if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then --progress-bar off --no-deps 'torch==2.14.1+cpu' --index-url https://download.pytorch.org/whl/cpu # Resolve Torch's dependencies on the normal index. Older host pip # rejects some dependency metadata on the PyTorch index. + cueq_stage=install_dependencies printf 'Installing Torch dependencies and cuEquivariance into %s\n' "$cueq_env" "$cueq_host_python" -m pip --python "$cueq_env" install \ --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ @@ -24,10 +38,13 @@ if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then --progress-bar off \ 'torch==2.14.1+cpu' 'cuequivariance==0.13.0' 'cuequivariance-torch==0.13.0' fi +cueq_stage=smoke_check printf 'Checking CPU environment: %s\n' "$cueq_env" "$cueq_env/bin/python" -u - <<'PY' import importlib.metadata as metadata import sys + +print("Smoke check runtime:", sys.executable, sys.version.split()[0], flush=True) import torch import cuequivariance_torch as cuet @@ -48,5 +65,6 @@ PY # A failed install or smoke check must not leave .venv pointing at a # partial environment. Preserve it for diagnosis and retry in a fresh path. if [ "$cueq_publish" = true ]; then + cueq_stage=publish_environment ln -sT "$cueq_env" "$cueq_root/.venv" fi From 0a477fed3ccf830a83f81ee43a9162c58289a7d2 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Wed, 21 Oct 2026 11:18:25 -0700 Subject: [PATCH 16/19] Use alternate PyTorch CPU index for Tier 3 setup Signed-off-by: Ohad Mosafi --- .github/tests/test_skill_eval_setup.py | 39 +++++++++++++++++++ skills/README.md | 10 +++-- .../evals/harbor/torch-001/task.toml | 2 +- .../evals/harbor/torch-002/task.toml | 2 +- .../evals/harbor/torch-003/task.toml | 2 +- .../evals/harbor/torch-004/task.toml | 2 +- .../evals/harbor/torch-005/task.toml | 2 +- .../evals/harbor/torch-006/task.toml | 2 +- .../cuequivariance-torch/evals/setup_cpu.sh | 7 ++-- 9 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.github/tests/test_skill_eval_setup.py b/.github/tests/test_skill_eval_setup.py index 32c6679b..1e9bbef2 100644 --- a/.github/tests/test_skill_eval_setup.py +++ b/.github/tests/test_skill_eval_setup.py @@ -196,6 +196,45 @@ def test_torch_setup_identifies_the_failed_stage( assert not os.path.lexists(tmp_path / ".venv") +@pytest.mark.parametrize("certificate", [None, "/custom/trusted-ca.pem"]) +def test_cpu_torch_uses_alternate_index_and_preserves_certificate_verification( + tmp_path, setup_sandbox, certificate +): + env = dict(setup_sandbox) + env.pop("PIP_CERT", None) + if certificate is not None: + env["PIP_CERT"] = certificate + result = subprocess.run( + ["bash", "-c", setup_command("torch")], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + installs = [ + call["args"] + for line in (tmp_path / "calls.jsonl").read_text().splitlines() + if (call := json.loads(line))["stage"] == "install" + ] + wheel, dependencies = installs + assert wheel[wheel.index("--index-url") + 1] == ( + "https://download-r2.pytorch.org/whl/cpu" + ) + assert "torch==2.14.1+cpu" in wheel + assert "--no-deps" in wheel and "--no-index" not in wheel + for args in installs: + assert args[args.index("--cert") + 1] == ( + certificate or "/etc/ssl/certs/ca-certificates.crt" + ) + assert "--trusted-host" not in args + assert "torch==2.14.1+cpu" in dependencies + assert "cuequivariance-torch==0.13.0" in dependencies + assert "--no-index" not in dependencies and "--no-deps" not in dependencies + assert "--index-url" not in dependencies + assert (tmp_path / ".venv").is_symlink() + + def test_native_torch_tasks_keep_regression_prompts_and_setup_budget(): skill = ROOT / "skills/cuequivariance-torch" BUILDER["write_tasks"](skill, check=True) diff --git a/skills/README.md b/skills/README.md index cea819d8..49372963 100644 --- a/skills/README.md +++ b/skills/README.md @@ -56,9 +56,13 @@ The JAX and PyTorch setup commands publish `/workspace/.venv` only after the pinned packages pass a CPU smoke check. Failed environments remain at their unique `.cueq-env.*` paths for diagnosis; the next attempt creates a fresh one. An existing `.venv`, including a broken symlink, is preserved and must pass the -check without being replaced. Setup uses binary wheels, skips bytecode -compilation, and fetches only the PyTorch wheel from the CPU index. A second -install resolves its dependencies and cuEquivariance from the normal index, +check without being replaced. Setup uses binary wheels and skips bytecode +compilation. Torch installs its pinned CPU build with `--no-deps` from +[PyTorch's alternate CPU index](https://download-r2.pytorch.org/whl/cpu/torch/). +Pip selects the compatible wheel for the host interpreter and architecture. +This avoids the original index host that failed certificate verification in +Astra while retaining HTTPS verification and the configured `PIP_CERT` bundle. +A second install resolves Torch's dependencies and cuEquivariance from the normal index, avoiding dependency-metadata incompatibilities with older host pip versions. Both arms use the same setup and package pins. diff --git a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml index 9ef0a1b8..c0b1a89d 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-001/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml index 480ad9f2..769a75b8 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-002/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml index df5b1aa8..d0637122 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-003/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml index adcea012..67e405bf 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-004/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml index d679a2bd..2fa5ceb9 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-005/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml index c22bafe7..e9ab5832 100644 --- a/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml +++ b/skills/cuequivariance-torch/evals/harbor/torch-006/task.toml @@ -29,7 +29,7 @@ skills_dir = "/workspace/skills" build_timeout_sec = 2400.0 [environment.healthcheck] -command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Reduce setup time: wheels only,\n # no bytecode compilation, and no PyPI lookup for the CPU Torch build.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' --index-url https://download.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" +command = "bash -c 'set -euo pipefail\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\nprintf \"125\\n\" > \"$cueq_root/cueq-setup.status\"\nset +e\nbash -lc '\"'\"'#!/usr/bin/env bash\nset -euo pipefail\ncueq_stage=initialize\ncueq_report_failure() {\n cueq_exit=$?\n if [ \"$cueq_exit\" -ne 0 ]; then\n printf '\"'\"'\"'\"'\"'\"'\"'\"'CUEQ_SETUP_FAILED stage=%s exit_code=%s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_stage\" \"$cueq_exit\" >&2\n fi\n}\ntrap cueq_report_failure EXIT\n\ncueq_root=\"${CUEQ_EVAL_ROOT:-/workspace}\"\ncueq_env=\"$cueq_root/.venv\"\ncueq_publish=false\nif [ ! -e \"$cueq_env\" ] && [ ! -L \"$cueq_env\" ]; then\n cueq_stage=select_interpreter\n cueq_host_python=$(command -v python3)\n cueq_stage=create_environment\n cueq_env=$(mktemp -d \"$cueq_root/.cueq-env.XXXXXX\")\n cueq_publish=true\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Creating CPU environment: %s (host Python: %s)\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\" \"$cueq_host_python\"\n \"$cueq_host_python\" -m venv --system-site-packages --without-pip \"$cueq_env\"\n # Let pip select the pinned CPU wheel from PyTorch'\"'\"'\"'\"'\"'\"'\"'\"'s alternate index.\n # Keep HTTPS verification and resolve dependencies on the normal index below.\n cueq_stage=install_cpu_torch\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing CPU PyTorch into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off --no-deps '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' \\\n --index-url https://download-r2.pytorch.org/whl/cpu\n # Resolve Torch'\"'\"'\"'\"'\"'\"'\"'\"'s dependencies on the normal index. Older host pip\n # rejects some dependency metadata on the PyTorch index.\n cueq_stage=install_dependencies\n printf '\"'\"'\"'\"'\"'\"'\"'\"'Installing Torch dependencies and cuEquivariance into %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n \"$cueq_host_python\" -m pip --python \"$cueq_env\" install \\\n --cert \"${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}\" \\\n --disable-pip-version-check --no-input --only-binary=:all: --no-compile \\\n --progress-bar off \\\n '\"'\"'\"'\"'\"'\"'\"'\"'torch==2.14.1+cpu'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"' '\"'\"'\"'\"'\"'\"'\"'\"'cuequivariance-torch==0.13.0'\"'\"'\"'\"'\"'\"'\"'\"'\nfi\ncueq_stage=smoke_check\nprintf '\"'\"'\"'\"'\"'\"'\"'\"'Checking CPU environment: %s\\n'\"'\"'\"'\"'\"'\"'\"'\"' \"$cueq_env\"\n\"$cueq_env/bin/python\" -u - <<'\"'\"'\"'\"'\"'\"'\"'\"'PY'\"'\"'\"'\"'\"'\"'\"'\"'\nimport importlib.metadata as metadata\nimport sys\n\nprint(\"Smoke check runtime:\", sys.executable, sys.version.split()[0], flush=True)\nimport torch\nimport cuequivariance_torch as cuet\n\nfor package, expected in {\n \"cuequivariance\": \"0.13.0\", \"cuequivariance-torch\": \"0.13.0\",\n \"torch\": \"2.14.1+cpu\",\n}.items():\n assert metadata.version(package) == expected, (package, metadata.version(package))\nassert torch.version.cuda is None, torch.__version__\nlayer = cuet.SphericalHarmonics([0, 1], normalize=False, method=\"naive\")\nx = torch.ones((1, 3), dtype=torch.float64, requires_grad=True)\ny = layer(x)\nassert y.shape == (1, 4) and bool(torch.isfinite(y).all())\ny.square().sum().backward()\nassert x.grad is not None and bool(torch.isfinite(x.grad).all())\nprint(\"CPU environment ready:\", sys.executable, sys.version.split()[0], torch.__version__)\nPY\n# A failed install or smoke check must not leave .venv pointing at a\n# partial environment. Preserve it for diagnosis and retry in a fresh path.\nif [ \"$cueq_publish\" = true ]; then\n cueq_stage=publish_environment\n ln -sT \"$cueq_env\" \"$cueq_root/.venv\"\nfi\n'\"'\"' 2>&1 | tee \"$cueq_root/cueq-setup.log\"\ncueq_codes=(\"${PIPESTATUS[@]}\")\nset -e\ncueq_status=\"${cueq_codes[0]}\"\nif [ \"$cueq_status\" -eq 0 ]; then cueq_status=\"${cueq_codes[1]}\"; fi\nprintf \"%s\\n\" \"$cueq_status\" > \"$cueq_root/cueq-setup.status\"\nexit \"$cueq_status\"\n'" interval_sec = 5.0 timeout_sec = 600.0 retries = 1 diff --git a/skills/cuequivariance-torch/evals/setup_cpu.sh b/skills/cuequivariance-torch/evals/setup_cpu.sh index f4102276..958bc7d9 100644 --- a/skills/cuequivariance-torch/evals/setup_cpu.sh +++ b/skills/cuequivariance-torch/evals/setup_cpu.sh @@ -20,14 +20,15 @@ if [ ! -e "$cueq_env" ] && [ ! -L "$cueq_env" ]; then cueq_publish=true printf 'Creating CPU environment: %s (host Python: %s)\n' "$cueq_env" "$cueq_host_python" "$cueq_host_python" -m venv --system-site-packages --without-pip "$cueq_env" - # Reduce setup time: wheels only, - # no bytecode compilation, and no PyPI lookup for the CPU Torch build. + # Let pip select the pinned CPU wheel from PyTorch's alternate index. + # Keep HTTPS verification and resolve dependencies on the normal index below. cueq_stage=install_cpu_torch printf 'Installing CPU PyTorch into %s\n' "$cueq_env" "$cueq_host_python" -m pip --python "$cueq_env" install \ --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" \ --disable-pip-version-check --no-input --only-binary=:all: --no-compile \ - --progress-bar off --no-deps 'torch==2.14.1+cpu' --index-url https://download.pytorch.org/whl/cpu + --progress-bar off --no-deps 'torch==2.14.1+cpu' \ + --index-url https://download-r2.pytorch.org/whl/cpu # Resolve Torch's dependencies on the normal index. Older host pip # rejects some dependency metadata on the PyTorch index. cueq_stage=install_dependencies From 32fdecb34ee171de03e6284ae202fcbb3715e57e Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 2 Nov 2026 10:52:06 -0800 Subject: [PATCH 17/19] Fix Tier 1 SC10 finding with hash-pinned PyTorch CPU wheels Signed-off-by: Ohad Mosafi --- skills/cuequivariance-torch/SKILL.md | 8 ++++---- .../cuequivariance-torch/scripts/requirements.txt | 4 ++-- .../scripts/torch-cpu-wheels.txt | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt diff --git a/skills/cuequivariance-torch/SKILL.md b/skills/cuequivariance-torch/SKILL.md index 0722afe9..bdc16e68 100644 --- a/skills/cuequivariance-torch/SKILL.md +++ b/skills/cuequivariance-torch/SKILL.md @@ -57,20 +57,20 @@ Choose the next step from the probe: Use one interpreter for pip and execution. Honor project requirements and compatible core/frontend revisions. Retaining installed PyTorch takes priority over setup examples. Optional CUDA/optimization warnings are not failures. -When isolation is needed, create a **new unique** environment. This Linux recipe uses host pip 22.3+ without `ensurepip`: +For isolation, create a **new unique** environment. From this skill's directory, use host pip 22.3+ and the bundled [hashed CPU wheels](scripts/torch-cpu-wheels.txt), then [requirements](scripts/requirements.txt): ```bash set -e cueq_python=python3 # Set to the selected project interpreter's path. cueq_env=$(mktemp -d "${TMPDIR:-/tmp}/cueq-cpu.XXXXXX") "$cueq_python" -m venv --without-pip "$cueq_env" -python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q 'torch==2.14.1+cpu' --extra-index-url https://download.pytorch.org/whl/cpu -python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q cuequivariance-torch +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q --no-deps -r scripts/torch-cpu-wheels.txt +python3 -m pip --python "$cueq_env" install --cert "${PIP_CERT:-/etc/ssl/certs/ca-certificates.crt}" -q -r scripts/requirements.txt cueq_python="$cueq_env/bin/python" "$cueq_python" -c 'import sys, torch, cuequivariance_torch; assert torch.version.cuda is None; print("CPU setup complete:", sys.executable, torch.__version__, cuequivariance_torch.__version__)' ``` -For supplied requirements, replace both install commands with one using `-r` and their path. Retain `cueq_python` for execution; restore its path in new tool calls if needed. Without host pip, use an installed `uv`. +Retain `cueq_python` for execution. For other platforms, use project setup. Without host pip, use installed `uv` with these requirements. Use approved `PIP_CERT` for a private CA; retain TLS verification. If downloading fails, try a configured CPU mirror/cache or report the blocker. Do not replace CPU wheels with CUDA builds. Finish installs before retrying. See [pip's certificates](https://pip.pypa.io/en/stable/topics/https-certificates/). diff --git a/skills/cuequivariance-torch/scripts/requirements.txt b/skills/cuequivariance-torch/scripts/requirements.txt index 76514d21..90de0a32 100644 --- a/skills/cuequivariance-torch/scripts/requirements.txt +++ b/skills/cuequivariance-torch/scripts/requirements.txt @@ -1,6 +1,6 @@ # CPU evaluation baseline for the APIs documented in this skill. -# Match evals/config.yml; keep the CPU-only wheel build explicit. +# Match evals/setup_cpu.sh; install the pinned wheel from torch-cpu-wheels.txt first. +# Resolve dependencies using the project's existing package index configuration. cuequivariance==0.13.0 cuequivariance-torch==0.13.0 ---extra-index-url=https://download.pytorch.org/whl/cpu torch==2.14.1+cpu diff --git a/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt b/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt new file mode 100644 index 00000000..2ee03951 --- /dev/null +++ b/skills/cuequivariance-torch/scripts/torch-cpu-wheels.txt @@ -0,0 +1,15 @@ +# CPU PyTorch artifacts for standard CPython 3.10-3.14 on Linux x86_64/aarch64. +# Requires glibc 2.28+; other platforms should use their project's dependency setup. +# Install with --no-deps, then resolve dependencies using requirements.txt. +# These direct references select only Torch; they do not add a package index. +# SHA-256 hashes published at https://download-r2.pytorch.org/whl/cpu/torch/ +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=4ce2a6ae7d302bba56ef7c5c92d9f752728896cfe2626a7f3e5293f879e36650 ; python_version == "3.10" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=5e38154c8896d426a5df58bf2276b603e82184dfae4f54d77dbb212f36e34f37 ; python_version == "3.11" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=5a6363570c753812540a05eb82380e329469cbe668643e88111414c12627711f ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=331fa474e26e428e2e6af1143b2fdab19cee492d7d21897b8899ce1c14755662 ; python_version == "3.13" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl#sha256=35103e180214793207f95d7e12e100ecba3ace6f5fb37e1fcefbb07e096ed180 ; python_version == "3.14" and sys_platform == "linux" and platform_machine == "x86_64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp310-cp310-manylinux_2_28_aarch64.whl#sha256=b869fe9358c90dd82d02db89aef89729df93a89a48b7c3e21a7719abf55586b3 ; python_version == "3.10" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=36a03a3f87ec875ca39c8e29eccbaf0fc704605f78a3f7f179f157048c3c395e ; python_version == "3.11" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl#sha256=2af649994b36dffc627175c3f306d9e513a309b58d4619ac7107577613336454 ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl#sha256=783a3dec10d272e40ed3f1a05ec228400300d3229b8f7521a9327645fae88646 ; python_version == "3.13" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" +torch @ https://download-r2.pytorch.org/whl/cpu/torch-2.14.1%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl#sha256=f56cd23a3bf003bfbed3da0645ba09b3ccc66c0a2f2ac726a9b98f34e0e4926f ; python_version == "3.14" and sys_platform == "linux" and platform_machine == "aarch64" and implementation_name == "cpython" From 605793b9a07f98af55d86d1ea05068865be8dc12 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 3 Nov 2026 10:05:58 -0800 Subject: [PATCH 18/19] Improve core cuEquivariance skill documentation Signed-off-by: Ohad Mosafi --- skills/cuequivariance/SKILL.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/skills/cuequivariance/SKILL.md b/skills/cuequivariance/SKILL.md index c9c7a6ea..b4a9f5f3 100644 --- a/skills/cuequivariance/SKILL.md +++ b/skills/cuequivariance/SKILL.md @@ -1,14 +1,15 @@ --- name: cuequivariance -description: Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics). Use when working with cuequivariance group theory, irreps, or segmented polynomials. +description: Use when defining cuEquivariance groups and irreps, building equivariant tensor products, or validating segmented polynomials with NumPy. license: Apache-2.0 metadata: author: Mario Geiger + tags: [equivariance, group-theory, tensor-products] --- # cuequivariance: Groups, Irreps, and Segmented Polynomials -## Overview +## Purpose `cuequivariance` (imported as `cue`) provides two core abstractions: @@ -17,6 +18,14 @@ metadata: - `EquivariantPolynomial` — dense operands with `IrrepsAndLayout` metadata - `IrDictPolynomial` — operands already split by irrep, with per-group `Irreps` metadata for the `dict[Irrep, Array]` workflow +## Inputs + +Required inputs: requested operation, group, input irreps and multiplicities, output irrep types, layout, batch shapes, dtype, and weight ownership. Take explicit user requirements first, then existing project definitions; state defaults for unspecified values. Derive operand sizes and output block order from the descriptor. For custom groups, also establish the selection rules, generators, and Clebsch-Gordan coefficients. Optional inputs include test rotations and a framework backend. + +## Prerequisites + +Use Python 3.10+ with `cuequivariance` and its NumPy, SciPy, opt_einsum, SymPy, and NetworkX dependencies. Reuse the project's interpreter and lockfile. If setup is needed outside an existing project, the bundled [CPU requirements](scripts/requirements.txt) pin the source revision used by these examples; install them with that interpreter's pip. Core NumPy execution requires no GPU stack or API credentials. Check the installed version and API availability before using `IrDictPolynomial` with an older package. + ## Instructions 1. Check the installed version and requested backend. Install `cuequivariance` for CPU NumPy work; JAX and PyTorch execution use the separate `cuequivariance-jax` and `cuequivariance-torch` skills. Inspect available APIs before using newer features such as `IrDictPolynomial` in an older environment. @@ -27,6 +36,21 @@ metadata: Source paths below are relative to the `cuequivariance/` project inside a source checkout. When using this skill separately, inspect the installed package with `cue.__file__` or consult https://github.com/NVIDIA/cuEquivariance. The repository is not required for the examples. +## Limitations + +- Core descriptors and NumPy evaluation do not provide JAX or PyTorch execution kernels; use the corresponding frontend for framework execution and gradients. +- Custom-group mathematics must be checked independently. A valid complex representation does not establish support in a particular CUDA kernel. +- `IrDictPolynomial` describes split operands; it does not convert high-level dictionaries into the ordered arrays accepted by its underlying polynomial. Preserve the descriptor's metadata and layout when assembling those arrays. + +## Troubleshooting + +| Symptom | Check and correction | +| --- | --- | +| Import fails or an API is absent | Check the interpreter, installed version, and module location. Use the project's compatible dependencies or the bundled requirements when setting up a standalone environment. | +| Operand size or output shape is wrong | Compare each array with the descriptor's input/output dimensions; include the weights operand first when present. | +| Values disagree after a layout conversion | Transpose multiplicity and representation axes within each irrep block before flattening; keep output data in the descriptor's block order. | +| Custom-group equivariance fails | Check selection rules, generator relations, and CG covariance on a small NumPy example before changing execution backends. | + ## Examples The examples below cover custom groups, descriptors, layouts, and NumPy execution. Some snippets extend variables from the preceding snippet; the final complete example can be run independently. From d3026c6441aca51cfbf2d50eaca8ab7dc334c07a Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 4 Nov 2026 22:11:09 +0000 Subject: [PATCH 19/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/cuequivariance-jax/BENCHMARK.md | 86 ++++++++++++------ skills/cuequivariance-jax/skill-card.md | 59 +++++++++---- skills/cuequivariance-jax/skill.oms.sig | 2 +- skills/cuequivariance-torch/BENCHMARK.md | 92 ++++++++++++------- skills/cuequivariance-torch/skill-card.md | 84 ++++++++++++------ skills/cuequivariance-torch/skill.oms.sig | 2 +- skills/cuequivariance/BENCHMARK.md | 103 ++++++++++++++-------- skills/cuequivariance/skill-card.md | 75 +++++++++++----- skills/cuequivariance/skill.oms.sig | 2 +- 9 files changed, 336 insertions(+), 169 deletions(-) diff --git a/skills/cuequivariance-jax/BENCHMARK.md b/skills/cuequivariance-jax/BENCHMARK.md index d97403a1..9712e366 100644 --- a/skills/cuequivariance-jax/BENCHMARK.md +++ b/skills/cuequivariance-jax/BENCHMARK.md @@ -1,15 +1,18 @@ # Skill Benchmark: cuequivariance-jax -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata - Skill: `cuequivariance-jax` -- Evaluation date: 2026-10-19 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) - Dataset digest: `sha256:f653a01cf39e3dffe96fbacf75b1b1ee214a79c4fc5b682f49f31e6963d7bb1a` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 @@ -19,6 +22,20 @@ Live evaluation did not show a material gain or regression. Collect more evidenc Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance-jax | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-jax | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance-jax`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -33,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.8% — baseline ran, but no comparable score was available; uplift unavailable | 83.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 100.0% (+16.7 points) | 50.0% → 83.3% (+33.3 points) | +| Overall | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 95.8% → 94.2% (-1.6 points) | 91.7% → 78.3% (-13.4 points) | -| Efficiency | 76.1% — baseline ran, but no comparable score was available; uplift unavailable | 69.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 99.6% — baseline ran, but no comparable score was available; uplift unavailable | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 points) | 79.6% → 90.0% (+10.4 points) | +| Efficiency | 80.7% — baseline ran, but no comparable score was available; uplift unavailable | 70.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -46,29 +63,40 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 93.8% (+1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.5% → 86.6% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,977,893 | 6,629,286 | -3,651,393 | -55.08% | skill 6/6; base 6/6 | -| claude-code | jax-001 | 527,791 | 1,099,326 | -571,535 | -51.99% | skill 1/1; base 1/1 | -| claude-code | jax-002 | 561,852 | 404,850 | +157,002 | +38.78% | skill 1/1; base 1/1 | -| claude-code | jax-003 | 793,159 | 788,453 | +4,706 | +0.60% | skill 1/1; base 1/1 | -| claude-code | jax-004 | 625,633 | 3,461,747 | -2,836,114 | -81.93% | skill 1/1; base 1/1 | -| claude-code | jax-005 | 439,809 | 845,382 | -405,573 | -47.98% | skill 1/1; base 1/1 | -| claude-code | jax-006 | 29,649 | 29,528 | +121 | +0.41% | skill 1/1; base 1/1 | -| codex | All cases | 2,124,104 | 3,306,453 | -1,182,349 | -35.76% | skill 6/6; base 6/6 | -| codex | jax-001 | 644,663 | 876,262 | -231,599 | -26.43% | skill 1/1; base 1/1 | -| codex | jax-002 | 487,638 | 394,272 | +93,366 | +23.68% | skill 1/1; base 1/1 | -| codex | jax-003 | 440,548 | 541,449 | -100,901 | -18.64% | skill 1/1; base 1/1 | -| codex | jax-004 | 425,932 | 1,438,609 | -1,012,677 | -70.39% | skill 1/1; base 1/1 | -| codex | jax-005 | 111,915 | 42,431 | +69,484 | +163.76% | skill 1/1; base 1/1 | -| codex | jax-006 | 13,408 | 13,430 | -22 | -0.16% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 5,101,997 | 9,935,739 | -4,833,742 | -48.65% | skill 12/12; base 12/12 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 1,124,797 | 1,645,841 | -521,044 | -31.66% | skill 6/6; base 6/6 | +| claude-code | jax-001 | 171,366 | 230,492 | -59,126 | -25.65% | skill 1/1; base 1/1 | +| claude-code | jax-002 | 217,234 | 228,341 | -11,107 | -4.86% | skill 1/1; base 1/1 | +| claude-code | jax-003 | 216,567 | 186,564 | +30,003 | +16.08% | skill 1/1; base 1/1 | +| claude-code | jax-004 | 265,945 | 574,629 | -308,684 | -53.72% | skill 1/1; base 1/1 | +| claude-code | jax-005 | 218,098 | 390,218 | -172,120 | -44.11% | skill 1/1; base 1/1 | +| claude-code | jax-006 | 35,587 | 35,597 | -10 | -0.03% | skill 1/1; base 1/1 | +| codex | All cases | 1,212,345 | 1,132,654 | +79,691 | +7.04% | skill 6/6; base 6/6 | +| codex | jax-001 | 365,435 | 283,409 | +82,026 | +28.94% | skill 1/1; base 1/1 | +| codex | jax-002 | 140,879 | 167,413 | -26,534 | -15.85% | skill 1/1; base 1/1 | +| codex | jax-003 | 194,388 | 216,005 | -21,617 | -10.01% | skill 1/1; base 1/1 | +| codex | jax-004 | 300,086 | 409,921 | -109,835 | -26.79% | skill 1/1; base 1/1 | +| codex | jax-005 | 198,193 | 42,608 | +155,585 | +365.15% | skill 1/1; base 1/1 | +| codex | jax-006 | 13,364 | 13,298 | +66 | +0.50% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,337,142 | 2,778,495 | -441,353 | -15.88% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status @@ -76,7 +104,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -93,7 +121,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-jax': 133 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code diff --git a/skills/cuequivariance-jax/skill-card.md b/skills/cuequivariance-jax/skill-card.md index 83096f9c..37202aa4 100644 --- a/skills/cuequivariance-jax/skill-card.md +++ b/skills/cuequivariance-jax/skill-card.md @@ -26,7 +26,7 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
-- [pip interpreter selection (--python option)](https://pip.pypa.io/en/stable/topics/python-option/)
+- [pip HTTPS certificates](https://pip.pypa.io/en/stable/topics/https-certificates/)
- [scripts/requirements.txt](scripts/requirements.txt)
@@ -34,48 +34,71 @@ Mitigation: Review and scan skill before deployment.
**Output Type(s):** [Code, Shell commands, Analysis]
**Output Format:** [Markdown with inline Python and bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Reports observed execution results (exit status, printed shapes, errors) and identifies unrun checks.]
+**Other Properties Related to Output:** [Reports observed execution results (exit status, numerical checks, errors) and labels unrun checks.]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod; evaluator version 1.5.6, evaluated 2026-10-19. Overall verdict: NEUTRAL.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); evaluator version `1.6.7`, evaluated 2026-11-04.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Is it safe to use? Scored from the `security` signal.
-- Correctness: Is the answer correct? Scored from the `accuracy` signal.
-- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
-- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
-- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` and 50% `token_efficiency`.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+**Overall verdict: PASS — Recommended for publication** + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.8% — baseline ran, but no comparable score was available; uplift unavailable | 83.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 100.0% (+16.7 points) | 50.0% → 83.3% (+33.3 points) | +| Overall | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | | Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 95.8% → 94.2% (-1.6 points) | 91.7% → 78.3% (-13.4 points) | -| Efficiency | 76.1% — baseline ran, but no comparable score was available; uplift unavailable | 69.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 99.6% — baseline ran, but no comparable score was available; uplift unavailable | 86.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 points) | 79.6% → 90.0% (+10.4 points) | +| Efficiency | 80.7% — baseline ran, but no comparable score was available; uplift unavailable | 70.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 93.8% (+1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.5% → 86.6% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-af4c1c9 (source: git SHA, committed 2026-10-19)
+605793b (source: git SHA, committed 2026-11-03)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-jax/skill.oms.sig b/skills/cuequivariance-jax/skill.oms.sig index d8401b6d..2ddea4f1 100644 --- a/skills/cuequivariance-jax/skill.oms.sig +++ b/skills/cuequivariance-jax/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQyYTQ1NzQ4YmJhNzNlMmE2N2JkODI4Y2VhY2M4NWM0MjgyODBlZjQzZTVhYzIxOWQxOTYzNzU0MWIzYjM3YjQiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJlOTQxZmFjOTEzNTc3YTM4NDM5NTAwMjYxZjFiZjc3ZmEwOWQyMzhmNTE5NjllMmZhMDY3M2FmNGJhNjcyNmY0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjQ5NTU1MzY2MjAyNzdkY2UyYjY5Yzk2NGIzYTNhMDMyMjE5ODA0MzJiNTYwYmViNDQwYTNmOWFiZWI3ODliZTEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmM4NWExZGUzYWRhNjRiYTgwODc1ZDQ0ODY2Mjk0MmUzYTc4MTIzNTU3OTc2M2M1NjAwM2UxYmNhMDA1ODliIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJkaWdlc3QiOiAiZmUxZDdiZDNmM2NiYjk3YzM5Y2VmNWVlN2RkMjgxZjE1ZDJhNWFkYTI0NGQ1YjU0MzA3NjYyMjQxMjBmNmRlNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjE4YTI3NjhhNmJjYWYxOGQyMDVmYzhhN2U0ZmY5MzBhZjNiNjFjZjdhYzRhYWNiMDM4NGVmZDJkMjEzZDU2ODEiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDeZhFR5vm9Qd41EAREoJ68XYH/qS0dFT9oMoJRhFrwxnTMFvTd6VMt2VcsS8eIYmQCME56gLCnF9DMpmhlQxiH/XPvvYvSr0wmDcvN/7ClUiKWjdBIlEL6939r+AW+Yq8/Xg==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtamF4IiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImFhMzU1MTUzZWUzOTViM2MwZjUxYmUwYjU5YTU4YTcyNzVmNTUwNThmZDExNjA1NzI4NTAxMmY3OWUzZjg3MGYiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI1M2UxNzYwNzhkZThhNzMwZjUwNmQ3YWFhYTZkYzQxMDRkNWM0YzE5MTM2YzQ1OTBmMjAyNWUwODNlOTVkNTIzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU5MjczNDcxMjc5NzE1YzIyMDk4ZmU3MTgwMWQzYWJkMGVjYzhkYWRjMjk0Y2M5NjA2ODlkMTc3NzU5NjE4MGMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jaGFsbGVuZ2VzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImExNDg0N2QyYzQ5ODZiMmJkOTZiYjQyNzI1ZGIwN2NjYjc2NDY3ZjBiYWI5NGZhYTQwMmYyYzUxZjU4Zjk0NmQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9jb25maWcueW1sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0MzdmNTQ5ZjVjY2JlNDg5MzE2NzYwNzc2NGJjMjVhNzNkZWQ4ODdlMDkxMjlmYTg4NmRiZGU5MzhiYzQ1OTAyIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2I2NWYxZjg1Y2RmNWRkNjQ5MjIyZDhhMzVkOGZmODM4NjdmNzFlNzQ1MzlkMTRjOWNiYmViODFlZTQ0OTk4NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVxdWlyZW1lbnRzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNmM5YjIzYzNiMjExMzExYTYxNjQyYTJmNTExMDNiODg2OWFhNTFiYTBiZThmMmQyMTEyODA1M2IxOTQ1MDVkMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ2NzU1YzMxYTIwYzIwMzZkYmIxZmM3ODE2YTZhYTBlMjBiYmE1MDE3N2MwYTE5MWZmNjYxMjJmMjJjMmExZDIiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCjC2fdcwi96n//rUpZ2BkFtiJJvADC5CsWK42JCFx55ygq16sfAFHfzz4XBGeMQckCMGUR8f1/6XnlQzkfvGkfKMa5Y2p2baXGefCNJp+2btf1CkWad2SF0ZBscMfNZ9LYow==","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance-torch/BENCHMARK.md b/skills/cuequivariance-torch/BENCHMARK.md index bc226f6e..fb0bea1d 100644 --- a/skills/cuequivariance-torch/BENCHMARK.md +++ b/skills/cuequivariance-torch/BENCHMARK.md @@ -1,15 +1,18 @@ # Skill Benchmark: cuequivariance-torch -> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** +> ✅ **Overall verdict: PASS — Recommended for publication** -Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ## Evaluation Metadata - Skill: `cuequivariance-torch` -- Evaluation date: 2026-10-19 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) - Dataset digest: `sha256:c59a90389aabc4ce9d6756217b3f98021106bda2f5a6df42bb36c78c5a0135db` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 @@ -19,6 +22,20 @@ Live evaluation did not show a material gain or regression. Collect more evidenc Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance-torch | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance-torch | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance-torch`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -33,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 75.0% → 100.0% (+25.0 points) | 83.3% → 66.7% (-16.6 points) | -| Correctness | 100.0% → 93.3% (-6.7 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 88.8% → 86.4% (-2.4 points) | 80.7% → 72.5% (-8.2 points) | -| Efficiency | 75.9% — baseline ran, but no comparable score was available; uplift unavailable | 60.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 90.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 89.6% → 97.7% (+8.1 points) | 88.8% → 92.3% (+3.5 points) | +| Efficiency | 83.9% — baseline ran, but no comparable score was available; uplift unavailable | 69.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -46,29 +63,40 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 90.4% → 89.5% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 86.2% → 86.3% (+0.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 5,223,194 | 6,398,169 | -1,174,975 | -18.36% | skill 6/6; base 6/6 | -| claude-code | torch-001 | 975,100 | 1,392,092 | -416,992 | -29.95% | skill 1/1; base 1/1 | -| claude-code | torch-002 | 1,453,016 | 1,741,134 | -288,118 | -16.55% | skill 1/1; base 1/1 | -| claude-code | torch-003 | 1,492,508 | 1,175,512 | +316,996 | +26.97% | skill 1/1; base 1/1 | -| claude-code | torch-004 | 411,116 | 966,688 | -555,572 | -57.47% | skill 1/1; base 1/1 | -| claude-code | torch-005 | 861,581 | 1,092,854 | -231,273 | -21.16% | skill 1/1; base 1/1 | -| claude-code | torch-006 | 29,873 | 29,889 | -16 | -0.05% | skill 1/1; base 1/1 | -| codex | All cases | 4,231,771 | 5,027,418 | -795,647 | -15.83% | skill 6/6; base 6/6 | -| codex | torch-001 | 1,646,932 | 1,677,817 | -30,885 | -1.84% | skill 1/1; base 1/1 | -| codex | torch-002 | 739,871 | 612,047 | +127,824 | +20.88% | skill 1/1; base 1/1 | -| codex | torch-003 | 608,840 | 840,494 | -231,654 | -27.56% | skill 1/1; base 1/1 | -| codex | torch-004 | 786,916 | 531,744 | +255,172 | +47.99% | skill 1/1; base 1/1 | -| codex | torch-005 | 435,590 | 1,351,661 | -916,071 | -67.77% | skill 1/1; base 1/1 | -| codex | torch-006 | 13,622 | 13,655 | -33 | -0.24% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 9,454,965 | 11,425,587 | -1,970,622 | -17.25% | skill 12/12; base 12/12 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 1,381,967 | 1,730,143 | -348,176 | -20.12% | skill 6/6; base 6/6 | +| claude-code | torch-001 | 269,802 | 189,690 | +80,112 | +42.23% | skill 1/1; base 1/1 | +| claude-code | torch-002 | 271,109 | 315,476 | -44,367 | -14.06% | skill 1/1; base 1/1 | +| claude-code | torch-003 | 316,508 | 475,103 | -158,595 | -33.38% | skill 1/1; base 1/1 | +| claude-code | torch-004 | 267,790 | 226,387 | +41,403 | +18.29% | skill 1/1; base 1/1 | +| claude-code | torch-005 | 220,588 | 487,495 | -266,907 | -54.75% | skill 1/1; base 1/1 | +| claude-code | torch-006 | 36,170 | 35,992 | +178 | +0.49% | skill 1/1; base 1/1 | +| codex | All cases | 1,219,438 | 997,915 | +221,523 | +22.20% | skill 6/6; base 6/6 | +| codex | torch-001 | 157,419 | 190,687 | -33,268 | -17.45% | skill 1/1; base 1/1 | +| codex | torch-002 | 294,988 | 196,951 | +98,037 | +49.78% | skill 1/1; base 1/1 | +| codex | torch-003 | 246,630 | 217,498 | +29,132 | +13.39% | skill 1/1; base 1/1 | +| codex | torch-004 | 269,906 | 85,850 | +184,056 | +214.39% | skill 1/1; base 1/1 | +| codex | torch-005 | 236,447 | 293,109 | -56,662 | -19.33% | skill 1/1; base 1/1 | +| codex | torch-006 | 14,048 | 13,820 | +228 | +1.65% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,601,405 | 2,728,058 | -126,653 | -4.64% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status @@ -76,7 +104,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c |---|---|---|---| | Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 6 task(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -86,14 +114,14 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - Schema & Repository Governance: Found skill manifest: SKILL.md - Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) - Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) -- PII Scan: Scanning 2 files for PII +- PII Scan: Scanning 3 files for PII - Code Integrity & Hygiene: Checking 1 markdown files for dead links -- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- Unicode Smuggling Detection: No invisible Unicode characters detected in 3 file(s) - QUALITY: Score: 100.0/100 (Grade: A) - SCRIPT_LINT: No Python scripts found in scripts/ - Context Deduplication: Collected 1 file(s) - Inter-Skill Deduplication: Parsed skill 'cuequivariance-torch': 151 char description -- AGENT_EVAL: Tier 3 evaluation complete: verdict NEUTRAL; best agent claude-code +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent claude-code diff --git a/skills/cuequivariance-torch/skill-card.md b/skills/cuequivariance-torch/skill-card.md index 5fdf3a3f..6778a55c 100644 --- a/skills/cuequivariance-torch/skill-card.md +++ b/skills/cuequivariance-torch/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and researchers building equivariant geometric neural networks in PyTorch use this skill to write, debug, and validate cuequivariance_torch code, including CPU reference calculations, backend selection, irrep layouts and weights, gather/scatter indexing, gradient checks, and trainable message-passing layers.
+Developers and engineers use this skill to build, run, and debug cuEquivariance descriptors with cuequivariance_torch in PyTorch, including CPU naive references, CUDA backends, equivariant layers and GNN message passing, irreps layouts, gather/scatter indexing, and autograd gradient checks.
### Deployment Geography for Use:
Global
@@ -26,8 +26,9 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
-- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
-- [pip interpreter selection (--python option)](https://pip.pypa.io/en/stable/topics/python-option/)
+- [pip HTTPS certificates](https://pip.pypa.io/en/stable/topics/https-certificates/)
+- [scripts/torch-cpu-wheels.txt](scripts/torch-cpu-wheels.txt)
+- [scripts/requirements.txt](scripts/requirements.txt)
## Skill Output:
@@ -37,45 +38,78 @@ Mitigation: Review and scan skill before deployment.
**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox; with-skill runs compared against no-skill baselines.
+6 evaluation tasks (5 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment); evaluator version `1.6.7`, evaluation date 2026-11-04. Baseline is the same task attempted without the target skill.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Is it safe to use?
-- Correctness: Is the answer correct?
-- Discoverability: Was the right skill loaded when needed?
-- Effectiveness: Did the skill help complete the task?
-- Efficiency: Did it avoid wasted tool calls and token usage?
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity.
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage.
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 76.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 75.0% → 100.0% (+25.0 points) | 83.3% → 66.7% (-16.6 points) | -| Correctness | 100.0% → 93.3% (-6.7 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 88.8% → 86.4% (-2.4 points) | 80.7% → 72.5% (-8.2 points) | -| Efficiency | 75.9% — baseline ran, but no comparable score was available; uplift unavailable | 60.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 90.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 89.6% → 97.7% (+8.1 points) | 88.8% → 92.3% (+3.5 points) | +| Efficiency | 83.9% — baseline ran, but no comparable score was available; uplift unavailable | 69.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 90.4% → 89.5% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 86.2% → 86.3% (+0.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Skill Version(s):
-0.13.0 (source: VERSION file, git commit af4c1c9 2026-10-19)
+605793b (source: git SHA, committed 2026-11-03)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance-torch/skill.oms.sig b/skills/cuequivariance-torch/skill.oms.sig index 9c4d0265..1c3ea842 100644 --- a/skills/cuequivariance-torch/skill.oms.sig +++ b/skills/cuequivariance-torch/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYjFkNjZmNmU1OTZlMTZhMTFhOGY4ZDY0OTg3NzU4NTU1Nzk0N2MyMzUzMTBmMWE0NDQzYjRjMTExOTg1Y2Y0OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjM1YjhmOTUwZjhkMmJiODc0MTkyZjc2YjI5YmQxY2VkODI0Mzk0NTRkOTQ3ODhmNDhjYzk3YmYyMTg1M2FjZjQiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhiYjcxNmY4YzdmNWJiYzZjMjBlY2JkNTg0MTI0OTI1MWIyZmY1MDhiMmVmMWJjZjU3ODY1MWYxODczYzM0MTEiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJhOTY5YjMwMTE4M2QwZjAyYjcyOGRmOWM1ZWM2OWRlZDYxMjA5NmM4MjM0Mzg1YWY3MWM4OGMwY2JjMjI2MzQwIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMmExYWJlM2Y0N2EyY2MwOWIzMjA3ZTQxYWM4NzM2ZWJhNWVlMzMxOTBiOWM2YjAxZDE3YmNiYjNlZjAwYjBkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDQ9SoymEBxNnx0WT5+siyJd9+6nMd+FYS9wK3IoUwz/OHnXcqXjkMMm+iUM20UmWACMQC8bcrdtVBO3r8ZtwouswAeURFCViReU9fadmElnWg3fjtx5a05i3V8QoeDReFQkEs=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UtdG9yY2giLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMzFmMTA3ODUyYmI3M2JlM2IyYWMyNTRlMTJiM2E2ZWUzYmI2M2M2NjZiNjliMzMzMjU2ODQ1YWU3OTUzNmI0NSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDE5YWRiODM5MzQ4MjE3YjM2MmI0NWFmNzMzNTNjOWQ4ZTcxZTcwZTAyNTdhZDczMjY4MTIxMzg5ZDgzM2I4ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxYmJlNGM5NzRiZGE4NmUwMmRjODhlMWU2MzQ4NDMzMWJkOWVkODE1OGY5Yjc1NDAyMjExZmQ5OWM4NTJiYTViIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY2hhbGxlbmdlcy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhNDcwNDA2YjU0ODI4ZWY1ZGZmMmQzNGZkMzJmYjNhN2YwYWZlODU5YzMyNTdhNDI3ZTg2OGUyOTIwMTg2M2RlIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTUyZmZmZjQxZjBlYjk4YTg3NGMyOTBjOWU0NTIxYmNmYTUxMTc3NDJlMTRkMDMwMDZiZTQ0MDUxMTI0YWUyNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImRjYzhkMWFiZmZlYjUzODA3OWM3ODlhYjU4OGM2MjRjZDM2OTA1N2JjNTIwOWQyNWQ5ZjMzYzUzNDMxMGQwNGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9oYXJib3IvZGF0YXNldC50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMWM4ZWI2MWQwNWQ5OGQ5Y2ZiYTYyYzM2NjQ1MmJmYTRiNDNkZGZkMGQzYmE3ZjQzMTE0NWViZDdlOWU3MDRiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMS9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMS9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTVjYjRmMjU1MDZhOTA0ZmMzMDkzZTdjZGM5MDRhODJiMzIxMTljZDUzZGZhNmNhNjViNGE4Nzc4OTA3MDVhZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDEvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NDE1ODc0MThjZTU0YTUwMzU3NzZhYWVjYzdiZWQxN2MyNWI3MjRjYmQxYzkwNWJjNzE3ZDBmNWVhZDhkNGU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMi9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMi9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYWMyN2FiZTI1OTllZTQ3ZGI5YTgzNDlhNDI4YWJmMGY5YjQ3NTdiMjMzN2U3ZGZmNzM3MmU0Zjk2ZGEwNmIzYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDIvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5MmIxNThlZDlhZDdhYzk3Y2FkODk3MzFjNzYxZDYyZDMxZTZmYjE0NTE3YzlmODYwYTQ1MjJkNzhlMjBlM2Y1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMy9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwMy9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2EyM2QyNWVmYjUzOWMzMGM3MzI5NjNkYzUyYWY4NTJkYmRjMjcyNWViYWNjNjEwOGEyZDZiNmM3YjhlM2M2YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDMvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkMTMzODQ4MGNjZjFmNjBhODEyYjdkN2MyZWE5MDBmYjU2YmZiYjViODg0NmFmMmMyYmQ2YmFlYTRlYzkxZDA5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNC9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNC9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDVkYzE3ZGI5YTIzYzBkOWM2NTgwNGU1YzBkNzdiNDY0NzljMWI5MjI5YWVlMWVmNWRjYzJhMjNiZGYzMTIxYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDQvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyNTZkMzM4MGUzMzEyMDdmMjQ1MDA0ZTU5ZDkyZmRiNzYxOGUyYWIwMWI4ZmFmNDdjNzg1YTM5Y2JhYTE4ZTAwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNS9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNS9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZWQ1N2M4MjdmY2Y1YTgyYjBkZjljZDZmMDQ2ZDUwNzY3NzJhODFjMzM3NGRkNjkyNzg0M2YxMjk3NmU4NTU3OSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDUvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NWM4ZTk3NzhhMWZkZDNkMDQwNjUzM2NmOGMzNzI1OWZlODExZjQwY2Q3MGQxNTA3MGRiOTcyMjk0NWIyYTMwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNi9lbnZpcm9ubWVudC9Eb2NrZXJmaWxlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlZGMzZjk2MTliYjE0NDhjN2NlMjY1NWUzMmYwNjU3NGNjYWY3OTljZDQ4MmYyOTgyMzc4ZTFkYjc5YWQ4MjgwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvaGFyYm9yL3RvcmNoLTAwNi9pbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNDc4MjJlY2U5YjQzYWUzMjRmZTY0YWIzODI1MjI2ZDUwYWUxMjUxZjJkMTc1ODRhZTA5MDFmMjBiZWRmNzg1YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2hhcmJvci90b3JjaC0wMDYvdGFzay50b21sIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZmZjNmYzMTZhMGEzMjM0NWJiOGI5YzM0MTc4OTg2MGY2Y2M1Y2JmOGM5YjI0YTRhNjZiYjcyZjRjOTY4N2Y1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvc2V0dXBfY3B1LnNoIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwMWRjMzNhYWM2ZTQxYmI4NzA1YzRkOGE4NzA4NTRjYmE2N2IzNjQ1ZWE4NTMzNTI0NWRiMjg3YzQxODhiNjZmIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0NjEyMjU0YzJjN2NmOGFlMTZjZjE5MTAzZjAxZWExODg1M2ZkYzU2NTI0ZTc3YzczZWZmMmQ4ZWM1YjIyOGY1IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy90b3JjaC1jcHUtd2hlZWxzLnR4dCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGMwOTc0MTExYjhmZWM0MGYxOGY2YTljNDQwYzczMzIzYzgzMzgxNWVhN2IzZDdjYWViMWZkMGY2MjE5MjU5NCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ1Zjg4ZWU0ZDg4NzI0YmRhNmM5ZDExN2VmZDc1MTE3NTZkNzA5NWM3OTI0MDk3MjE4NjdhM2I2ZDQ0MWVhZTIiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDx5Ozhc/dFSwFYrwgBtDGDcS5XNVqiDalxrz3vwQ0GiUKVqjMBs81JEZQ7oRwmr4ECMQDmdzF3q5JkVVFDW4m6Ux8h6qIMucmJ5mPUu/H0gh3d61CX5OaGg3PvVpbJq7z/OPw=","keyid":""}]}} \ No newline at end of file diff --git a/skills/cuequivariance/BENCHMARK.md b/skills/cuequivariance/BENCHMARK.md index 122d5f91..845485ab 100644 --- a/skills/cuequivariance/BENCHMARK.md +++ b/skills/cuequivariance/BENCHMARK.md @@ -9,9 +9,10 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `cuequivariance` -- Evaluation date: 2026-10-19 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) - Dataset digest: `sha256:64264cc94192671086b1204edef9cf61e0c0710d636da7afc0c1f0c594e2cd55` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 @@ -21,6 +22,20 @@ Recommended for publication based on the completed evaluation evidence in this r Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| cuequivariance | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| cuequivariance | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `cuequivariance`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -35,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 85.5% — baseline ran, but no comparable score was available; uplift unavailable | 91.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 83.3% (±0.0 points) | 83.3% → 100.0% (+16.7 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 94.2% → 92.9% (-1.3 points) | 88.3% → 99.2% (+10.9 points) | -| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 75.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 33.3% (-50.0 points) | 100.0% → 75.0% (-25.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 91.7% (-4.1 points) | 74.2% → 79.6% (+5.4 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 68.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -48,36 +63,47 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 87.7% → 86.7% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 78.6% → 82.9% (+4.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 5,881,399 | 7,298,202 | -1,416,803 | -19.41% | skill 6/6; base 6/6 | -| claude-code | core-001 | 956,726 | 1,188,219 | -231,493 | -19.48% | skill 1/1; base 1/1 | -| claude-code | core-002 | 510,535 | 869,031 | -358,496 | -41.25% | skill 1/1; base 1/1 | -| claude-code | core-003 | 710,993 | 2,235,618 | -1,524,625 | -68.20% | skill 1/1; base 1/1 | -| claude-code | core-004 | 708,256 | 959,125 | -250,869 | -26.16% | skill 1/1; base 1/1 | -| claude-code | core-005 | 2,964,658 | 2,015,952 | +948,706 | +47.06% | skill 1/1; base 1/1 | -| claude-code | core-006 | 30,231 | 30,257 | -26 | -0.09% | skill 1/1; base 1/1 | -| codex | All cases | 2,400,367 | 2,780,675 | -380,308 | -13.68% | skill 6/6; base 6/6 | -| codex | core-001 | 720,238 | 494,550 | +225,688 | +45.64% | skill 1/1; base 1/1 | -| codex | core-002 | 474,477 | 618,515 | -144,038 | -23.29% | skill 1/1; base 1/1 | -| codex | core-003 | 429,875 | 645,457 | -215,582 | -33.40% | skill 1/1; base 1/1 | -| codex | core-004 | 356,473 | 692,463 | -335,990 | -48.52% | skill 1/1; base 1/1 | -| codex | core-005 | 405,363 | 315,871 | +89,492 | +28.33% | skill 1/1; base 1/1 | -| codex | core-006 | 13,941 | 13,819 | +122 | +0.88% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 8,281,766 | 10,078,877 | -1,797,111 | -17.83% | skill 12/12; base 12/12 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 3,055,508 | 3,257,506 | -201,998 | -6.20% | skill 6/6; base 6/6 | +| claude-code | core-001 | 705,861 | 677,606 | +28,255 | +4.17% | skill 1/1; base 1/1 | +| claude-code | core-002 | 541,291 | 453,626 | +87,665 | +19.33% | skill 1/1; base 1/1 | +| claude-code | core-003 | 763,691 | 607,148 | +156,543 | +25.78% | skill 1/1; base 1/1 | +| claude-code | core-004 | 600,043 | 462,929 | +137,114 | +29.62% | skill 1/1; base 1/1 | +| claude-code | core-005 | 408,028 | 1,019,801 | -611,773 | -59.99% | skill 1/1; base 1/1 | +| claude-code | core-006 | 36,594 | 36,396 | +198 | +0.54% | skill 1/1; base 1/1 | +| codex | All cases | 1,993,897 | 2,339,732 | -345,835 | -14.78% | skill 6/6; base 6/6 | +| codex | core-001 | 419,817 | 297,436 | +122,381 | +41.15% | skill 1/1; base 1/1 | +| codex | core-002 | 364,535 | 472,134 | -107,599 | -22.79% | skill 1/1; base 1/1 | +| codex | core-003 | 513,014 | 586,486 | -73,472 | -12.53% | skill 1/1; base 1/1 | +| codex | core-004 | 450,250 | 493,023 | -42,773 | -8.68% | skill 1/1; base 1/1 | +| codex | core-005 | 219,832 | 463,843 | -244,011 | -52.61% | skill 1/1; base 1/1 | +| codex | core-006 | 26,449 | 26,810 | -361 | -1.35% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 5,049,405 | 5,597,238 | -547,833 | -9.79% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 7 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 1 finding(s) | +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Findings and Observations @@ -85,16 +111,17 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **HIGH** DUPLICATE/duplicate: Duplicate content found within SKILL.md: - "### IrDictPolynomial descriptors" in SKILL.md (lines 247-257) - vs "### IrDictPolynomial" in SKILL.md (lines 287-301) - vs "### split_polynomial_by_irreps" in SKILL.md (lines 312-322) - vs "# Split an operand into per-irrep pieces (for ir_dict interface)" in SKILL.md (lines 336-338) (`SKILL.md:247`) -- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/cuequivariance/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: Description very long (310 chars, recommend 50-150) (`skills/cuequivariance/SKILL.md`) -- **LOW** QUALITY/quality_discoverability: No '## Purpose' section (`skills/cuequivariance/SKILL.md`) -- **LOW** QUALITY/quality_reliability: No prerequisites/requirements documented (`skills/cuequivariance/SKILL.md`) -- 3 additional finding(s) are available in the full evaluation artifacts. +- Schema & Repository Governance: Found skill manifest: SKILL.md +- Semantic Version Validation: No semantic version label present; resource will use commit-hash history (opting back out of an existing label is allowed) +- Security Scan: No security vulnerabilities detected (secrets, API keys, credentials) +- PII Scan: Scanning 2 files for PII +- Code Integrity & Hygiene: Checking 1 markdown files for dead links +- Unicode Smuggling Detection: No invisible Unicode characters detected in 2 file(s) +- QUALITY: Score: 100.0/100 (Grade: A) +- SCRIPT_LINT: No Python scripts found in scripts/ +- Context Deduplication: Collected 1 file(s) +- Inter-Skill Deduplication: Parsed skill 'cuequivariance': 137 char description +- AGENT_EVAL: Tier 3 evaluation complete: verdict PASS; best agent codex
diff --git a/skills/cuequivariance/skill-card.md b/skills/cuequivariance/skill-card.md index 218b8628..b1789a3a 100644 --- a/skills/cuequivariance/skill-card.md +++ b/skills/cuequivariance/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Define custom groups (Irrep subclasses), build segmented tensor products with CG coefficients, create equivariant polynomials and IrDictPolynomials, and use built-in descriptors (linear, tensor products, spherical harmonics) when working with cuequivariance group theory, irreps, or segmented polynomials.
+Use when defining cuEquivariance groups and irreps, building equivariant tensor products, or validating segmented polynomials with NumPy.
This skill is ready for commercial/non-commercial use.
@@ -9,14 +9,14 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers building equivariant geometric neural networks use this skill to define groups and irreps, construct segmented tensor products and equivariant polynomials with the cuequivariance library, and validate them with CPU NumPy execution.
+Developers and engineers building geometric (equivariant) neural networks use this skill to define custom groups and irreps, build segmented tensor products with Clebsch-Gordan coefficients, use built-in descriptors, and validate equivariant polynomials with NumPy on CPU.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Requires API Key or External Credential:** [No]
+**Credential Type(s):** [None]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
@@ -25,9 +25,9 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [cuEquivariance Documentation](https://docs.nvidia.com/cuda/cuequivariance/)
-- [cuEquivariance GitHub Repository](https://github.com/NVIDIA/cuEquivariance)
-- [scripts/requirements.txt](scripts/requirements.txt)
+- [cuEquivariance documentation](https://docs.nvidia.com/cuda/cuequivariance/)
+- [cuEquivariance GitHub repository](https://github.com/NVIDIA/cuEquivariance)
+- [Bundled CPU requirements](scripts/requirements.txt)
## Skill Output:
@@ -37,45 +37,72 @@ Mitigation: Review and scan skill before deployment.
**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Is it safe to use? Scored from the `security` signal.
-- Correctness: Is the answer correct? Scored from the `accuracy` signal.
-- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
-- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
-- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` and 50% `token_efficiency`.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+**Overall verdict:** ✅ PASS — Recommended for publication + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 85.5% — baseline ran, but no comparable score was available; uplift unavailable | 91.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 83.3% → 83.3% (±0.0 points) | 83.3% → 100.0% (+16.7 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 94.2% → 92.9% (-1.3 points) | 88.3% → 99.2% (+10.9 points) | -| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 72.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 75.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 83.3% → 33.3% (-50.0 points) | 100.0% → 75.0% (-25.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 96.7% (-3.3 points) | +| Discoverability | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | 83.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.8% → 91.7% (-4.1 points) | 74.2% → 79.6% (+5.4 points) | +| Efficiency | 66.3% — baseline ran, but no comparable score was available; uplift unavailable | 68.1% — baseline ran, but no comparable score was available; uplift unavailable | + +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 87.7% → 86.7% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 78.6% → 82.9% (+4.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED** | 11 validator(s); 0 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Skill Version(s):
-af4c1c9 (source: git SHA, committed 2026-10-19)
+605793b (source: git SHA, committed 2026-11-03)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/cuequivariance/skill.oms.sig b/skills/cuequivariance/skill.oms.sig index 7f65490e..695c19ac 100644 --- a/skills/cuequivariance/skill.oms.sig +++ b/skills/cuequivariance/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZjFmNWUyNmI3NDIyZGEwNTlmNzQ4NjAxYzlmMzU5NTc0NThiZGIyNjBiNzYyMGQzMjdkZTcwYzM1YTIwNjMzMyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjEyNjg5MDI0MGJmYzJkOWU3ZThmOTg1NjVmYzVkMDU3MzE1NmU5MTk4N2YzOGI2YzcxNTA4NWNkNTZlOWJmOGMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNiYTJjODcyNWY0OTkwZWFiZGE0YTBkMjBkNWFiOTE5NGQyYTJlNmY5YTY3N2E4Y2I3MWU5ODI0MWQxN2ZmNGIiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXF1aXJlbWVudHMudHh0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiNDYwZTVhYmNkM2U2YWJkZDM4NDgzZTBiNzY2Mjc1ODJhMjQ4MjliODVjZmVjYjhlMWJkZjQyYzU0MGY4MzZjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMFyx+hockE2Zeybx5xB0kGUFv7gg6Uo697YKc9waXocvFJ5PmA0gvjJb6W57dRSZbgIwN7b9iCGzPLPepPT10WHSExXYk2hxsU8irPwQeL26CGQg9Uk0ao1+6VkTz2JReVIr","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiY3VlcXVpdmFyaWFuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNjZmYjkzM2NjNDgxZjBjNDRkODJlZGEyZDQ3OGM4ZWVjYzhjMjg1Y2UzNDhhNDQyMjMxMjE5YzVkMDQwZjhmOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ5MjgwNWQ1ZjFkNjYyMGI1M2JiZWY3NzE2NGQ3NmVlMmU1NWI3NDAyMjI5MGQwNDk5Njg1OTY0Y2Y4MTM4NmEiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjczNjRjYzdkZTNmMDExMDZkNzFjNDZjNDBkMzcyYjFjZGFjOTUxMmJlMWQ0M2ZlMDkxMGI5M2NhMTNiYjUyMWYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTJjODVhMWRlM2FkYTY0YmE4MDg3NWQ0NDg2NjI5NDJlM2E3ODEyMzU1Nzk3NjNjNTYwMDNlMWJjYTAwNTg5YiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImI5Y2U2ZjQ3NTRjMWU3YWMyZDU0NzNjMjMwOGRiNzk2M2U5MTI4YjM1NTI2NjFmYTIxNDU0MTEyZDAxNTllZTMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MThhZDFjMWE4OWYxNWRiMWRiMTEzYjBiOWJmNDQ3MTMzMDExMmU1MzliMTBmZDhhMjRhN2M0NGQyOWI3MDA5IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JlcXVpcmVtZW50cy50eHQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNGY1YjU2NTYyZmUzYTY1NDAzYzE3YWNhNmEzYjYzN2Y0ZDNjNmRmNzczN2FhYTdjZGFkNWE5NTAyOTQyNTUzIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMBAu9wTJNRJ7WiH56Rwv6LrXwAOkJiir9Do+keqlD6iQ2QrM4C5Qgb4Fg5tZr1VxGwIwLenEHwhR3dqB/BeGwkSgXOYDaExvesDVp/qhU2WPHzeJQ4TtE1CmcoW59Pdu0gZ6","keyid":""}]}} \ No newline at end of file