Add prebuilt Metal variants for torch 2.12 and 2.13

Verified against the pure-torch reference dequantizer: dequantize is bit-exact
for q8_0/q4_K/q5_K/q6_K and mul_mat_vec agrees to ~1e-6 relative on all four.

.gitattributes

@@ -48,3 +48,5 @@ build/torch211-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filte
 build/torch212-cxx11-cu126-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch212-cxx11-cu130-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch212-cxx11-cu132-x86_64-linux/_gguf_kernels_cuda_e4e2164.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_8eddc6f.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_8eddc6f.abi3.so filter=lfs diff=lfs merge=lfs -text

build/torch212-metal-aarch64-darwin/__init__.py

@@ -0,0 +1,55 @@
+"""Compute directly on the packed blocks of a GGUF checkpoint.
+
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+
+import torch
+
+from ._ops import add_op_namespace_prefix, ops
+
+
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+
+# ggml type ids `mul_mat_vec` implements: Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, and the IQ quants.
+# `dequantize` covers more, so check this one before choosing the fused path.
+GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+
+
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+
+
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+
+
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+
+
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch212-metal-aarch64-darwin/_gguf_kernels_metal_8eddc6f.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a3bb5ec82d6f44ab3dfd809791c7934380dd1d104e08af2eb97fa224d79c597
+size 7378384

build/torch212-metal-aarch64-darwin/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _gguf_kernels_metal_8eddc6f
+ops = torch.ops._gguf_kernels_metal_8eddc6f
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_metal_8eddc6f::{op_name}"

build/torch212-metal-aarch64-darwin/metadata.json

@@ -0,0 +1,29 @@
+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_metal_8eddc6f",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "metal"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "Xl1/3Qvp7VYrhYXje6GLBwPa9H7ICY5jHKLc7frZvlo=",
+      "_gguf_kernels_metal_8eddc6f.abi3.so": "Gju17ILW9Eqz39gJeRx5NDgN0dEE4Iry65f6Ik15xZc=",
+      "_ops.py": "661PiUKH8VTMPv3Ls7VBKY+93qr+jFOKGTo1kZZV/yM="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "8eddc6f8d9c7c81f19fdb11f8725c4bf9801f62d",
+      "dirty": false
+    }
+  }
+}

build/torch213-metal-aarch64-darwin/__init__.py

@@ -0,0 +1,55 @@
+"""Compute directly on the packed blocks of a GGUF checkpoint.
+
+A GGUF weight is stored as blocks of 32 or 256 values sharing a scale. These kernels read that
+layout as it is, so a quantized model can be loaded and run without ever materializing a dense
+copy — which is the whole memory saving.
+
+Ported from llama.cpp's ggml-cuda; `vendor/UPSTREAM` pins the revision. The ops are named for what
+they do rather than for a backend: each backend registers its own implementation of the same
+schema, so calls dispatch on the tensor's device.
+"""
+
+import torch
+
+from ._ops import add_op_namespace_prefix, ops
+
+
+__all__ = ["GEMV_TYPES", "MAX_GEMV_ROWS", "dequantize", "mul_mat_vec"]
+
+# Upstream's MMVQ_MAX_BATCH_SIZE: `mul_mat_vec` has no implementation beyond this many rows, so a
+# caller with more (prefill) dequantizes and uses an ordinary matmul.
+MAX_GEMV_ROWS = 8
+
+# ggml type ids `mul_mat_vec` implements: Q4_0/Q4_1/Q5_0/Q5_1/Q8_0, the K quants, and the IQ quants.
+# `dequantize` covers more, so check this one before choosing the fused path.
+GEMV_TYPES = frozenset({2, 3, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39, 40, 41, 42})
+
+
+def dequantize(
+    blocks: torch.Tensor, ggml_type: int, rows: int, cols: int, dtype: torch.dtype
+) -> torch.Tensor:
+    """`(rows, bytes_per_row)` uint8 blocks -> `(rows, cols)` values of `dtype`."""
+    return ops.dequantize(blocks, ggml_type, rows, cols, dtype)
+
+
+def mul_mat_vec(
+    blocks: torch.Tensor, x: torch.Tensor, ggml_type: int, out_features: int
+) -> torch.Tensor:
+    """Fused dequantize-gemv: `x @ blocks.T` without unpacking `blocks`.
+
+    `x` is `(rows, in_features)` and `rows` must be at most `MAX_GEMV_ROWS`. The result is f32
+    whatever `x`'s dtype was, since the kernel writes an f32 destination; cast it at the call site,
+    where the cast can be fused into whatever consumes it.
+    """
+    return ops.mul_mat_vec(blocks, x, ggml_type, out_features)
+
+
+# Without these, torch.compile cannot trace the ops and breaks the graph at every call.
+@torch.library.register_fake(add_op_namespace_prefix("dequantize"))
+def _dequantize_fake(blocks, ggml_type, rows, cols, dtype):
+    return blocks.new_empty((rows, cols), dtype=dtype)
+
+
+@torch.library.register_fake(add_op_namespace_prefix("mul_mat_vec"))
+def _mul_mat_vec_fake(blocks, x, ggml_type, out_features):
+    return x.new_empty((x.shape[0], out_features), dtype=torch.float32)

build/torch213-metal-aarch64-darwin/_gguf_kernels_metal_8eddc6f.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6f177c4b3987b307fd2430c2381e0f125e06bde41f001cf2f3684d677a9cfaf
+size 7378384

build/torch213-metal-aarch64-darwin/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _gguf_kernels_metal_8eddc6f
+ops = torch.ops._gguf_kernels_metal_8eddc6f
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_gguf_kernels_metal_8eddc6f::{op_name}"

build/torch213-metal-aarch64-darwin/metadata.json

@@ -0,0 +1,29 @@
+{
+  "name": "gguf-kernels",
+  "id": "_gguf_kernels_metal_8eddc6f",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "metal"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "Xl1/3Qvp7VYrhYXje6GLBwPa9H7ICY5jHKLc7frZvlo=",
+      "_gguf_kernels_metal_8eddc6f.abi3.so": "tvF3xLOYezB/0kMMI4Hg8SXga95B8AHPLzaE1nepz68=",
+      "_ops.py": "661PiUKH8VTMPv3Ls7VBKY+93qr+jFOKGTo1kZZV/yM="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "81f55ea30fd8f819dcf93a3c934dd584c895bd2f",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "8eddc6f8d9c7c81f19fdb11f8725c4bf9801f62d",
+      "dirty": false
+    }
+  }
+}