scripts/test_unixcoder_encoder.py

"""Validate UniXcoderEncoder against the official microsoft/CodeBERT recipe.

Tests, in order of strength:
  1. GOLDEN: element-wise match vs the official unixcoder.py (fetched from GitHub)
     on single unpadded sequences — where our 1-D attention mask and upstream's
     2-D mask are provably identical, so raw pooled vectors must match to ~1e-5.
  2. ID framing: [<s>=0, <encoder-only>=6, </s>=2] + bpe + [</s>=2].
  3. Padding-invariance: batched (padded) == each item alone, to ~1e-5 (mask test).
  4. Unit-norm + dim==768 after normalize.
  5. Toy retrieval: 5 code snippets vs their NL descriptions -> MRR == 1.0, and the
     correct-pair cosine is discriminative (vs the degenerate ~0.20 of ST's default).

Exit non-zero on any failure.
"""
from __future__ import annotations

import sys
import types
import urllib.request

import numpy as np
import torch

sys.path.insert(0, "src")
from codesearch.embedding import UniXcoderEncoder  # noqa: E402

MODEL = "microsoft/unixcoder-base"
_OFFICIAL_URL = (
    "https://raw.githubusercontent.com/microsoft/CodeBERT/master/UniXcoder/unixcoder.py"
)

CODE = [
    "def add(a, b):\n    return a + b",
    "def is_even(n):\n    return n % 2 == 0",
    "def reverse(s):\n    return s[::-1]",
    "def to_upper(s):\n    return s.upper()",
    "def read_file(path):\n    with open(path) as f:\n        return f.read()",
]
NL = [
    "add two numbers",
    "check whether a number is even",
    "reverse a string",
    "convert a string to uppercase",
    "read the contents of a file",
]

_FAILS: list[str] = []


def check(cond: bool, msg: str) -> None:
    print(("  PASS  " if cond else "  FAIL  ") + msg)
    if not cond:
        _FAILS.append(msg)


def load_official():
    """Fetch official unixcoder.py, import it as a module, return the UniXcoder class."""
    src = urllib.request.urlopen(_OFFICIAL_URL, timeout=30).read().decode()
    mod = types.ModuleType("official_unixcoder")
    exec(compile(src, "official_unixcoder.py", "exec"), mod.__dict__)
    return mod.UniXcoder


def main() -> int:
    enc = UniXcoderEncoder(MODEL)

    # --- 2. ID framing -------------------------------------------------------
    ids = enc._build_ids(CODE[0])
    check(ids[:3] == [0, 6, 2] and ids[-1] == 2,
          f"ID framing [<s>,<enc-only>,</s>]...[</s>]  got {ids[:3]}...{ids[-1]}")

    # --- 1. GOLDEN cross-check vs the official recipe ------------------------
    # The as-shipped official forward() cannot run under transformers 5.x: its 2-D
    # (mask_i*mask_j) attention mask crashes on batched input and goes silently
    # causal on single input. So we validate against the official recipe run the
    # way tf5 requires: (a) its tokenizer framing must match ours id-for-id, and
    # (b) the official model config (is_decoder=True) fed a full-bidirectional 4-D
    # mask + mean-pool must match our vectors element-wise. is_decoder is a
    # mask-only flag (zero weight effect), so this is a true faithfulness check.
    try:
        from transformers import AutoConfig, AutoModel

        Official = load_official()
        off = Official(MODEL)  # only used for its .tokenize()

        # (a) framing: official ids == ours
        max_id_mismatch = 0
        for text in CODE + NL:
            off_ids = off.tokenize([text], max_length=512, mode="<encoder-only>")[0]
            max_id_mismatch = max(max_id_mismatch, int(off_ids != enc._build_ids(text)))
        check(max_id_mismatch == 0, "golden framing: official.tokenize ids == _build_ids")

        # (b) element-wise: official config (is_decoder=True) + full bidirectional
        cfg = AutoConfig.from_pretrained(MODEL)
        cfg.is_decoder = True
        ref_model = AutoModel.from_pretrained(MODEL, config=cfg)
        ref_model.eval()
        max_ediff = 0.0
        for text in CODE + NL:
            ids = torch.tensor([enc._build_ids(text)])       # single, unpadded
            full4d = torch.zeros(1, 1, ids.shape[1], ids.shape[1])  # all-zeros => full bidir
            with torch.no_grad():
                ref = ref_model(ids, attention_mask=full4d)[0][0].mean(0)
            ref_vec = ref.numpy().astype(np.float32)
            mine_raw = enc.encode(text, normalize_embeddings=False)
            max_ediff = max(max_ediff, float(np.abs(mine_raw - ref_vec).max()))
        check(max_ediff < 1e-4,
              f"golden element-wise match vs official bidirectional recipe: max|Δ|={max_ediff:.2e} (<1e-4)")
    except Exception as e:  # network / upstream unavailable
        print(f"  SKIP  golden cross-check unavailable: {type(e).__name__}: {e}")

    # --- 3. Padding-invariance (batched vs single) ---------------------------
    batched = enc.encode(CODE, normalize_embeddings=True)
    singles = np.vstack([enc.encode(c, normalize_embeddings=True) for c in CODE])
    pad_diff = float(np.abs(batched - singles).max())
    check(pad_diff < 1e-5, f"padding-invariance: max|Δ|={pad_diff:.2e} (<1e-5)")

    # --- 4. Unit norm + dim --------------------------------------------------
    norms = np.linalg.norm(batched, axis=1)
    check(batched.shape[1] == 768, f"dim==768  got {batched.shape[1]}")
    check(np.allclose(norms, 1.0, atol=1e-4), f"unit-norm rows  norms in [{norms.min():.4f},{norms.max():.4f}]")

    # --- 5. Toy retrieval sanity (MRR) --------------------------------------
    code_emb = enc.encode(CODE, normalize_embeddings=True)
    nl_emb = enc.encode(NL, normalize_embeddings=True)
    sims = nl_emb @ code_emb.T                     # (query, doc) cosine
    ranks = []
    for i in range(len(NL)):
        order = np.argsort(-sims[i])
        ranks.append(int(np.where(order == i)[0][0]) + 1)
    mrr = float(np.mean([1.0 / r for r in ranks]))
    diag = float(np.mean(np.diag(sims)))
    check(mrr == 1.0, f"toy retrieval MRR==1.0  ranks={ranks}  mrr={mrr:.3f}")
    check(diag > 0.30, f"correct-pair cosine discriminative: mean diag={diag:.3f} (>>0.20 degenerate)")

    print("\n" + ("ALL PASS" if not _FAILS else f"{len(_FAILS)} FAILURE(S): " + "; ".join(_FAILS)))
    return 1 if _FAILS else 0


if __name__ == "__main__":
    sys.exit(main())