{
  "digline_version": "0.15.0",
  "captured_at": "2026-09-17T13:40:13+00:00",
  "environment": {
    "network": "none: no provider is configured, proxies point at http://127.0.0.1:9",
    "stripped": [
      "ANTHROPIC_API_KEY",
      "OPENAI_API_KEY",
      "AWS_*"
    ]
  },
  "runtime_dependencies": {
    "count": 1,
    "names": [
      "jsonschema"
    ],
    "source": "importlib.metadata.requires('digline') in the interpreter that ran every command above: the requirements the installed digline declares, keeping those whose marker holds in that interpreter. Extras are never active, so a requirement that exists only for an extra is not counted."
  },
  "requires_python": {
    "specifier": ">=3.12",
    "source": "importlib.metadata.metadata('digline')['Requires-Python'] in the interpreter that ran every command above: the interpreter range the installed digline declares, as its own metadata carries it, not as the interpreter that happened to run this script satisfies it."
  },
  "cli_commands": {
    "items": [
      {
        "name": "run",
        "help": "execute the suite and write a run"
      },
      {
        "name": "rejudge",
        "help": "judge a stored run's recorded answers again, declaring the source"
      },
      {
        "name": "compare",
        "help": "compare a run with the baseline"
      },
      {
        "name": "diff",
        "help": "report what differs between two runs; judges neither"
      },
      {
        "name": "list",
        "help": "list stored runs, newest first"
      },
      {
        "name": "migrate",
        "help": "bring stored runs and the baseline up to the current schema"
      },
      {
        "name": "promote",
        "help": "make a run the baseline"
      },
      {
        "name": "register",
        "help": "record a person's disposition about a comparison; commit the line"
      },
      {
        "name": "view",
        "help": "browse stored runs, compare any two, promote"
      },
      {
        "name": "explain",
        "help": "read a run back at length; compares if there is a baseline"
      },
      {
        "name": "log",
        "help": "which model answered, read down the stored runs; never a gate"
      },
      {
        "name": "report",
        "help": "render the report"
      }
    ],
    "source": "digline.cli.build_parser(), the parser `digline` itself runs: every subcommand in the order it is declared, with the one-line help `digline --help` prints beside it. Read from argparse's own list of subcommands, not from a list kept for the home."
  },
  "checks": {
    "items": [
      {
        "name": "F1",
        "kind": "aggregate",
        "anchor": "f1"
      },
      {
        "name": "Accuracy",
        "kind": "aggregate",
        "anchor": "accuracy"
      },
      {
        "name": "Affix",
        "kind": "deterministic",
        "anchor": "affix"
      },
      {
        "name": "Contains",
        "kind": "deterministic",
        "anchor": "contains"
      },
      {
        "name": "CostBudget",
        "kind": "budget",
        "anchor": "costbudget"
      },
      {
        "name": "Equals",
        "kind": "deterministic",
        "anchor": "equals"
      },
      {
        "name": "Faithfulness",
        "kind": "judged",
        "anchor": "faithfulness"
      },
      {
        "name": "FromAutoevals",
        "kind": "wrapper",
        "anchor": "fromautoevals"
      },
      {
        "name": "IsJson",
        "kind": "deterministic",
        "anchor": "isjson"
      },
      {
        "name": "JsonSchema",
        "kind": "deterministic",
        "anchor": "jsonschema"
      },
      {
        "name": "LatencyBudget",
        "kind": "budget",
        "anchor": "latencybudget"
      },
      {
        "name": "Length",
        "kind": "deterministic",
        "anchor": "length"
      },
      {
        "name": "Levenshtein",
        "kind": "deterministic",
        "anchor": "levenshtein"
      },
      {
        "name": "LlmRubric",
        "kind": "judged",
        "anchor": "llmrubric"
      },
      {
        "name": "NotContains",
        "kind": "deterministic",
        "anchor": "notcontains"
      },
      {
        "name": "PiiAbsent",
        "kind": "deterministic",
        "anchor": "piiabsent"
      },
      {
        "name": "Precision",
        "kind": "aggregate",
        "anchor": "precision"
      },
      {
        "name": "Recall",
        "kind": "aggregate",
        "anchor": "recall"
      },
      {
        "name": "Regex",
        "kind": "deterministic",
        "anchor": "regex"
      },
      {
        "name": "ToolCalledWith",
        "kind": "deterministic",
        "anchor": "toolcalledwith"
      },
      {
        "name": "ToolsCalled",
        "kind": "deterministic",
        "anchor": "toolscalled"
      },
      {
        "name": "Repeated",
        "kind": "wrapper",
        "anchor": "repeated"
      }
    ],
    "source": "Every class in digline.core.__all__ that subclasses AssertionBase or RunAssertionBase, in the order of __all__, with the `KIND` the class declares and the anchor of its card in docs/metrics.md, derived from the card's heading the way the site's Markdown derives a heading id."
  },
  "scenarios": {
    "quickstart": {
      "run_ids": [
        "2026-09-17T13-40-13-029556-00-00-282b0c02d6511fb4"
      ],
      "change": null,
      "band": "absent: samples=1",
      "commands": [
        {
          "cmd": "digline run --suite support.py",
          "stdout": "2026-09-17T13-40-13-029556-00-00-282b0c02d6511fb4\n",
          "stderr": "digline: 3 cases × 1 sample = 3 calls to the target\n",
          "exit": 0
        },
        {
          "cmd": "digline promote --suite support.py --run latest",
          "stdout": "support baseline set to 2026-09-17T13-40-13-029556-00-00-282b0c02d6511fb4\n",
          "stderr": "",
          "exit": 0
        },
        {
          "cmd": "digline compare --suite support.py --run latest",
          "stdout": "Nothing got worse compared with the reference. Every case could be judged. No case is suspended. The suite is unchanged from the reference.\n",
          "stderr": "",
          "exit": 0
        }
      ]
    },
    "prompt_regression": {
      "run_ids": [
        "2026-09-17T13-40-13-412027-00-00-282b0c02d6511fb4",
        "2026-09-17T13-40-13-686329-00-00-282b0c02d6511fb4"
      ],
      "model": "The model and the judge are canned: no provider is called. The reply depends on the prompt by construction — each canned answer is signed with whatever the signature line of prompt.md asks for — and that line is the only edit made between the two runs.",
      "change": {
        "description": "One line of prompt.md, the signature line, changed.",
        "files": [
          {
            "path": "prompt.md",
            "outcome": "changed",
            "diff": [
              "--- a/prompt.md",
              "+++ b/prompt.md",
              "@@ -1,3 +1,3 @@",
              " You are the support assistant for Northwind.",
              " Answer the customer's question in one sentence.",
              "-Sign every reply as \"— Northwind Support\".",
              "+Sign every reply as \"— the Northwind team\"."
            ]
          }
        ]
      },
      "band": "absent: samples=1",
      "commands": [
        {
          "cmd": "digline run --suite support.py",
          "stdout": "2026-09-17T13-40-13-412027-00-00-282b0c02d6511fb4\n",
          "stderr": "digline: 3 cases × 1 sample = 3 calls to the target\n",
          "exit": 0
        },
        {
          "cmd": "digline promote --suite support.py --run latest",
          "stdout": "support baseline set to 2026-09-17T13-40-13-412027-00-00-282b0c02d6511fb4\n",
          "stderr": "",
          "exit": 0
        },
        {
          "cmd": "digline run --suite support.py",
          "stdout": "2026-09-17T13-40-13-686329-00-00-282b0c02d6511fb4\n",
          "stderr": "digline: 3 cases × 1 sample = 3 calls to the target\n",
          "exit": 0
        },
        {
          "cmd": "digline compare --suite support.py --run latest",
          "stdout": "6 checks got worse compared with the reference. Every case could be judged. No case is suspended. The suite is unchanged from the reference. 1 file under test changed since the reference.\n\n  prompt.md · +1 −1 lines\n\nhow-do-i-return · llm_rubric · Went from passing to failing (1.000000 → 0.400000).\nhow-do-i-return · contains · Went from passing to failing (1.000000 → 0.000000).\nis-it-waterproof · llm_rubric · Went from passing to failing (1.000000 → 0.400000).\nis-it-waterproof · contains · Went from passing to failing (1.000000 → 0.000000).\nwhere-is-my-order · llm_rubric · Went from passing to failing (1.000000 → 0.400000).\nwhere-is-my-order · contains · Went from passing to failing (1.000000 → 0.000000).\n",
          "stderr": "",
          "exit": 1
        },
        {
          "cmd": "digline compare --suite support.py --run latest --json full",
          "stderr": "",
          "exit": 1
        }
      ],
      "compare_json": {
        "source": "The stdout of `digline compare --suite support.py --run latest --json full` above, decoded with json.loads and otherwise unchanged: every key but this one is digline's.",
        "artifacts_changed": true,
        "canary_moved": false,
        "config_changed": false,
        "counts": {
          "errored": 0,
          "improved": 0,
          "missing": 0,
          "new": 0,
          "regressed": 6,
          "unchanged": 3
        },
        "deltas": [
          {
            "after": 0.814996,
            "assertion": "cost_budget",
            "before": 0.815328,
            "calibration": false,
            "canary": false,
            "case_id": "how-do-i-return",
            "delta": -0.000332,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "unchanged",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.4,
            "assertion": "llm_rubric",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "how-do-i-return",
            "delta": -0.6,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.0,
            "assertion": "contains",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "how-do-i-return",
            "delta": -1.0,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.813339,
            "assertion": "cost_budget",
            "before": 0.81367,
            "calibration": false,
            "canary": false,
            "case_id": "is-it-waterproof",
            "delta": -0.000331,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "unchanged",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.4,
            "assertion": "llm_rubric",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "is-it-waterproof",
            "delta": -0.6,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.0,
            "assertion": "contains",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "is-it-waterproof",
            "delta": -1.0,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.817327,
            "assertion": "cost_budget",
            "before": 0.817661,
            "calibration": false,
            "canary": false,
            "case_id": "where-is-my-order",
            "delta": -0.000334,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "unchanged",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.4,
            "assertion": "llm_rubric",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "where-is-my-order",
            "delta": -0.6,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          },
          {
            "after": 0.0,
            "assertion": "contains",
            "before": 1.0,
            "calibration": false,
            "canary": false,
            "case_id": "where-is-my-order",
            "delta": -1.0,
            "noise_max": null,
            "noise_min": null,
            "noise_samples": 0,
            "outcome": "regressed",
            "scope": "case",
            "within_noise": false
          }
        ],
        "exit_code": 1,
        "judge_config_changed": false,
        "judge_config_deltas": [],
        "on_the_line": 0,
        "output_version": 1,
        "reasons_available": true,
        "rejudged": false,
        "scale_lost": false,
        "sentence": "6 checks got worse compared with the reference. Every case could be judged. No case is suspended. The suite is unchanged from the reference. 1 file under test changed since the reference.",
        "shape": [
          {
            "assertion_id": "4e7ba5d3e226025c",
            "check": "llm_rubric",
            "reference": {
              "claims_unrecorded": 0,
              "extremes": 3,
              "sample_means": 0,
              "scores": 3,
              "single_claim": 0
            },
            "run": {
              "claims_unrecorded": 0,
              "extremes": 0,
              "sample_means": 0,
              "scores": 3,
              "single_claim": 0
            }
          }
        ],
        "suspended": 0,
        "target_config_changed": false,
        "target_config_deltas": [],
        "target_echoed": false,
        "unjudged": 0,
        "within_noise": 0,
        "worse": true
      }
    }
  }
}