{"$schema":"https://verifymcp.io/schemas/trust-report.json","schemaVersion":1,"license":{"name":"CC BY-NC 4.0","url":"https://creativecommons.org/licenses/by-nc/4.0/"},"generatedAt":"2026-11-05T17:25:16.702Z","url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp","markdownUrl":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp.md","server":{"slug":"diegoperezg7-evalsmith-mcp","name":"io.github.diegoperezg7/evalsmith-mcp","registryName":"io.github.diegoperezg7/evalsmith-mcp","description":"Evaluate RAG retrieval, agent tool calls and prompt regressions with deterministic metrics.","registryStatus":"active","deletedAt":null,"statusMessage":null,"liveness":"live","publisher":{"class":"unknown","claimed":false}},"component":{"slug":"evalsmith-mcp","type":"npm","identifier":"evalsmith-mcp","withdrawn":false,"url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp","version":"0.1.0","transport":"stdio","scoredAt":"2026-11-05T06:37:21.729Z"},"score":{"total":71,"tier":"medium","status":"scored","weekDelta":0,"history":[{"date":"2026-11-05","total":71},{"date":"2026-11-04","total":71},{"date":"2026-11-03","total":71},{"date":"2026-11-02","total":71},{"date":"2026-11-01","total":71},{"date":"2026-10-31","total":71},{"date":"2026-10-30","total":71},{"date":"2026-10-29","total":71},{"date":"2026-10-28","total":56}]},"categories":[{"name":"Supply Chain Security","score":98,"verdict":"partial","reasons":["No malware found by supply-chain analysis.","No known CVEs affecting this package version or its production dependencies.","No install/post-install scripts declared.","34 of 107 dependencies flagged as unhealthy."],"codes":[{"code":"supplychain.malware_clean","args":{}},{"code":"cve.none","args":{}},{"code":"installscript.none","args":{}},{"code":"dephealth.assessed","args":{"deps":"107","unhealthy":"34"}}]},{"name":"Provenance & Transparency","score":48,"verdict":"fail","reasons":["Source repository is publicly reachable at the declared URL.","Provenance check failed: no build-provenance attestation is published.","Clear OSI-approved license (MIT).","Actively maintained (last published 7 days ago).","Publishes a security disclosure policy (SECURITY.md)."],"codes":[{"code":"provenance.repo_verified","args":{}},{"code":"provenance.none","args":{}},{"code":"license.osi","args":{"spdx":"MIT"}},{"code":"maintenance.recent","args":{"age_days":"7"}},{"code":"disclosure.present","args":{}}]},{"name":"Schema Quality & AI Usability","score":81,"verdict":"fail","reasons":["100% of prompts and resources have a non-trivial description (not blank, and not just the item's name).","AI-judged instruction clarity (good).","Tool/resource definitions use about 1308 tokens (~72/item across 18 items; 5 tools + 13 resources), lean.","Usage-examples check failed: none of the tools include examples."],"codes":[{"code":"schema.descriptions","args":{"pct":"100"}},{"code":"schema.ai_quality","args":{"band":"good"}},{"code":"schema.footprint_lean","args":{"avg":"72","items":"18","tools":"5","tokens":"1308","resources":"13"}},{"code":"schema.examples_absent","args":{}}]},{"name":"Stability & Change Management","score":0,"verdict":"unverified","reasons":["Stability not yet verified: not enough scan history yet (needs a 30-day window)."],"codes":[{"code":"stability.insufficient_history","args":{}}]},{"name":"Tool Coverage","score":97,"verdict":"partial","reasons":["100% of tools have a non-trivial description (not blank, and not just the tool's name).","91% of tool parameters carry a description.","Structured output schemas are declared (100% of tools); any adoption earns full credit."],"codes":[{"code":"toolquality.descriptions","args":{"pct":"100"}},{"code":"toolquality.param_docs","args":{"pct":"91"}},{"code":"toolquality.output_present","args":{"pct":"100"}}]},{"name":"Tool Safety","score":100,"verdict":"pass","reasons":["No prompt-injection markers were found in the server instructions, tool names or descriptions we captured.","We read all 5 captured tool definition(s), and no name or description among them implies an irreversible operation.","An AI judge read all 7 captured unit(s) of tool text and found none that tries to manipulate the model reading it."],"codes":[{"code":"toolsafety.injection_clean","args":{}},{"code":"toolsafety.destructive_none","args":{"tools":"5"}},{"code":"toolsafety.manipulation_clean","args":{"units":"7","judged":"7"}}]},{"name":"Capabilities","score":100,"verdict":"pass","reasons":["Implements a supported MCP spec version (2025-11-25); the latest is 2026-07-28."],"codes":[{"code":"protocol.supported","args":{"latest":"2026-07-28","version":"2025-11-25"}}]}],"inconclusive":["Stability not yet verified: not enough scan history yet (needs a 30-day window)."],"malware":[],"components":[{"slug":"evalsmith-mcp","type":"npm","identifier":"evalsmith-mcp","withdrawn":false,"score":71,"current":true,"url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp","markdownUrl":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp.md"}],"facts":[{"label":"Registry","value":"npm"},{"label":"Package","value":"evalsmith-mcp"},{"label":"Version","value":"0.1.0"},{"label":"Transport","value":"stdio"}],"install":{"available":true,"bundleSha256":null},"tools":[{"name":"evaluate_retrieval","title":"Evaluate retrieval","description":"Score a ranked retrieval run against relevance judgements (qrels). Returns recall@k, precision@k, nDCG@k (graded, linear gain), hit rate@k and MRR, averaged over judged queries, plus per-query values. Deterministic; no model calls.","inputs":[{"name":"k","type":"array","required":false,"description":"Cut-offs to evaluate. Defaults to [1, 3, 5, 10]."},{"name":"qrels","type":"object","required":true,"description":"Relevance judgements: { queryId: { docId: grade } }. Grade 0 means not relevant; higher grades are more relevant (used by nDCG). Example: { \"q1\": { \"d3\": 2, \"d7\": 1 } }"},{"name":"run","type":"object","required":true,"description":"Ranked retrieval output: { queryId: [docId, ...] }, best match first. Example: { \"q1\": [\"d7\", \"d3\", \"d9\"] }"}],"outputs":[{"name":"metrics","type":"object","required":true,"description":"Metric name to value, e.g. { \"recall@5\": 0.83, \"mrr\": 0.71 }"},{"name":"missingFromRun","type":"array","required":true,"description":"Judged queries absent from the run (scored as an empty ranking)."},{"name":"perQuery","type":"array","required":true,"description":null},{"name":"queries","type":"integer","required":true,"description":"Queries with at least one relevant document."},{"name":"skippedQueries","type":"array","required":true,"description":"Queries in qrels with no relevant documents (excluded from averages)."}],"hasOutputSchema":true,"hasExamples":false,"tokenEstimate":204},{"name":"compare_configs","title":"Compare retrieval configurations","description":"Compare 2-3 retrieval configurations on the same qrels. Ranks them on a primary metric, attaches seeded paired-bootstrap confidence intervals, and says whether the winner's lead over the runner-up is significant (interval of the gap excludes zero).","inputs":[{"name":"bootstrap","type":"object","required":false,"description":"Paired bootstrap over queries. Defaults: 1000 samples, seed 42, 95% confidence."},{"name":"configs","type":"array","required":true,"description":"Two or three runs over the same queries."},{"name":"k","type":"array","required":false,"description":"Cut-offs to evaluate. Defaults to [1, 3, 5, 10]."},{"name":"metric","type":"string","required":false,"description":"Primary metric that decides the winner, e.g. ndcg@10, recall@5, mrr."},{"name":"qrels","type":"object","required":true,"description":"Relevance judgements: { queryId: { docId: grade } }. Grade 0 means not relevant; higher grades are more relevant (used by nDCG). Example: { \"q1\": { \"d3\": 2, \"d7\": 1 } }"}],"outputs":[{"name":"bootstrap","type":"object","required":true,"description":null},{"name":"configs","type":"array","required":true,"description":null},{"name":"delta","type":"object","required":true,"description":"Paired bootstrap interval of winner minus runner-up."},{"name":"metric","type":"string","required":true,"description":null},{"name":"runnerUp","type":"string","required":true,"description":null},{"name":"significant","type":"boolean","required":true,"description":"True when the delta interval excludes zero."},{"name":"summary","type":"string","required":true,"description":null},{"name":"winner","type":"string","required":true,"description":null}],"hasOutputSchema":true,"hasExamples":false,"tokenEstimate":218},{"name":"check_tool_calls","title":"Check agent tool calls","description":"Compare an agent's tool-call trace with the expected calls: right tool, right order (strict or lax) and arguments that satisfy exact / regex / subset / any matchers. Returns a pass flag, a score and a reason for every mismatch.","inputs":[{"name":"expected","type":"array","required":true,"description":"Expected calls. Argument matchers: {\"type\":\"exact\",\"value\":...}, {\"type\":\"regex\",\"pattern\":\"...\"}, {\"type\":\"subset\",\"value\":{...}}, {\"type\":\"any\"}."},{"name":"order","type":"string","required":false,"description":"strict: the trace must be exactly the expected calls, in order. lax: every expected call must appear somewhere; order and extra calls are allowed."},{"name":"trace","type":"array","required":true,"description":"Tool calls the agent made, in order."}],"outputs":[{"name":"order","type":"string","required":true,"description":"strict: the trace must be exactly the expected calls, in order. lax: every expected call must appear somewhere; order and extra calls are allowed."},{"name":"passed","type":"boolean","required":true,"description":null},{"name":"score","type":"number","required":true,"description":"Matched expected calls / expected calls."},{"name":"steps","type":"array","required":true,"description":null},{"name":"summary","type":"string","required":true,"description":null},{"name":"unexpected","type":"array","required":true,"description":null}],"hasOutputSchema":true,"hasExamples":false,"tokenEstimate":157},{"name":"detect_prompt_regression","title":"Detect prompt regression","description":"Run a golden set's deterministic checks (exact, regex, contains, JSON Schema, length) against baseline and candidate outputs. A regression is a case that passes on baseline and fails on candidate. An LLM judge can be enabled per call for a second opinion; the verdict only depends on the checks.","inputs":[{"name":"baseline","type":"object","required":true,"description":"Outputs of the current prompt/model, keyed by case id."},{"name":"candidate","type":"object","required":true,"description":"Outputs of the proposed prompt/model, keyed by case id."},{"name":"golden","type":"array","required":true,"description":"Golden cases with deterministic checks."},{"name":"judge","type":"object","required":false,"description":"Optional pairwise LLM judge on the server's configured OpenAI-compatible endpoint (e.g. Ollama). Off by default; it never changes the verdict."}],"outputs":[{"name":"baselinePassRate","type":"number","required":true,"description":null},{"name":"candidatePassRate","type":"number","required":true,"description":null},{"name":"cases","type":"integer","required":true,"description":null},{"name":"improvements","type":"array","required":true,"description":"Cases that fail on baseline and pass on candidate."},{"name":"judge","type":"object","required":false,"description":null},{"name":"perCase","type":"array","required":true,"description":null},{"name":"regressions","type":"array","required":true,"description":"Cases that pass on baseline and fail on candidate."},{"name":"summary","type":"string","required":true,"description":null},{"name":"verdict","type":"string","required":true,"description":null}],"hasOutputSchema":true,"hasExamples":false,"tokenEstimate":157},{"name":"generate_report","title":"Generate report","description":"Turn the results of the other evalsmith tools into a Markdown summary, a JSON document or JUnit XML for CI dashboards. Optional gates (metric -> minimum) decide the overall pass/fail.","inputs":[{"name":"comparison","type":"object","required":false,"description":"structuredContent of compare_configs."},{"name":"format","type":"string","required":false,"description":null},{"name":"gates","type":"object","required":false,"description":"Minimum values, e.g. { \"recall@5\": 0.8, \"tool_calls\": 1 }."},{"name":"name","type":"string","required":false,"description":null},{"name":"regression","type":"object","required":false,"description":"structuredContent of detect_prompt_regression."},{"name":"retrieval","type":"object","required":false,"description":"structuredContent of evaluate_retrieval."},{"name":"toolCalls","type":"object","required":false,"description":"structuredContent of check_tool_calls."}],"outputs":[{"name":"content","type":"string","required":true,"description":null},{"name":"format","type":"string","required":true,"description":null},{"name":"passed","type":"boolean","required":true,"description":null}],"hasOutputSchema":true,"hasExamples":false,"tokenEstimate":147}],"toolsTruncatedAtCapture":false,"diagnostics":{"capturedAt":"2026-11-05T06:37:21.000Z","track":"package","endpoint":null,"declaredUrl":null,"truncated":false,"tls":null,"dnssec":null,"auth":null,"httpsRedirect":null,"transports":[],"provenance":{"outcome":"none","label":"No attestation","reason":null,"reasonLabel":null,"discovery":null,"discoveryLabel":null,"sourceRepo":null,"attestationSkipped":null,"attestationSkippedLabel":null},"installScripts":[],"installScriptsTotal":null,"vulnerabilities":[],"vulnerabilitiesTotal":null,"dependencies":{"nodes":107,"partial":false,"deprecated":null,"stale":34,"noRepo":null}},"mismatches":[],"changelog":{"trackingSince":"2026-10-28","recent":[{"id":"chg_01a18642-b1fb-7706-84f6-251d58f1da4b","kind":"check.reverified","summary":"Malware scan (unverified → pass)","materiality":"security","direction":"improvement","scoreDelta":null,"occurredOn":"2026-10-29","url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp#chg-chg_01a18642-b1fb-7706-84f6-251d58f1da4b"},{"id":"chg_01a18642-b1fc-72a2-8be1-e208b91ea84f","kind":"score.total_changed","summary":"Trust score (56 → 71)","materiality":"functional","direction":"improvement","scoreDelta":15,"occurredOn":"2026-10-29","url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp#chg-chg_01a18642-b1fc-72a2-8be1-e208b91ea84f"},{"id":"chg_01a1821f-7e89-7ab7-9677-5ab01b292c3b","kind":"score.first_scored","summary":"First indexed and scored","materiality":"functional","direction":"neutral","scoreDelta":null,"occurredOn":"2026-10-28","url":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp#chg-chg_01a1821f-7e89-7ab7-9677-5ab01b292c3b"}],"truncated":false,"feedUrl":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp/changelog.json"},"links":{"html":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp","markdown":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp.md","changelogJson":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp/changelog.json","changelogRss":"https://verifymcp.io/servers/diegoperezg7-evalsmith-mcp/evalsmith-mcp.xml","external":[{"label":"npm package","href":"https://www.npmjs.com/package/evalsmith-mcp"},{"label":"Socket report","href":"https://socket.dev/npm/package/evalsmith-mcp"},{"label":"Repository","href":"https://github.com/diegoperezg7/evalsmith-mcp"}]}}