diff --git a/docs/assets/images/blogs/chennai-healthcare.svg b/docs/assets/images/blogs/chennai-healthcare.svg new file mode 100644 index 00000000..7c153d4f --- /dev/null +++ b/docs/assets/images/blogs/chennai-healthcare.svg @@ -0,0 +1,10 @@ + diff --git a/docs/assets/images/blogs/digital-twins.svg b/docs/assets/images/blogs/digital-twins.svg new file mode 100644 index 00000000..819a8ee2 --- /dev/null +++ b/docs/assets/images/blogs/digital-twins.svg @@ -0,0 +1,10 @@ + diff --git a/docs/assets/images/blogs/explanation-fidelity.svg b/docs/assets/images/blogs/explanation-fidelity.svg new file mode 100644 index 00000000..9887924b --- /dev/null +++ b/docs/assets/images/blogs/explanation-fidelity.svg @@ -0,0 +1,9 @@ + diff --git a/docs/assets/images/blogs/image-editing.svg b/docs/assets/images/blogs/image-editing.svg new file mode 100644 index 00000000..74d3ed33 --- /dev/null +++ b/docs/assets/images/blogs/image-editing.svg @@ -0,0 +1,12 @@ + diff --git a/docs/assets/images/blogs/llm-responses.svg b/docs/assets/images/blogs/llm-responses.svg new file mode 100644 index 00000000..4c423ac5 --- /dev/null +++ b/docs/assets/images/blogs/llm-responses.svg @@ -0,0 +1,10 @@ + diff --git a/docs/assets/images/blogs/local-explanations.svg b/docs/assets/images/blogs/local-explanations.svg new file mode 100644 index 00000000..1852c285 --- /dev/null +++ b/docs/assets/images/blogs/local-explanations.svg @@ -0,0 +1,10 @@ + diff --git a/docs/assets/images/blogs/skin-cancer.svg b/docs/assets/images/blogs/skin-cancer.svg new file mode 100644 index 00000000..cd7bc5b6 --- /dev/null +++ b/docs/assets/images/blogs/skin-cancer.svg @@ -0,0 +1,10 @@ + diff --git a/docs/assets/images/blogs/smile-workflow.svg b/docs/assets/images/blogs/smile-workflow.svg new file mode 100644 index 00000000..b902a051 --- /dev/null +++ b/docs/assets/images/blogs/smile-workflow.svg @@ -0,0 +1,12 @@ + diff --git a/docs/assets/stylesheets/extra.css b/docs/assets/stylesheets/extra.css index c2a4c37a..f0b52c76 100644 --- a/docs/assets/stylesheets/extra.css +++ b/docs/assets/stylesheets/extra.css @@ -304,3 +304,195 @@ display: block; margin: 0 auto; } + +/* Research blog index: the cards respond to the space left by the site navigation. */ +.md-main__inner:has(.xwhy-blog-index) { + max-width: 94rem; +} + +.md-typeset .xwhy-blog-index { + container-type: inline-size; + padding-bottom: 1.5rem; +} + +.md-typeset .xwhy-blog-intro { + margin: 0 0 1.35rem; +} + +.md-typeset .xwhy-blog-intro__rule { + background: #2861b2; + display: block; + height: 0.17rem; + margin-bottom: 0.55rem; + width: 2.4rem; +} + +.md-typeset .xwhy-blog-intro h1 { + color: var(--md-default-fg-color); + font-size: clamp(1.65rem, 4vw, 2.3rem); + font-weight: 760; + letter-spacing: -0.025em; + line-height: 1.12; + margin: 0 0 0.35rem; +} + +.md-typeset .xwhy-blog-intro p { + color: var(--md-default-fg-color--light); + font-size: 0.85rem; + margin: 0; + text-align: left; +} + +.md-typeset .xwhy-blog-section { + margin: 1.4rem 0 1.8rem; +} + +.md-typeset .xwhy-blog-section__heading { + align-items: baseline; + display: flex; + flex-wrap: wrap; + gap: 0.3rem 1rem; + justify-content: space-between; + margin-bottom: 0.75rem; +} + +.md-typeset .xwhy-blog-section__heading h2 { + border: 0; + color: var(--md-default-fg-color); + font-size: 1.16rem; + font-weight: 720; + margin: 0; +} + +.md-typeset .xwhy-blog-section__heading > a { + font-size: 0.72rem; +} + +.md-typeset .xwhy-blog-grid { + display: grid; + gap: 0.75rem; + grid-template-columns: repeat(4, minmax(0, 1fr)); +} + +.md-typeset .xwhy-blog-card { + background: var(--md-default-bg-color); + border: 1px solid var(--md-default-fg-color--lightest); + border-bottom: 0.26rem solid var(--blog-card-accent); + border-radius: 0.38rem; + box-shadow: 0 0.2rem 0.7rem rgba(15, 23, 42, 0.11); + margin: 0; + min-width: 0; + overflow: hidden; + transition: box-shadow 160ms ease, transform 160ms ease; +} + +.md-typeset .xwhy-blog-card:hover, +.md-typeset .xwhy-blog-card:focus-within { + box-shadow: 0 0.42rem 1.1rem rgba(15, 23, 42, 0.18); + transform: translateY(-0.12rem); +} + +.md-typeset .xwhy-blog-card--blue { --blog-card-accent: #2861b2; } +.md-typeset .xwhy-blog-card--teal { --blog-card-accent: #008991; } +.md-typeset .xwhy-blog-card--violet { --blog-card-accent: #6241a9; } +.md-typeset .xwhy-blog-card--purple { --blog-card-accent: #663796; } + +.md-typeset .xwhy-blog-card > a { + color: var(--md-default-fg-color); + display: flex; + flex-direction: column; + height: 100%; + text-decoration: none !important; +} + +.md-typeset .xwhy-blog-card > a:focus-visible { + outline: 3px solid #2861b2; + outline-offset: -3px; +} + +.md-typeset .xwhy-blog-card img { + aspect-ratio: 3 / 1; + display: block; + object-fit: cover; + width: 100%; +} + +.md-typeset .xwhy-blog-card__body { + display: flex; + flex: 1; + flex-direction: column; + gap: 0.3rem; + padding: 0.65rem 0.75rem 0.75rem; +} + +.md-typeset .xwhy-blog-card__category { + color: var(--blog-card-accent); + font-size: 0.64rem; + font-weight: 750; + line-height: 1.2; +} + +.md-typeset .xwhy-blog-card__title { + color: var(--md-default-fg-color); + font-size: 0.81rem; + font-weight: 750; + line-height: 1.27; +} + +.md-typeset .xwhy-blog-card__description { + color: var(--md-default-fg-color--light); + font-size: 0.7rem; + line-height: 1.38; + text-align: left; +} + +.md-typeset .xwhy-blog-card__read { + color: var(--blog-card-accent); + font-size: 0.67rem; + font-weight: 700; + margin-top: auto; + padding-top: 0.38rem; +} + +.md-typeset .xwhy-blog-footer { + border-top: 1px solid var(--md-default-fg-color--lightest); + color: var(--md-default-fg-color--light); + padding-top: 0.8rem; + text-align: left; +} + +.md-typeset .xwhy-blog-back { + display: inline-block; + font-size: 0.78rem; + margin-bottom: 0.8rem; +} + +@media screen and (max-width: 74em) { + .md-typeset .xwhy-blog-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media screen and (max-width: 40em) { + .md-typeset .xwhy-blog-grid { + grid-template-columns: minmax(0, 1fr); + } +} + +@container (max-width: 49rem) { + .md-typeset .xwhy-blog-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@container (max-width: 29rem) { + .md-typeset .xwhy-blog-grid { + grid-template-columns: minmax(0, 1fr); + } +} + +@media (prefers-reduced-motion: reduce) { + .md-typeset .xwhy-blog-card { + transition: none; + } +} diff --git a/docs/research/reddit/blogs.md b/docs/research/reddit/blogs.md index f9b0a8a7..c549d012 100644 --- a/docs/research/reddit/blogs.md +++ b/docs/research/reddit/blogs.md @@ -1,26 +1,79 @@ --- -title: Research Blogs and Explainers -description: Published SMILE and responsible-AI blog posts, plus XWhy guides about local explanations, LLMs, and evaluating attribution. +title: Research & Insights +description: Research stories and practical XWhy explainers, with every featured article available directly on this site. +hide: + - toc --- -# Research blogs +
+
+ +

Research & Insights

+

Explore our research articles, explainers and practical resources.

+
-Explore published articles from the University of Hull research group and longer XWhy explainers. For research citations, use the [publication catalogue](../publications.md). +
+
+

Research group stories

+ Browse the original Hull research blog +
+
+ + + + +
+
-## Published research group blog posts +
+

XWhy explainers and practical reading

+
+ + + + +
+
-- **[Mapping the Mind of an Instruction-Based Image Editing Using SMILE](https://www.responsibleaihull.com/post/mapping-the-mind-of-an-instruction-based-image-editing-using-smile)** — Koorosh Aslansefat explains how changes to words in an editing instruction can be connected to changes in the generated image. -- **[Pint of Science: Can we trust AI for skin cancer? Are they fair?](https://www.responsibleaihull.com/post/pint-of-science-trustworthy-ai-can-we-trust-ai-for-skin-cancer-are-they-fair)** — A public discussion of fairness and reliability in medical image classification, featuring Koorosh Aslansefat and Kuniko Paxton. -- **[Research Visits of Responsible AI Team to Chennai, India](https://www.responsibleaihull.com/post/research-visits-of-responsible-ai-team-to-chennai-india)** — A report on research exchanges about trustworthy AI in healthcare. -- **[Responsible Digital Twin Research Theme in Chennai, India](https://www.responsibleaihull.com/post/responsible-digital-twin-research-theme-in-chennai-india)** — A related research visit on digital twins and dependable intelligent systems. - -[Browse all posts from the Centre for Responsible AI at Hull](https://www.responsibleaihull.com/research-blog). - -## XWhy explainers and practical reading - -- **[How SMILE works](../../concepts/smile.md)** — The perturbation, weighting, and local surrogate workflow behind an explanation. -- **[What a local explanation tells you](../../concepts/local-explanations.md)** — How to interpret results for a particular input and model output. -- **[Explaining LLM responses](../../explainers/llm/index.md)** — Prompt perturbations and token contributions in language-model workflows. -- **[Image generation and editing examples](../../explainers/image-generation/image-generation-editing-examples.md)** — Examples and scope for visual generation workflows. -- **[Evaluating explanation fidelity](../../evaluation/attribution-fidelity.md)** — Ways to test whether an attribution reflects the model's behaviour. -- **[Limitations and responsible interpretation](../../concepts/limitations.md)** — Where local explanations need additional checks. +

Looking for academic citations? Browse the publication catalogue.

+
diff --git a/docs/research/reddit/blogs/digital-twins-chennai.md b/docs/research/reddit/blogs/digital-twins-chennai.md new file mode 100644 index 00000000..2b104da0 --- /dev/null +++ b/docs/research/reddit/blogs/digital-twins-chennai.md @@ -0,0 +1,28 @@ +--- +title: Digital Twin Research in Chennai +description: Research connections between digital twins, dependable intelligent systems and wind energy. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Digital twin research in Chennai + +![An original illustration of a city and its digital twin](../../../assets/images/blogs/digital-twins.svg) + +*Digital twins · Research note based on [the Hull research group visit report](https://www.responsibleaihull.com/post/responsible-digital-twin-research-theme-in-chennai-india), which credits Xinhui Ma as blog author.* + +A digital twin connects observations of a physical system with a model that can help people understand and test its behaviour. For a wind turbine, such a system might bring together sensor readings, an engineering model and predictions about operating conditions. The usefulness of the twin depends on whether its model remains dependable when the physical system or its environment changes. + +## A research exchange across disciplines + +During a University of Hull visit to IIT Madras in October 2024, the research team discussed digital twins, wind energy, intelligent systems and simulation. Topics included physics-based and machine-learning predictions, sensor development, operations and maintenance. The group also visited facilities at the National Technology Centre for Ports, Waterways and Coasts. + +The meeting linked researchers working on physical infrastructure, AI and engineering. A digital twin used for operational decisions needs more than a good forecast: it also needs clear assumptions, an account of uncertainty and checks that its outputs continue to match observations. + +## Where explanations can help + +When a prediction changes, engineers need to know whether a new sensor reading, a change in operating conditions or a modelling assumption drove it. A local explanation can help investigate a particular output, provided its fidelity and stability are checked. It is one form of evidence alongside engineering constraints and direct measurements. + +The visit was an exchange about research directions, not a report of a newly deployed digital twin or a measured improvement in turbine performance. + +**Original report:** [Responsible Digital Twin Research Theme in Chennai, India](https://www.responsibleaihull.com/post/responsible-digital-twin-research-theme-in-chennai-india). This page is an original XWhy summary of the visit and its research context. diff --git a/docs/research/reddit/blogs/explaining-llm-responses.md b/docs/research/reddit/blogs/explaining-llm-responses.md new file mode 100644 index 00000000..4323de44 --- /dev/null +++ b/docs/research/reddit/blogs/explaining-llm-responses.md @@ -0,0 +1,28 @@ +--- +title: Explaining LLM Responses +description: What token contributions from a prompt can and cannot tell us about an LLM response. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Explaining LLM responses + +![An original illustration of prompt tokens and their contribution levels](../../../assets/images/blogs/llm-responses.svg) + +*LLM explainability · XWhy explainer* + +Imagine asking a language model to summarise a report. If its answer changes when a single condition is removed from the prompt, that condition deserves attention. But which prompt terms mattered most to the response, and how reliably can we estimate their influence? + +## Perturb the prompt carefully + +XWhy's `LLMExplainer` changes selected parts of a prompt and measures how those changed prompts relate to a reference response. It fits a local model to produce estimated token contributions to its response-alignment score. This is a practical way to explore sensitivity around the chosen prompt. + +For instance, an explanation might show that a condition such as “include only 2025 figures” is influential. A useful follow-up is to remove or rephrase that condition and compare the observed behaviour. The contribution alone does not tell us what computation happened inside the model. + +## Match the claim to the measurement + +The current documented workflow queries the provider for the original response and uses a semantic-distance measure in the local comparison. Its token contributions therefore describe the *configured response-alignment approximation* for the original prompt, rather than a complete causal account of how the model generated every output token. A prompt heatmap is not a view into private chain-of-thought. + +Language model outputs can vary across runs and providers. Report the provider, model version, prompt, embedding and distance settings, perturbation choices and local fit. If a finding is used in an assessment, check whether it survives a few sensible paraphrases and whether the model's observed answer changes in the expected way. + +The [LLM explainer guide](../../../explainers/llm/index.md) and [worked LLM example](../../../llm_explainer.md) describe the available interface and interpretation boundary. The related [gSMILE research preprint](https://arxiv.org/abs/2505.21657) provides additional background. diff --git a/docs/research/reddit/blogs/fairness-skin-cancer-ai.md b/docs/research/reddit/blogs/fairness-skin-cancer-ai.md new file mode 100644 index 00000000..b41c74ca --- /dev/null +++ b/docs/research/reddit/blogs/fairness-skin-cancer-ai.md @@ -0,0 +1,28 @@ +--- +title: Can We Trust AI for Skin Cancer? +description: Why fairness, reliability and human oversight matter when AI is used with skin images. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Can we trust AI for skin cancer? + +![An original illustration of a medical image review screen](../../../assets/images/blogs/skin-cancer.svg) + +*Responsible AI · Research note based on [the Hull Pint of Science event report](https://www.responsibleaihull.com/post/pint-of-science-trustworthy-ai-can-we-trust-ai-for-skin-cancer-are-they-fair) by Dhaval Thakker.* + +At a Hull Pint of Science event in May 2024, Kuniko Paxton and Koorosh Aslansefat discussed fairness and reliability in AI systems for skin cancer detection. Their public discussion raises a question that applies well beyond this one task: who benefits from a system's reported accuracy, and who might face a greater risk of error? + +## Accuracy needs context + +An overall test score can conceal differences across skin tones, image capture conditions and patient groups. A careful evaluation should report performance across relevant groups and examine the data behind those results. Group definitions, sample sizes and clinical context matter: a small or unrepresentative test set cannot establish that a system works equally well for everyone. + +Explanations can help investigators ask whether a model is responding to plausible image regions or irrelevant cues. They cannot, by themselves, establish clinical validity or fairness. A visually attractive heatmap might still be unstable or fail to reflect what drives the prediction. + +## What responsible use requires + +Before clinical use, an AI system needs evaluation with data that reflects the intended patients and setting, an account of uncertainty and failure modes, and clear human responsibility for decisions. Testing should include cases where image quality is poor or a case differs from the development data. If an explanation changes sharply after a harmless change to the image, that is a reason to investigate, not a basis for greater confidence. + +XWhy's [guidance on the limits of local explanations](../../../concepts/limitations.md) describes some of the checks needed when explanations are used in high-impact settings. + +**Original event report:** [Pint of Science! Trustworthy AI: Can we trust AI for skin cancer? Are they fair?](https://www.responsibleaihull.com/post/pint-of-science-trustworthy-ai-can-we-trust-ai-for-skin-cancer-are-they-fair). This page is an original XWhy summary of the themes, not a clinical recommendation. diff --git a/docs/research/reddit/blogs/how-smile-works.md b/docs/research/reddit/blogs/how-smile-works.md new file mode 100644 index 00000000..70719e5d --- /dev/null +++ b/docs/research/reddit/blogs/how-smile-works.md @@ -0,0 +1,31 @@ +--- +title: How SMILE Works +description: A readable introduction to SMILE's perturbations, local models and explanation checks. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# How SMILE works + +![An original illustration of the SMILE explanation workflow](../../../assets/images/blogs/smile-workflow.svg) + +*SMILE · XWhy explainer* + +Suppose an image classifier labels one picture as a dog. You can ask whether its decision depended on the animal, the background, or some other part of the image. SMILE starts with this *one* input and investigates what the model does when meaningful parts of it are changed nearby. + +## The explanation in four steps + +1. **Create controlled variations.** Change selected image regions, words or other features while retaining enough of the original example to study its neighbourhood. +2. **Observe the model.** Run those variations through the model and record how the relevant output changes. +3. **Measure closeness and fit a local model.** Weight the observations according to a suitable distance or similarity measure, then fit a simpler model around the selected input. +4. **Inspect contributions.** The local model estimates which components are associated with the observed changes. Its accuracy in approximating the collected outputs gives an initial indication of how much to trust those estimates. + +SMILE uses statistical distance measures in its weighting and comparison steps. The suitable measure depends on whether the outputs are class scores, generated images or text. A distance appropriate for one task should not be assumed to work for another. + +## A useful result has limits + +If a highlighted region changes an image classification under several sensible perturbations, that is stronger evidence about the model's local behaviour than a single coloured heatmap. Yet an explanation of one image does not describe all images. The local model can also fit badly or change when the perturbation scheme changes. + +When the result matters, report the model and data version, the chosen input, how variants were produced, the distance and surrogate settings, and the quality of the local fit. Repeat the analysis with small input changes and alternative perturbation settings. + +Continue with the [technical explanation of SMILE](../../../concepts/smile.md) or the [foundational paper](https://doi.org/10.1109/MS.2023.3321282). diff --git a/docs/research/reddit/blogs/image-editing-smile.md b/docs/research/reddit/blogs/image-editing-smile.md new file mode 100644 index 00000000..e4e0d475 --- /dev/null +++ b/docs/research/reddit/blogs/image-editing-smile.md @@ -0,0 +1,28 @@ +--- +title: Mapping the Mind of Image Editing with SMILE +description: How changing a few words in an image-editing instruction can help explain the image a model produces. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Mapping the mind of image editing with SMILE + +![An original illustration of two image regions connected by an editing arrow](../../../assets/images/blogs/image-editing.svg) + +*Explainable AI · Research note based on a [2025 Hull research group article](https://www.responsibleaihull.com/post/mapping-the-mind-of-an-instruction-based-image-editing-using-smile) by Koorosh Aslansefat.* + +An image-editing model can follow an instruction such as “replace the cloudy sky with a clear blue sky,” but a convincing result does not tell us how strongly each part of the instruction shaped the edit. That question matters when a visual change could hide something important, particularly in applications involving medical or safety-related images. + +## Change the instruction and watch the output + +The image-editing SMILE approach makes controlled changes to the text instruction. It removes or changes selected words, runs the editor again, and compares each result with the reference edit. A local model then estimates which parts of the instruction are associated with the largest changes. The explanation can be shown as contributions for individual terms rather than a single score for the whole prompt. + +For example, removing “blue” might change the colour of the generated sky while removing “replace” might change whether an edit happens at all. Those are illustrative possibilities, not measured results from a particular model. The actual conclusion depends on the model, input image, perturbations and way the outputs are compared. + +## Why the checks matter + +A plausible heatmap is only a starting point. Repeat the edit under controlled settings, assess whether the local model fits the observed outputs, and check whether small rewordings produce similar explanations. The explanation concerns the *editing instruction* and the measured image changes; it does not reveal the editor's private reasoning or automatically identify causal source-image pixels. + +Read the [XWhy image-editing guide](../../../explainers/image-generation/image-editing.md) for the currently supported workflow and the [research preprint](https://arxiv.org/abs/2412.16277) for the underlying study. + +**Original article:** [Mapping the Mind of an Instruction-Based Image Editing Using SMILE](https://www.responsibleaihull.com/post/mapping-the-mind-of-an-instruction-based-image-editing-using-smile). This page is an original XWhy summary and does not reproduce the source article. diff --git a/docs/research/reddit/blogs/reading-local-explanations.md b/docs/research/reddit/blogs/reading-local-explanations.md new file mode 100644 index 00000000..89af9155 --- /dev/null +++ b/docs/research/reddit/blogs/reading-local-explanations.md @@ -0,0 +1,31 @@ +--- +title: What a Local Explanation Tells You +description: How to read a single-input explanation without treating it as a claim about the whole model. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# What a local explanation tells you + +![An original illustration of an image and its local attribution map](../../../assets/images/blogs/local-explanations.svg) + +*Explainability · XWhy explainer* + +A local explanation describes how a model behaved around a *particular* input under a chosen set of changes. If an image classifier responds strongly when the dog's face is hidden, that observation can help explain this prediction. It cannot tell you that the face is always the decisive feature across the entire dataset. + +## Read the result together with its question + +Before interpreting a chart, identify the selected input, the model output being explained and what counts as a changed feature. For language models, that might mean removing prompt words. For an image model, it might mean changing image regions. The explanation should also tell you what comparison and local model produced the contributions. + +A large positive contribution often means a feature was associated with a higher local score under those settings. A small value could mean the feature was unimportant nearby, or that the chosen perturbations did not isolate its influence well. The direction and units of a score depend on the output and explanation method; do not assume every heatmap or contribution plot uses the same scale. + +## Questions worth asking + +- Does the simpler local model reproduce the observed behaviour of the model being explained? +- Do small, harmless changes to the input leave the main finding intact? +- Would another sensible way of changing the input produce a similar conclusion? +- Are the highlighted regions or words relevant to the actual task? + +An explanation is a tool for examining behaviour, not proof of an internal reasoning process. It also does not establish whether a prediction is fair, safe or correct. For high-impact uses, combine local explanations with broader performance tests, subgroup checks and expert review. + +Read the [local explanations guide](../../../concepts/local-explanations.md) and [limitations](../../../concepts/limitations.md) for more detail. diff --git a/docs/research/reddit/blogs/responsible-ai-chennai.md b/docs/research/reddit/blogs/responsible-ai-chennai.md new file mode 100644 index 00000000..fe0f7fdf --- /dev/null +++ b/docs/research/reddit/blogs/responsible-ai-chennai.md @@ -0,0 +1,28 @@ +--- +title: Responsible AI Research in Chennai +description: Lessons from a University of Hull research exchange about trustworthy AI in healthcare. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Responsible AI research in Chennai + +![An original illustration of a city skyline and research collaborators](../../../assets/images/blogs/chennai-healthcare.svg) + +*Research visit · Research note based on [Koorosh Aslansefat's Hull research group report](https://www.responsibleaihull.com/post/research-visits-of-responsible-ai-team-to-chennai-india).* + +Researchers from the University of Hull visited Chennai in October 2024 to discuss how AI could be developed and evaluated responsibly in healthcare. Meetings with researchers and clinicians explored what happens when a model that appears promising in a laboratory meets the variety of patients, images and clinical practices in real services. + +## Three connected questions + +At IIT Madras, discussions addressed responsible AI research, privacy and the use of AI in clinical tasks. Clinicians from the Manipal Academy of Higher Education described the difficulty of transferring advice across different healthcare policies and patient populations. At the Madras Diabetes Research Foundation and Sankara Nethralaya, the conversations also covered representative data, diagnostic applications and the need for meaningful visual evidence in eye care. + +These settings pose practical questions for any model: Does its training data represent the people and conditions it will encounter? Can clinicians understand the evidence for an output? What happens when the input is poor quality or a recommendation is outside the system's competence? + +## Bringing explainability into the evaluation + +A method such as SMILE can help examine which input regions or terms are associated with changes in a particular prediction. It should be checked against clinical knowledge and paired with tests of accuracy, uncertainty and failure under changing conditions. A local explanation cannot replace a clinician's judgement or a well-designed evaluation across sites. + +The visit report describes an exchange of perspectives and prospective collaborations. It does not claim that a medical AI system was clinically validated during the visit. + +**Original report:** [Research Visits of Responsible AI Team to Chennai, India](https://www.responsibleaihull.com/post/research-visits-of-responsible-ai-team-to-chennai-india). This page is an original XWhy summary, with further context in the source report. diff --git a/docs/research/reddit/blogs/testing-explanation-fidelity.md b/docs/research/reddit/blogs/testing-explanation-fidelity.md new file mode 100644 index 00000000..7849563f --- /dev/null +++ b/docs/research/reddit/blogs/testing-explanation-fidelity.md @@ -0,0 +1,28 @@ +--- +title: Does the Explanation Hold Up? +description: Practical checks for the fit, stability and faithfulness of a local explanation. +--- + +[← All research and insights](../blogs.md){ .xwhy-blog-back } + +# Does the explanation hold up? + +![An original illustration of an explanation passing several evaluation checks](../../../assets/images/blogs/explanation-fidelity.svg) + +*Evaluation · XWhy explainer* + +An explanation can be easy to read and still be misleading. A highlighted word or image region should invite a further test: does the model behave as the explanation suggests when that part of the input changes? + +## Three checks answer different questions + +**Local fit or fidelity** asks whether the simpler explanation model approximates the target model on the sampled inputs around the example. A poor fit weakens confidence in the reported contributions even when the chart looks convincing. A good fit is useful, but it remains limited to the neighbourhood that was sampled. + +**Stability** asks whether the explanation remains similar after a small change that should not change its meaning. If a harmless rephrasing moves all the important terms, investigate the perturbation method, sampling variation and the model's own sensitivity. + +**Intervention checks** compare a stated attribution with the model's observed response after a selected feature is removed, replaced or preserved. If a feature claimed to dominate has little measured effect under relevant interventions, the claim needs revision. Feature changes must remain meaningful; replacing an image patch with an unnatural artefact can create a different problem. + +## Report the evidence, not only the visual + +An evaluation should identify the target model and input, the output being measured, the perturbation rules, the number of samples, and the score or comparison used. Repeat runs where randomness matters. When possible, show both the explanation and a small set of actual model outputs that support or challenge it. + +The [XWhy evaluation overview](../../../evaluation/index.md), [fidelity guidance](../../../evaluation/attribution-fidelity.md) and [faithfulness guidance](../../../evaluation/attribution-faithfulness.md) provide more specific measures. No one measure proves that an explanation is complete or that the underlying model is safe. diff --git a/properdocs.yml b/properdocs.yml index 2e1fa205..0521eec7 100644 --- a/properdocs.yml +++ b/properdocs.yml @@ -93,6 +93,14 @@ plugins: Research: - research/index.md: Research and reproducibility entry point - research/reddit/blogs.md: Research blogs and XWhy explainers + - research/reddit/blogs/image-editing-smile.md: Explaining image-editing instructions + - research/reddit/blogs/fairness-skin-cancer-ai.md: Fairness in skin cancer AI + - research/reddit/blogs/responsible-ai-chennai.md: Responsible AI in healthcare research visit + - research/reddit/blogs/digital-twins-chennai.md: Digital twin research visit + - research/reddit/blogs/how-smile-works.md: How SMILE works + - research/reddit/blogs/reading-local-explanations.md: Interpreting local explanations + - research/reddit/blogs/explaining-llm-responses.md: Explaining LLM responses + - research/reddit/blogs/testing-explanation-fidelity.md: Evaluating explanation fidelity - research/citation.md: How to cite XWhy Releases: - release-notes.md: Version history and detailed release notes @@ -172,6 +180,7 @@ exclude_docs: | gen_ref_pages.py not_in_nav: | + research/reddit/blogs/ concepts/index.md concepts/smile.md concepts/local-explanations.md