From 6fa5fbaf2fc88582d070adc2b0e765682206a984 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:55 +0000 Subject: [PATCH 01/21] diff: rename and group the line-range filter for clarity The line-range filter that mm/line-log-cleanup added uses names that obscure its model. The cursors lno_post/lno_pre and the index lno_0 share an lno_ prefix but conflate the pre/post-image axis with the 0-based/1-based axis, the hunk state is a flat set of rhunk_* fields, and the filter-state pointer is just s. The filter bridges two layers of diff.c, and its fields already used each layer's vocabulary, but in cryptic abbreviations. Spell them out to the form the rest of the file uses, so that the patches that follow can simplify and fix it with those clearer names in place: - lno_post/lno_pre -> lno_in_postimage/lno_in_preimage, the line-number cursors, matching the counters in struct emit_callback - lno_0 -> idx_in_postimage, the 0-based range index - the hunk-header geometry stays old/new (old_begin, new_begin, and counts) to match the xdiff_emit_hunk_fn callback and the "@@ - + @@" header it feeds, but moves from flat rhunk_* fields into a "hunk" sub-struct, so accesses read filter->hunk.old_begin - flush_rhunk -> flush_range_hunk - the filter-state pointer in each callback: s -> filter Also rename the struct line_range_callback to line_range_filter: it is a filter over xdiff output, not merely a callback. No behavior change. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 192 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 97 insertions(+), 95 deletions(-) diff --git a/diff.c b/diff.c index 5a584fa1d569e7..1e043c959ff2b4 100644 --- a/diff.c +++ b/diff.c @@ -623,15 +623,15 @@ struct emit_callback { * reveals whether they precede an in-range line (flush into range hunk) or * an out-of-range line (discard). */ -struct line_range_callback { +struct line_range_filter { xdiff_emit_line_fn orig_line_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ /* Post/pre-image line counters (1-based, set from hunk headers) */ - long lno_post; - long lno_pre; + long lno_in_postimage; + long lno_in_preimage; /* * Function name from most recent xdiff hunk header; @@ -640,12 +640,14 @@ struct line_range_callback { char func[80]; long funclen; - /* Range hunk being accumulated for the current range */ - struct strbuf rhunk; - long rhunk_old_begin, rhunk_old_count; - long rhunk_new_begin, rhunk_new_count; - int rhunk_active; - int rhunk_has_changes; /* any '+' or '-' lines? */ + /* The range hunk being accumulated for the current range. */ + struct { + struct strbuf lines; /* buffered in-range diff lines */ + long old_begin, old_count; + long new_begin, new_count; + int active; + int has_changes; /* any '+' or '-' line? */ + } hunk; /* Removal lines not yet known to be in-range */ struct strbuf pending_rm; @@ -2540,26 +2542,26 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_callback *s) +static void discard_pending_rm(struct line_range_filter *filter) { - strbuf_reset(&s->pending_rm); - s->pending_rm_count = 0; + strbuf_reset(&filter->pending_rm); + filter->pending_rm_count = 0; } -static void flush_rhunk(struct line_range_callback *s) +static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; - if (!s->rhunk_active || s->ret) + if (!filter->hunk.active || filter->ret) return; /* Drain any pending removal lines into the range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + if (filter->pending_rm_count) { + strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); + filter->hunk.old_count += filter->pending_rm_count; + filter->hunk.has_changes = 1; + discard_pending_rm(filter); } /* @@ -2568,22 +2570,22 @@ static void flush_rhunk(struct line_range_callback *s) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!s->rhunk_has_changes) { - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + if (!filter->hunk.has_changes) { + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); return; } strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - s->rhunk_old_begin, s->rhunk_old_count, - s->rhunk_new_begin, s->rhunk_new_count); - if (s->funclen > 0) { + filter->hunk.old_begin, filter->hunk.old_count, + filter->hunk.new_begin, filter->hunk.new_count); + if (filter->funclen > 0) { strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, s->func, s->funclen); + strbuf_add(&hdr, filter->func, filter->funclen); } strbuf_addch(&hdr, '\n'); - s->ret = s->orig_line_fn(s->orig_cb_data, hdr.buf, hdr.len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); /* @@ -2591,18 +2593,18 @@ static void flush_rhunk(struct line_range_callback *s) * The cast discards const because xdiff_emit_line_fn takes * char *, though fn_out_consume does not modify the buffer. */ - p = s->rhunk.buf; - end = p + s->rhunk.len; - while (!s->ret && p < end) { + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (!filter->ret && p < end) { const char *eol = memchr(p, '\n', end - p); unsigned long line_len = eol ? (unsigned long)(eol - p + 1) : (unsigned long)(end - p); - s->ret = s->orig_line_fn(s->orig_cb_data, (char *)p, line_len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, (char *)p, line_len); p += line_len; } - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); } static void line_range_hunk_fn(void *data, @@ -2610,7 +2612,7 @@ static void line_range_hunk_fn(void *data, long new_begin, long new_nr UNUSED, const char *func, long funclen) { - struct line_range_callback *s = data; + struct line_range_filter *filter = data; /* * When count > 0, begin is 1-based. When count == 0, begin is @@ -2622,104 +2624,104 @@ static void line_range_hunk_fn(void *data, * flush or discard them when the next content line reveals * whether the removals precede in-range content. */ - s->lno_post = new_begin; - s->lno_pre = old_begin; + filter->lno_in_postimage = new_begin; + filter->lno_in_preimage = old_begin; if (funclen > 0) { - if (funclen > (long)sizeof(s->func)) - funclen = sizeof(s->func); - memcpy(s->func, func, funclen); + if (funclen > (long)sizeof(filter->func)) + funclen = sizeof(filter->func); + memcpy(filter->func, func, funclen); } - s->funclen = funclen; + filter->funclen = funclen; } static int line_range_line_fn(void *priv, char *line, unsigned long len) { - struct line_range_callback *s = priv; + struct line_range_filter *filter = priv; const struct range *cur; - long lno_0, cur_pre; + long idx_in_postimage, cur_pre; - if (s->ret) - return s->ret; + if (filter->ret) + return filter->ret; if (line[0] == '-') { - if (!s->pending_rm_count) - s->pending_rm_pre_begin = s->lno_pre; - s->lno_pre++; - strbuf_add(&s->pending_rm, line, len); - s->pending_rm_count++; - return s->ret; + if (!filter->pending_rm_count) + filter->pending_rm_pre_begin = filter->lno_in_preimage; + filter->lno_in_preimage++; + strbuf_add(&filter->pending_rm, line, len); + filter->pending_rm_count++; + return filter->ret; } if (line[0] == '\\') { - if (s->pending_rm_count) - strbuf_add(&s->pending_rm, line, len); - else if (s->rhunk_active) - strbuf_add(&s->rhunk, line, len); + if (filter->pending_rm_count) + strbuf_add(&filter->pending_rm, line, len); + else if (filter->hunk.active) + strbuf_add(&filter->hunk.lines, line, len); /* otherwise outside tracked range; drop silently */ - return s->ret; + return filter->ret; } if (line[0] != '+' && line[0] != ' ') BUG("unexpected diff line type '%c'", line[0]); - lno_0 = s->lno_post - 1; - cur_pre = s->lno_pre; /* save before advancing for context lines */ - s->lno_post++; + idx_in_postimage = filter->lno_in_postimage - 1; + cur_pre = filter->lno_in_preimage; /* save before advancing for context lines */ + filter->lno_in_postimage++; if (line[0] == ' ') - s->lno_pre++; + filter->lno_in_preimage++; /* Advance past ranges we've passed */ - while (s->cur_range < s->ranges->nr && - lno_0 >= s->ranges->ranges[s->cur_range].end) { - if (s->rhunk_active) - flush_rhunk(s); - discard_pending_rm(s); - s->cur_range++; + while (filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { + if (filter->hunk.active) + flush_range_hunk(filter); + discard_pending_rm(filter); + filter->cur_range++; } /* Past all ranges */ - if (s->cur_range >= s->ranges->nr) { - discard_pending_rm(s); - return s->ret; + if (filter->cur_range >= filter->ranges->nr) { + discard_pending_rm(filter); + return filter->ret; } - cur = &s->ranges->ranges[s->cur_range]; + cur = &filter->ranges->ranges[filter->cur_range]; /* Before current range */ - if (lno_0 < cur->start) { - discard_pending_rm(s); - return s->ret; + if (idx_in_postimage < cur->start) { + discard_pending_rm(filter); + return filter->ret; } /* In range so start a new range hunk if needed */ - if (!s->rhunk_active) { - s->rhunk_active = 1; - s->rhunk_has_changes = 0; - s->rhunk_new_begin = lno_0 + 1; - s->rhunk_old_begin = s->pending_rm_count - ? s->pending_rm_pre_begin : cur_pre; - s->rhunk_old_count = 0; - s->rhunk_new_count = 0; - strbuf_reset(&s->rhunk); + if (!filter->hunk.active) { + filter->hunk.active = 1; + filter->hunk.has_changes = 0; + filter->hunk.new_begin = idx_in_postimage + 1; + filter->hunk.old_begin = filter->pending_rm_count + ? filter->pending_rm_pre_begin : cur_pre; + filter->hunk.old_count = 0; + filter->hunk.new_count = 0; + strbuf_reset(&filter->hunk.lines); } /* Flush pending removals into range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + if (filter->pending_rm_count) { + strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); + filter->hunk.old_count += filter->pending_rm_count; + filter->hunk.has_changes = 1; + discard_pending_rm(filter); } - strbuf_add(&s->rhunk, line, len); - s->rhunk_new_count++; + strbuf_add(&filter->hunk.lines, line, len); + filter->hunk.new_count++; if (line[0] == '+') - s->rhunk_has_changes = 1; + filter->hunk.has_changes = 1; else - s->rhunk_old_count++; + filter->hunk.old_count++; - return s->ret; + return filter->ret; } static void pprint_rename(struct strbuf *name, const char *a, const char *b) @@ -4086,7 +4088,7 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_callback lr_state; + struct line_range_filter lr_state; unsigned int i; long max_span = 0; @@ -4094,7 +4096,7 @@ static void builtin_diff(const char *name_a, lr_state.orig_line_fn = fn_out_consume; lr_state.orig_cb_data = &ecbdata; lr_state.ranges = line_ranges; - strbuf_init(&lr_state.rhunk, 0); + strbuf_init(&lr_state.hunk.lines, 0); strbuf_init(&lr_state.pending_rm, 0); /* @@ -4125,11 +4127,11 @@ static void builtin_diff(const char *name_a, die("unable to generate diff for %s", one->path); - flush_rhunk(&lr_state); + flush_range_hunk(&lr_state); if (lr_state.ret) die("unable to generate diff for %s", one->path); - strbuf_release(&lr_state.rhunk); + strbuf_release(&lr_state.hunk.lines); strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) From 5a508c13ac159a6c8e938ba6ce544a33f234e58b Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:56 +0000 Subject: [PATCH 02/21] diff: simplify the line-range filter by classifying removals immediately The filter buffered '-' lines in a pending_rm strbuf, deferring their classification until a '+' or ' ' line revealed the post-image position. That buffering is unnecessary: a removal occupies no post-image line, so it does not advance lno_in_postimage, and xdiff emits removals before additions within a change. A '-' therefore arrives while lno_in_postimage already holds the index the following '+'/' ' will occupy, and can be classified against the ranges as it arrives. The buffering also hid a bug: flush_range_hunk() drained pending_rm into the range hunk whenever the hunk was active, even after lno_in_postimage had advanced past the tracked range, so a deletion just after the tracked function leaked into the patch. Classifying each line as it arrives removes the pending_rm buffer, the discard_pending_rm() helper, three struct fields, and makes that bug impossible by construction. With every line classified on arrival, the buffered lines are the hunk's single source of truth, so the old/new counts need not be kept alongside them: flush_range_hunk() derives the counts (and whether the hunk holds any change) from the buffer when it builds the header. Drop the per-line counting and the old_count, new_count, and has_changes fields; there is no longer a second tally that could fall out of sync with the buffer. Add begin_range_hunk() to open the accumulator at the first in-range line, seeding both begins from the live image cursors, as the counterpart to flush_range_hunk(). With the counting gone too, line_range_line_fn() now only appends an in-range line. Document the coordinate model: a block comment on struct line_range_filter states it (the pre/post-image cursors, the 0-based idx_in_postimage, removals classified by the following line) with a worked example. Add tests for the leaked trailing deletion this fixes, the symmetric leading-deletion case, and the filter's range boundaries (a change at the first and last line of a range, and a pure in-range deletion). Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 215 ++++++++++++++++++++++++-------------------- t/t4211-line-log.sh | 125 ++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 97 deletions(-) diff --git a/diff.c b/diff.c index 1e043c959ff2b4..ee765d7ac2183d 100644 --- a/diff.c +++ b/diff.c @@ -610,18 +610,58 @@ struct emit_callback { }; /* - * State for the line-range callback wrappers that sit between - * xdi_diff_outf() and fn_out_consume(). xdiff produces a normal, - * unfiltered diff; the wrappers intercept each hunk header and line, - * track post-image position, and forward only lines that fall within - * the requested ranges. Contiguous in-range lines are collected into - * range hunks and flushed with a synthetic @@ header so that - * fn_out_consume() sees well-formed unified-diff fragments. + * Line-range filter: scopes "git log -L" output to the tracked ranges. * - * Removal lines ('-') cannot be classified by post-image position, so - * they are buffered in pending_rm until the next '+' or ' ' line - * reveals whether they precede an in-range line (flush into range hunk) or - * an out-of-range line (discard). + * It sits between xdi_diff_outf() and an output callback (fn_out_consume, + * diffstat_consume, checkdiff_consume). xdiff produces a normal diff; the + * filter forwards only the lines inside the requested ranges, collecting + * contiguous in-range lines into a "range hunk" emitted with a synthetic + * @@ header so the callback sees well-formed unified-diff fragments. + * + * A diff describes the change from a pre-image to a post-image. Each + * line is context (' ', in both), a removal ('-', pre-image only), or + * an addition ('+', post-image only). -L tracks ranges in the + * post-image, so a line is in range by its post-image position. + * + * Two 1-based cursors track the next line in each image, named as in + * struct emit_callback and seeded from the xdiff hunk header: + * + * lno_in_postimage advances on '+' and ' ' (lines in the post-image) + * lno_in_preimage advances on '-' and ' ' (lines in the pre-image) + * + * Ranges are 0-based half-open [start, end), so a line is tested at the + * 0-based index idx_in_postimage = lno_in_postimage - 1. + * + * A '-' is not present in the post-image, so it has no post-image line + * number of its own. Since it does not advance lno_in_postimage, it is + * classified at the idx_in_postimage that the following '+'/' ' will + * occupy. xdiff emits a change's removals before its additions, so that + * index is already known when the '-' arrives. + * + * The synthetic "@@ - + @@" header has two sides, old (the + * pre-image) and new (the post-image), matching the xdiff_emit_hunk_fn + * callback; the hunk.old_begin / hunk.new_begin fields below hold those + * begins, and flush_range_hunk() derives the counts from the buffered + * lines. + * + * Example, tracking post-image line 2 (range [1, 2)) of: + * + * pre-image post-image + * 1 a 1 a + * 2 b 2 X (b -> X) + * 3 c 3 c + * + * classify each line by idx_in_postimage. The pre and post columns + * are each cursor's value while that line is classified, i.e. before + * the line advances them (pre = lno_in_preimage, + * post = lno_in_postimage, idx = idx_in_postimage): + * ' a' pre 1 post 1 idx 0 -> before start, skip + * '-b' pre 2 post 2 idx 1 -> keep (removal) + * '+X' pre 3 post 2 idx 1 -> keep (addition) + * ' c' pre 3 post 3 idx 2 -> past end, flush + * + * -b and +X share idx = 1 because -b did not advance lno_in_postimage; + * both land in the range hunk, flushed when ' c' crosses the range end. */ struct line_range_filter { xdiff_emit_line_fn orig_line_fn; @@ -640,20 +680,18 @@ struct line_range_filter { char func[80]; long funclen; - /* The range hunk being accumulated for the current range. */ + /* + * The range hunk being accumulated. At most one is live at a time: + * it is flushed and reset as the cursor leaves each range (and once + * more at end of diff), then reused for the next range. + */ struct { struct strbuf lines; /* buffered in-range diff lines */ - long old_begin, old_count; - long new_begin, new_count; + long old_begin; + long new_begin; int active; - int has_changes; /* any '+' or '-' line? */ } hunk; - /* Removal lines not yet known to be in-range */ - struct strbuf pending_rm; - int pending_rm_count; - long pending_rm_pre_begin; /* pre-image line of first pending */ - int ret; /* latched error from orig_line_fn */ }; @@ -2542,26 +2580,48 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_filter *filter) +/* + * Begin a range hunk at the first in-range line. Its position fixes the + * hunk's begins, taken from the two image cursors before they advance: + * new_begin from the post-image, old_begin from the pre-image. The line + * counts are not tracked here; flush_range_hunk() derives them from the + * buffered lines. + */ +static void begin_range_hunk(struct line_range_filter *filter) { - strbuf_reset(&filter->pending_rm); - filter->pending_rm_count = 0; + filter->hunk.active = 1; + filter->hunk.new_begin = filter->lno_in_postimage; + filter->hunk.old_begin = filter->lno_in_preimage; + strbuf_reset(&filter->hunk.lines); } static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; + long old_count = 0, new_count = 0; + int has_changes = 0; if (!filter->hunk.active || filter->ret) return; - /* Drain any pending removal lines into the range hunk */ - if (filter->pending_rm_count) { - strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); - filter->hunk.old_count += filter->pending_rm_count; - filter->hunk.has_changes = 1; - discard_pending_rm(filter); + /* + * Derive the hunk's geometry from the buffered lines: a ' ' + * counts on both sides, a '-' on the old side, a '+' on the new. + * A '-' or '+' marks a real change; the "\ No newline at end of + * file" marker (line[0] == '\\') counts on neither side. + */ + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + if (*p == ' ' || *p == '-') + old_count++; + if (*p == ' ' || *p == '+') + new_count++; + if (*p == '-' || *p == '+') + has_changes = 1; + p = eol ? eol + 1 : end; } /* @@ -2570,15 +2630,15 @@ static void flush_range_hunk(struct line_range_filter *filter) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!filter->hunk.has_changes) { + if (!has_changes) { filter->hunk.active = 0; strbuf_reset(&filter->hunk.lines); return; } strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - filter->hunk.old_begin, filter->hunk.old_count, - filter->hunk.new_begin, filter->hunk.new_count); + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count); if (filter->funclen > 0) { strbuf_addch(&hdr, ' '); strbuf_add(&hdr, filter->func, filter->funclen); @@ -2618,11 +2678,6 @@ static void line_range_hunk_fn(void *data, * When count > 0, begin is 1-based. When count == 0, begin is * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of * that type will arrive, so the value is unused. - * - * Any pending removal lines from the previous xdiff hunk are - * intentionally left in pending_rm: the line callback will - * flush or discard them when the next content line reveals - * whether the removals precede in-range content. */ filter->lno_in_postimage = new_begin; filter->lno_in_preimage = old_begin; @@ -2638,88 +2693,56 @@ static void line_range_hunk_fn(void *data, static int line_range_line_fn(void *priv, char *line, unsigned long len) { struct line_range_filter *filter = priv; - const struct range *cur; - long idx_in_postimage, cur_pre; + long idx_in_postimage; + int in_range; if (filter->ret) return filter->ret; - if (line[0] == '-') { - if (!filter->pending_rm_count) - filter->pending_rm_pre_begin = filter->lno_in_preimage; - filter->lno_in_preimage++; - strbuf_add(&filter->pending_rm, line, len); - filter->pending_rm_count++; - return filter->ret; - } - if (line[0] == '\\') { - if (filter->pending_rm_count) - strbuf_add(&filter->pending_rm, line, len); - else if (filter->hunk.active) + if (filter->hunk.active) strbuf_add(&filter->hunk.lines, line, len); - /* otherwise outside tracked range; drop silently */ return filter->ret; } - if (line[0] != '+' && line[0] != ' ') + if (line[0] != '+' && line[0] != ' ' && line[0] != '-') BUG("unexpected diff line type '%c'", line[0]); + /* + * idx_in_postimage is this line's 0-based post-image index (see the model on + * struct line_range_filter). The cursors are advanced only after + * the line is classified, so a '-' is tested at the same idx_in_postimage as + * the '+'/' ' that follows it. + */ idx_in_postimage = filter->lno_in_postimage - 1; - cur_pre = filter->lno_in_preimage; /* save before advancing for context lines */ - filter->lno_in_postimage++; - if (line[0] == ' ') - filter->lno_in_preimage++; - /* Advance past ranges we've passed */ + /* Retire ranges we have passed, flushing the one we leave. */ while (filter->cur_range < filter->ranges->nr && idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { if (filter->hunk.active) flush_range_hunk(filter); - discard_pending_rm(filter); filter->cur_range++; } - /* Past all ranges */ - if (filter->cur_range >= filter->ranges->nr) { - discard_pending_rm(filter); - return filter->ret; - } + in_range = filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].start && + idx_in_postimage < filter->ranges->ranges[filter->cur_range].end; - cur = &filter->ranges->ranges[filter->cur_range]; + if (in_range) { + if (!filter->hunk.active) + begin_range_hunk(filter); - /* Before current range */ - if (idx_in_postimage < cur->start) { - discard_pending_rm(filter); - return filter->ret; + strbuf_add(&filter->hunk.lines, line, len); } - /* In range so start a new range hunk if needed */ - if (!filter->hunk.active) { - filter->hunk.active = 1; - filter->hunk.has_changes = 0; - filter->hunk.new_begin = idx_in_postimage + 1; - filter->hunk.old_begin = filter->pending_rm_count - ? filter->pending_rm_pre_begin : cur_pre; - filter->hunk.old_count = 0; - filter->hunk.new_count = 0; - strbuf_reset(&filter->hunk.lines); - } - - /* Flush pending removals into range hunk */ - if (filter->pending_rm_count) { - strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); - filter->hunk.old_count += filter->pending_rm_count; - filter->hunk.has_changes = 1; - discard_pending_rm(filter); - } - - strbuf_add(&filter->hunk.lines, line, len); - filter->hunk.new_count++; - if (line[0] == '+') - filter->hunk.has_changes = 1; - else - filter->hunk.old_count++; + /* + * Advance each image's cursor: a line present in that image (see + * the model) consumes one of its line numbers. + */ + if (line[0] != '-') + filter->lno_in_postimage++; + if (line[0] != '+') + filter->lno_in_preimage++; return filter->ret; } @@ -4097,7 +4120,6 @@ static void builtin_diff(const char *name_a, lr_state.orig_cb_data = &ecbdata; lr_state.ranges = line_ranges; strbuf_init(&lr_state.hunk.lines, 0); - strbuf_init(&lr_state.pending_rm, 0); /* * Inflate ctxlen so that all changes within @@ -4132,7 +4154,6 @@ static void builtin_diff(const char *name_a, die("unable to generate diff for %s", one->path); strbuf_release(&lr_state.hunk.lines); - strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index ca4eb7bbc713ef..e9691066deea74 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -738,6 +738,131 @@ test_expect_success '-L with -G filters to diff-text matches' ' grep "F2 + 2" actual ' +test_expect_success 'setup for trailing deletion test' ' + git checkout --orphan trailing-del && + git reset --hard && + cat >file.c <<-\EOF && + void tracked() + { + return 1; + } + // trailing comment + EOF + git add file.c && + test_tick && + git commit -m "add file with trailing comment" && + # Modify tracked() AND delete the trailing comment in + # one commit, so the commit touches the tracked range + # and is not filtered out by the revision walker. + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete trailing comment" +' + +test_expect_success '-L does not include deletions past end of tracked range' ' + git log -L:tracked:file.c --format= -1 -p >actual && + # The trailing comment deletion is outside the tracked + # range and should not appear in the patch output. + test_grep "return 2" actual && + test_grep ! "trailing comment" actual +' + +test_expect_success '-L includes leading deletions resolved by in-range line' ' + git checkout --orphan leading-del && + git reset --hard && + cat >file.c <<-\EOF && + // leading comment + void tracked() + { + return 1; + } + EOF + git add file.c && + test_tick && + git commit -m "add file with leading comment" && + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete leading comment" && + git log -L:tracked:file.c --format= -1 -p >actual && + # The leading comment deletion is resolved by the next + # non-removal line (void tracked), which is in range: a + # removal is classified by the position of the following + # line, so it joins the range that line falls in. + test_grep "return 2" actual && + test_grep "leading comment" actual +' + +test_expect_success 'setup for line-range filter edge cases' ' + git checkout --orphan filter-edge && + git reset --hard && + cat >file.c <<-\EOF && + void before() + { + return 0; + } + + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + return a + b + c; + } + + void after() + { + return 9; + } + EOF + git add file.c && + test_tick && + git commit -m "initial" +' + +test_expect_success '-L change at exact first line of range' ' + git checkout filter-edge && + # Change the function signature (first line of range) + sed "s/void tracked/int tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change first line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change first line" actual && + test_grep "+int tracked" actual && + test_grep "\\-void tracked" actual +' + +test_expect_success '-L change at exact last line of range' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Change the closing brace line (last line of range) + sed "s/^}$/} \/\/ end tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change last line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change last line" actual && + test_grep "end tracked" actual +' + +test_expect_success '-L pure deletion in range (no additions)' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Delete a line inside tracked() without adding anything + sed "/int c/d" file.c >tmp && + mv tmp file.c && + git commit -a -m "pure deletion" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "pure deletion" actual && + test_grep "\\-.*int c" actual +' + test_expect_success '-L with --diff-filter=M excludes root commit' ' git checkout parent-oids && git log -L:func2:file.c --diff-filter=M --format=%s --no-patch >actual && From 7555510bf32835fae50363b86a8a667c1b681803 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:57 +0000 Subject: [PATCH 03/21] diff: emit -L hunk headers via xdiff's formatter The line-range filter builds its own "@@ - + @@" header for each range hunk. For a side with no lines (count 0, such as the old side of a pure insertion), the begin should be the number of the line before the change, per the convention git diff and xdl_emit_hunk_hdr() follow. The hand-rolled code's begin was one too high; in t4211 this produced @@ -25,0 +18,9 @@ an old begin of 25 in a 24-line file, where git diff would give 24. Stop hand-rolling the header. flush_range_hunk() now formats it through xdiff's own emitter: a new xdiff_emit_hunk_header() helper wraps xdl_emit_hunk_hdr(), the function that produces every other diff's hunk headers. The count-0 begin is then correct by construction, and as a side effect -L headers match git diff exactly, including its omission of a count of 1 ("@@ -22 +22 @@" rather than "@@ -22,1 +22,1 @@"). xdiff's hunk callback already hands line_range_hunk_fn() a count-0 begin decremented, so undo that when seeding the cursors and let the formatter re-apply the convention once, at emit time. The off-by-one predates this series, and the two regenerated fixtures reach it from different origins: no-assertion-error has carried it since its test was added in ab60c693a2 (line-log: fix assertion error, 2025-08-18), while vanishes-early acquired it when 86e986f166 (line-log: route -L output through the standard diff pipeline) reshaped its tracked line into a pure insertion. vanishes-early also drops its count-1 counts. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 27 +++++++++++------------- t/t4211/sha1/expect.no-assertion-error | 2 +- t/t4211/sha1/expect.vanishes-early | 6 +++--- t/t4211/sha256/expect.no-assertion-error | 2 +- t/t4211/sha256/expect.vanishes-early | 6 +++--- xdiff-interface.c | 19 +++++++++++++++++ xdiff-interface.h | 15 +++++++++++++ 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/diff.c b/diff.c index ee765d7ac2183d..9751bb679874b9 100644 --- a/diff.c +++ b/diff.c @@ -2636,14 +2636,9 @@ static void flush_range_hunk(struct line_range_filter *filter) return; } - strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - filter->hunk.old_begin, old_count, - filter->hunk.new_begin, new_count); - if (filter->funclen > 0) { - strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, filter->func, filter->funclen); - } - strbuf_addch(&hdr, '\n'); + xdiff_emit_hunk_header(&hdr, filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); @@ -2668,19 +2663,21 @@ static void flush_range_hunk(struct line_range_filter *filter) } static void line_range_hunk_fn(void *data, - long old_begin, long old_nr UNUSED, - long new_begin, long new_nr UNUSED, + long old_begin, long old_nr, + long new_begin, long new_nr, const char *func, long funclen) { struct line_range_filter *filter = data; /* - * When count > 0, begin is 1-based. When count == 0, begin is - * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of - * that type will arrive, so the value is unused. + * Seed the per-image line cursors from the hunk header's begins. For + * a side with no lines (count 0), xdiff's callback has already moved + * its begin to the line before the change, so add one back to recover + * the true 1-based start. xdiff_emit_hunk_header() reapplies that -1 + * when the clipped hunk is emitted. */ - filter->lno_in_postimage = new_begin; - filter->lno_in_preimage = old_begin; + filter->lno_in_postimage = new_nr ? new_begin : new_begin + 1; + filter->lno_in_preimage = old_nr ? old_begin : old_begin + 1; if (funclen > 0) { if (funclen > (long)sizeof(filter->func)) diff --git a/t/t4211/sha1/expect.no-assertion-error b/t/t4211/sha1/expect.no-assertion-error index 54c568f273a6d2..95faf51a7b46d9 100644 --- a/t/t4211/sha1/expect.no-assertion-error +++ b/t/t4211/sha1/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index bf79c2f..27c829c 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha1/expect.vanishes-early b/t/t4211/sha1/expect.vanishes-early index a413ad36598ddf..e4b1a201d5d254 100644 --- a/t/t4211/sha1/expect.vanishes-early +++ b/t/t4211/sha1/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index 0b9cae5..5de3ea4 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 100b61a6f2f720f812620a9d10afb3a960ccb73c @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index 5e709a1..0b9cae5 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..444e415 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4211/sha256/expect.no-assertion-error b/t/t4211/sha256/expect.no-assertion-error index c25f2ce19c05d9..815d27f7f17b7e 100644 --- a/t/t4211/sha256/expect.no-assertion-error +++ b/t/t4211/sha256/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index 69cb69c..a0d566e 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha256/expect.vanishes-early b/t/t4211/sha256/expect.vanishes-early index bc33b963dc8570..263fc9eaace442 100644 --- a/t/t4211/sha256/expect.vanishes-early +++ b/t/t4211/sha256/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index e4fa1d8..62c1fc2 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 29f32ac3141c48b22803e5c4127b719917b67d0f8ca8c5248bebfa2a19f7da10 @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index d325124..e4fa1d8 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..9f550c3 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/xdiff-interface.c b/xdiff-interface.c index 5ee2b96d0a756f..32e04630ee2ee8 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -91,6 +91,25 @@ static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) return 0; } +static int strbuf_out_line(void *priv, mmbuffer_t *mb, int nbuf) +{ + struct strbuf *out = priv; + int i; + for (i = 0; i < nbuf; i++) + strbuf_add(out, mb[i].ptr, mb[i].size); + return 0; +} + +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen) +{ + xdemitcb_t ecb = { .priv = out, .out_line = strbuf_out_line }; + xdl_emit_hunk_hdr(old_begin, old_count, new_begin, new_count, + func, funclen, &ecb); +} + /* * Trim down common substring at the end of the buffers, * but end on a complete line. diff --git a/xdiff-interface.h b/xdiff-interface.h index ce54e1c0e002f8..51c88296ed5e68 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -76,4 +76,19 @@ int xdiff_compare_lines(const char *l1, long s1, */ unsigned long xdiff_hash_string(const char *s, size_t len, long flags); +struct strbuf; + +/* + * Append a unified-diff hunk header to `out`, e.g. + * "@@ - + @@ func\n". The header comes from wrapping xdiff's + * own hunk-header emitter, so it matches what a normal diff would + * produce for these begins and counts. For a side with no lines + * (count 0) the begin is the line before the change, and a count of 1 + * is omitted. + */ +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen); + #endif From 56cf30f68c6ab06e34c295c62b081da8904b1c41 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:58 +0000 Subject: [PATCH 04/21] diff: extract a line-range diff helper for reuse builtin_diff() open-codes the line-range filter setup and teardown around its xdi_diff_outf() call: zero the struct, point it at the output callback, inflate ctxlen to the largest range span so each range yields a single xdiff hunk, run the diff, flush the trailing range hunk, and release the buffer. The upcoming -L stat and check formats need the same sequence. Extract line_range_filter_init() for the setup and a line_range_filter_diff() helper that prepares the xdiff config the filter needs, runs an initialized filter through xdi_diff_outf(), flushes the final range hunk, and releases it, returning the latched error. The helper inflates ctxlen to the largest range span so each range yields a single xdiff hunk, and clears XDL_EMIT_NO_HUNK_HDR so the hunk headers the filter seeds its position from are always emitted. Folding both into the helper keeps these invariants, which the filter's position tracking relies on, in a single place for every consumer. builtin_diff() now does init + line_range_filter_diff(); the next two patches reuse them in builtin_diffstat() and builtin_checkdiff() instead of repeating the boilerplate. No behavior change: builtin_diff() leaves XDL_EMIT_NO_HUNK_HDR unset, so clearing it is a no-op until the suppressing consumers arrive. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 100 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/diff.c b/diff.c index 9751bb679874b9..6233a96bf02252 100644 --- a/diff.c +++ b/diff.c @@ -2580,6 +2580,18 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } +static void line_range_filter_init(struct line_range_filter *filter, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, + void *cb_data) +{ + memset(filter, 0, sizeof(*filter)); + filter->orig_line_fn = line_fn; + filter->orig_cb_data = cb_data; + filter->ranges = ranges; + strbuf_init(&filter->hunk.lines, 0); +} + /* * Begin a range hunk at the first in-range line. Its position fixes the * hunk's begins, taken from the two image cursors before they advance: @@ -2744,6 +2756,50 @@ static int line_range_line_fn(void *priv, char *line, unsigned long len) return filter->ret; } +/* + * Run an xdiff pass through an initialized line-range filter, flush the + * final range hunk, and release the filter. Inflates ctxlen to the largest + * range span first, so that every change within a single range lands in one + * xdiff hunk and the inter-change context is emitted; the filter then clips + * back to range boundaries. The optimal ctxlen depends on where changes fall + * within the range, which is only known after xdiff runs, so the max span is + * the upper bound that guarantees correctness in a single pass. Every + * consumer (patch, diffstat, check) relies on one xdiff hunk per range, so + * this lives here rather than at each call site. Also clears + * XDL_EMIT_NO_HUNK_HDR: the filter seeds its per-image position from the hunk + * headers, so a consumer that otherwise suppresses them (diffstat) still gets + * them here. Returns non-zero if xdiff or any forwarded callback failed. + */ +static int line_range_filter_diff(struct line_range_filter *filter, + mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + const struct range_set *ranges = filter->ranges; + long max_span = 0; + unsigned int i; + int ret; + + for (i = 0; i < ranges->nr; i++) { + long span = ranges->ranges[i].end - ranges->ranges[i].start; + if (span > max_span) + max_span = span; + } + if (max_span > xecfg->ctxlen) + xecfg->ctxlen = max_span; + + /* the filter seeds its per-image position from hunk headers */ + xecfg->flags &= ~XDL_EMIT_NO_HUNK_HDR; + + ret = xdi_diff_outf(mf1, mf2, line_range_hunk_fn, + line_range_line_fn, filter, xpp, xecfg); + if (!ret) { + flush_range_hunk(filter); + ret = filter->ret; + } + strbuf_release(&filter->hunk.lines); + return ret; +} + static void pprint_rename(struct strbuf *name, const char *a, const char *b) { const char *old_name = a; @@ -4108,49 +4164,15 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_filter lr_state; - unsigned int i; - long max_span = 0; + struct line_range_filter lr_filter; - memset(&lr_state, 0, sizeof(lr_state)); - lr_state.orig_line_fn = fn_out_consume; - lr_state.orig_cb_data = &ecbdata; - lr_state.ranges = line_ranges; - strbuf_init(&lr_state.hunk.lines, 0); - - /* - * Inflate ctxlen so that all changes within - * any single range are merged into one xdiff - * hunk and the inter-change context is emitted. - * The callback clips back to range boundaries. - * - * The optimal ctxlen depends on where changes - * fall within the range, which is only known - * after xdiff runs; the max range span is the - * upper bound that guarantees correctness in a - * single pass. - */ - for (i = 0; i < line_ranges->nr; i++) { - long span = line_ranges->ranges[i].end - - line_ranges->ranges[i].start; - if (span > max_span) - max_span = span; - } - if (max_span > xecfg.ctxlen) - xecfg.ctxlen = max_span; - - if (xdi_diff_outf(&mf1, &mf2, - line_range_hunk_fn, - line_range_line_fn, - &lr_state, &xpp, &xecfg)) - die("unable to generate diff for %s", - one->path); + line_range_filter_init(&lr_filter, line_ranges, + fn_out_consume, &ecbdata); - flush_range_hunk(&lr_state); - if (lr_state.ret) + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) die("unable to generate diff for %s", one->path); - strbuf_release(&lr_state.hunk.lines); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); From 660aae7323bca80e4441479ccd3b83e60dc16477 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:59 +0000 Subject: [PATCH 05/21] line-log: support diff stat formats with -L Reuse the line_range_filter in builtin_diffstat() so the stat formats count only the lines within the tracked range. When a filepair carries line_ranges, the filter wraps diffstat_consume() as its output callback, forwarding only the lines inside the range for counting. flush_range_hunk() replays buffered content through diffstat_consume(), which ignores synthetic @@ headers since it only counts '+' and '-' lines. Expand the output format allowlist in setup_revisions() to accept --stat, --numstat, and --shortstat with -L. Leave --dirstat out of the allowlist so it is rejected like any other unsupported format. Its default mode counts each file's whole-file byte damage via diffcore_count_changes(), outside the line-based pipeline that the -L filter scopes, so bare --dirstat cannot honor the tracked range. The --dirstat=lines mode could: it aggregates the same per-file line counts as --numstat, which -L already scopes. But accepting only that sub-mode while bare --dirstat keeps erroring is a confusing split, so the whole format is deferred to a follow-up; --numstat already reports the exact per-file counts within the tracked range. Also drop "yet" from the generic -L rejection message ("does not yet support the requested diff format"). Some rejected formats do not fit a line range at all, so "yet" wrongly implied they are all just awaiting support. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 12 ++- diff.c | 13 ++- revision.c | 6 +- t/t4211-line-log.sh | 150 ++++++++++++++++++++++---- 4 files changed, 155 insertions(+), 26 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 72f639b5e79ea4..a111a492b49881 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -9,10 +9,14 @@ __ and __ (or __) must exist in the starting revision. You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. - Non-patch diff formats `--raw`, `--name-only`, `--name-status`, - and `--summary` are supported. Diff stat formats - (`--stat`, `--numstat`, `--shortstat`, `--dirstat`) are not - currently implemented. + The following non-patch diff formats are supported: `--raw`, + `--name-only`, `--name-status`, `--summary`, + `--stat`, `--numstat`, and `--shortstat`. + The stat formats count only lines within the tracked range. + `--dirstat` is not supported + with `-L`: it summarizes change as each directory's share of + the total churn, not as counts for the tracked lines. Use + `--numstat` for exact per-file counts within the range. + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, diff --git a/diff.c b/diff.c index 6233a96bf02252..026fafeb90b6fc 100644 --- a/diff.c +++ b/diff.c @@ -4289,7 +4289,18 @@ static void builtin_diffstat(const char *name_a, const char *name_b, xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (xdi_diff_outf(&mf1, &mf2, NULL, + + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, p->line_ranges, + diffstat_consume, diffstat); + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, diffstat_consume, diffstat, &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); diff --git a/revision.c b/revision.c index 6a8101e8b7ef5f..2c76e15778de32 100644 --- a/revision.c +++ b/revision.c @@ -3193,8 +3193,10 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s (revs->diffopt.output_format & ~(DIFF_FORMAT_PATCH | DIFF_FORMAT_NO_OUTPUT | DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | - DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY)))) - die(_("-L does not yet support the requested diff format")); + DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | + DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | + DIFF_FORMAT_SHORTSTAT)))) + die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) revs->expand_tabs_in_log = revs->expand_tabs_in_log_default; diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index e9691066deea74..b9ca336dbc4f99 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -176,24 +176,15 @@ test_expect_success '--name-status shows status and path' ' test_grep ! "^@@" actual ' -test_expect_success '--stat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --stat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--numstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --numstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--shortstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --shortstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--dirstat is not yet supported with -L' ' +test_expect_success '--dirstat is not supported with -L' ' + # --dirstat is not supported with -L: its default mode measures + # whole-file change, not the tracked lines, and the + # --dirstat=lines variant is deferred too, so both forms are + # rejected like any other unsupported format. test_must_fail git log -L1,24:b.c --dirstat 2>err && - test_grep "does not yet support" err + test_grep "does not support" err && + test_must_fail git log -L1,24:b.c --dirstat=lines 2>err && + test_grep "does not support" err ' test_expect_success 'setup for checking fancy rename following' ' @@ -887,9 +878,9 @@ test_expect_success '-L with -S suppresses non-matching commits' ' test_cmp expect actual ' -test_expect_success '--full-diff is not yet supported with -L' ' +test_expect_success '--full-diff is not supported with -L' ' test_must_fail git log -L1,24:b.c --full-diff 2>err && - test_grep "does not yet support" err + test_grep "does not support" err ' test_expect_success '-L --oneline has no extra blank line before diff' ' @@ -900,6 +891,127 @@ test_expect_success '-L --oneline has no extra blank line before diff' ' test_grep "^diff --git" line2 ' +test_expect_success 'setup for stat range-scoping tests' ' + git checkout --orphan stat-scoping && + git reset --hard && + cat >file.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add file.c && + test_tick && + git commit -m "Add func1() and func2()" && + + # Modify both functions in a single commit so that + # whole-file stats differ from the counts for the tracked range. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" file.c >tmp && + mv tmp file.c && + git commit -a -m "Modify both functions" +' + +test_expect_success '--numstat counts only lines in tracked range' ' + # "Modify both functions" changes one line in func1 and one in + # func2. Whole-file numstat would show 2 added, 2 deleted. + # numstat for func2 within the tracked range should show only 1 and 1. + git log -L:func2:file.c --numstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "^1 1 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat counts only additions for root commit' ' + # Root commit creates both func1 (4 lines) and func2 (4 lines). + # Whole-file numstat would show 9 lines added. numstat for func2 + # within the tracked range should show only 4. + git log -L:func2:file.c --numstat --format=%s >actual && + test_grep "Add func1() and func2()" actual && + test_grep "^4 0 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--stat counts only lines in tracked range' ' + git log -L:func2:file.c --stat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "file.c |" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--shortstat counts only lines in tracked range' ' + # --shortstat prints only the summary line: no per-file "file.c |" + # line. Counts cover only the tracked range, as for --numstat above. + git log -L:func2:file.c --shortstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "file.c |" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat across renames and multiple commits' ' + # parallel-change carries the tracked function f across an a.c -> b.c + # rename and a merge of two parallel histories. With -M, --numstat + # follows the rename and reports added/removed counts for f within + # the tracked range (not whole-file) per commit; the file column flips from + # b.c to a.c at the rename as the walk goes back in time. Commits + # that do not change the range of f emit no row (the merge and the + # pure file-move produce nothing), so there are fewer rows than + # commits. + git checkout parallel-change && + git log -M -L ":f:b.c" --format= --numstat >actual && + cat >expect <<-\EOF && + 1 1 b.c + 1 1 a.c + 1 1 a.c + 1 1 a.c + 1 0 a.c + 13 0 a.c + EOF + test_cmp expect actual +' + +test_expect_success '-L multiple ranges with --numstat excludes untracked change' ' + git checkout --orphan multi-range && + git reset --hard && + cat >m.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + + int func3() + { + return F3; + } + EOF + git add m.c && + test_tick && + git commit -m "add m.c" && + # Change all three functions but track only func1 and func2. + # Whole-file numstat would be 3 3; a 2 2 result proves the + # untracked func3 change is excluded and the two ranges just sum. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" -e "s/F3/F3 + 3/" m.c >tmp && + mv tmp m.c && + git commit -a -m "Modify all three functions" && + git log -L:func1:m.c -L:func2:m.c --numstat --format=%s -1 >actual && + test_grep "Modify all three functions" actual && + test_grep "^2 2 m.c$" actual && + test_grep ! "^3 3 m.c$" actual +' + test_expect_success '--summary shows new file on root commit' ' git checkout parent-oids && git log -L:func2:file.c --summary --format= >actual && From 54438a56a4edb4c70459180f37af3e43e5d7e6a3 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:29:00 +0000 Subject: [PATCH 06/21] diff: support --check with -L line ranges builtin_checkdiff() runs its own xdiff pass to detect whitespace errors in newly added lines. When -L is active, the check should be scoped to the tracked line ranges rather than the whole file. Reuse the line_range_filter to wrap checkdiff_consume(), the same pattern already used for patch output and diffstat. The filter forwards only in-range lines for whitespace checking. checkdiff reports the file line number of each error, which it normally learns from the hunk header via checkdiff_consume_hunk(). The filter synthesizes its own hunk headers, so give it an optional hunk callback and route checkdiff_consume_hunk() through it; this sets the post-image position before the in-range lines are replayed. Without it the reported line numbers would count from the start of the range hunk rather than the start of the file. The trailing blank-at-eof check is a second pass that scans the whole file via check_blank_at_eof(), so gate its report on the tracked ranges as well; otherwise a blank line added at end of file is reported even when it lies outside the range. Add DIFF_FORMAT_CHECKDIFF to the -L output format allowlist in setup_revisions() so that -L --check is accepted, and list --check among the supported formats in the documentation. Add tests covering that whitespace errors are reported, scoped to the tracked range, and labeled with the correct file line number, including when two errors in one range are separated by a gap that would otherwise split into multiple xdiff hunks. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 2 +- diff.c | 65 ++++++++++++++++++- revision.c | 2 +- t/t4211-line-log.sh | 92 +++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index a111a492b49881..33b4e948815aac 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -10,7 +10,7 @@ You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. The following non-patch diff formats are supported: `--raw`, - `--name-only`, `--name-status`, `--summary`, + `--name-only`, `--name-status`, `--summary`, `--check`, `--stat`, `--numstat`, and `--shortstat`. The stat formats count only lines within the tracked range. `--dirstat` is not supported diff --git a/diff.c b/diff.c index 026fafeb90b6fc..519c5133566032 100644 --- a/diff.c +++ b/diff.c @@ -665,6 +665,12 @@ struct emit_callback { */ struct line_range_filter { xdiff_emit_line_fn orig_line_fn; + /* + * Optional; consumers that report file line numbers (e.g. + * checkdiff) need the synthetic hunk header to set their + * post-image position before in-range lines are replayed. + */ + xdiff_emit_hunk_fn orig_hunk_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ @@ -2652,6 +2658,17 @@ static void flush_range_hunk(struct line_range_filter *filter) filter->hunk.new_begin, new_count, filter->func, filter->funclen); + /* + * Inform a line-numbering consumer of the post-image position + * before replaying lines, mirroring the hunk callback xdiff + * would have issued for a non-scoped diff. + */ + if (filter->orig_hunk_fn) + filter->orig_hunk_fn(filter->orig_cb_data, + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); @@ -4330,11 +4347,29 @@ static void builtin_diffstat(const char *name_a, const char *name_b, diff_free_filespec_data(two); } +/* + * Is the 0-based line index within any of the tracked ranges? + * (range_set ranges are 0-based, half-open [start, end).) This is a + * one-shot query for a single line and scans; the streaming filter + * (line_range_line_fn) uses a forward cursor instead. + */ +static int idx_in_ranges(const struct range_set *ranges, long idx) +{ + unsigned int i; + + for (i = 0; i < ranges->nr; i++) + if (idx >= ranges->ranges[i].start && + idx < ranges->ranges[i].end) + return 1; + return 0; +} + static void builtin_checkdiff(const char *name_a, const char *name_b, const char *attr_path, struct diff_filespec *one, struct diff_filespec *two, - struct diff_options *o) + struct diff_options *o, + const struct range_set *line_ranges) { mmfile_t mf1, mf2; struct checkdiff_t data; @@ -4374,7 +4409,19 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 1; /* at least one context line */ xpp.flags = 0; - if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, + + if (line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, line_ranges, + checkdiff_consume, &data); + lr_filter.orig_hunk_fn = checkdiff_consume_hunk; + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate checkdiff for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, checkdiff_consume, &data, &xpp, &xecfg)) die("unable to generate checkdiff for %s", one->path); @@ -4387,6 +4434,17 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, check_blank_at_eof(&mf1, &mf2, &ecbdata); blank_at_eof = ecbdata.blank_at_eof_in_postimage; + /* + * check_blank_at_eof() scans the whole file; with -L, + * keep the report only when its line is in a tracked + * range. The error's location is the first trailing + * blank line (blank_at_eof, 1-based; ranges 0-based), so + * we scope by that line. + */ + if (blank_at_eof && line_ranges && + !idx_in_ranges(line_ranges, blank_at_eof - 1)) + blank_at_eof = 0; + if (blank_at_eof) { static char *err; if (!err) @@ -5179,7 +5237,8 @@ static void run_checkdiff(struct diff_filepair *p, struct diff_options *o) diff_fill_oid_info(p->one, o->repo->index); diff_fill_oid_info(p->two, o->repo->index); - builtin_checkdiff(name, other, attr_path, p->one, p->two, o); + builtin_checkdiff(name, other, attr_path, p->one, p->two, o, + p->line_ranges); } void repo_diff_setup(struct repository *r, struct diff_options *options) diff --git a/revision.c b/revision.c index 2c76e15778de32..7abb287451ba14 100644 --- a/revision.c +++ b/revision.c @@ -3195,7 +3195,7 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | - DIFF_FORMAT_SHORTSTAT)))) + DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_CHECKDIFF)))) die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index b9ca336dbc4f99..68576418f41895 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -1018,4 +1018,96 @@ test_expect_success '--summary shows new file on root commit' ' test_grep "create mode 100644 file.c" actual ' +test_expect_success 'setup for --check test' ' + git checkout --orphan check-test && + git reset --hard && + cat >check.c <<-\EOF && + void tracked() + { + return; + } + + void other() + { + return; + } + EOF + git add check.c && + test_tick && + git commit -m "add check.c" && + # Introduce trailing whitespace errors in both functions + sed "s/return;/return; /" check.c >check.c.tmp && + mv check.c.tmp check.c && + git commit -a -m "introduce trailing whitespace" +' + +test_expect_success '--check scoped to tracked range with correct file line' ' + # tracked() trailing whitespace is at check.c:3; report it with the + # real file line number, not a count from the start of the range + # hunk. other() at check.c:8 is outside the range and is excluded. + test_must_fail git log -L:tracked:check.c --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep ! "check.c:8:" actual +' + +test_expect_success '--check reports each of several tracked ranges' ' + # Track both functions as separate ranges. Each range is flushed + # as its own hunk, so the second error must report its real file + # line (check.c:8), not continue the numbering from the first + # range (check.c:3). + test_must_fail git log -L:tracked:check.c -L:other:check.c \ + --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep "check.c:8: trailing whitespace" actual +' + +test_expect_success '--check line numbers stay correct across a gap in one range' ' + git checkout --orphan check-gap && + git reset --hard && + cat >gap.c <<-\EOF && + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + int d = 4; + int e = 5; + int g = 7; + return; + } + EOF + git add gap.c && + test_tick && + git commit -m "add gap.c" && + # Two trailing-whitespace errors within one tracked range, + # separated by clean lines. ctxlen is inflated to the range span, + # so they land in a single xdiff hunk with the gap as context; + # both must report their real file line number, with the context + # lines between them counted. + sed -e "s/int a = 1;/int a = 1; /" -e "s/int g = 7;/int g = 7; /" gap.c >tmp && + mv tmp gap.c && + git commit -a -m "ws errors with a gap" && + test_must_fail git log -L:tracked:gap.c --check --format= >actual && + test_grep "gap.c:3: trailing whitespace" actual && + test_grep "gap.c:8: trailing whitespace" actual +' + +test_expect_success '--check does not report blank-at-eof outside the range' ' + git checkout --orphan check-eof && + git reset --hard && + printf "void tracked()\n{\n return;\n}\n\nint tail = 1;\n" >eof.c && + git add eof.c && + test_tick && + git commit -m "add eof.c" && + # One commit introduces a trailing-whitespace error inside tracked() + # (line 3) and a blank line at end of file (line 7, outside the + # range). The blank-at-eof check scans the whole file, so it must be + # scoped: report the in-range error, not the out-of-range EOF blank. + printf "void tracked()\n{\n return; \n}\n\nint tail = 1;\n\n" >eof.c && + git commit -a -m "ws in range, blank at eof out of range" && + test_must_fail git log -L:tracked:eof.c --check --format= >actual && + test_grep "eof.c:3: trailing whitespace" actual && + test_grep ! "blank line at EOF" actual +' + test_done From f67c51df064d2b64b257bd8c17d757cc0ce1b7fc Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:29:01 +0000 Subject: [PATCH 07/21] diffcore-pickaxe: scope -G to the -L tracked range git log -L scopes its diff output to the tracked range, but pickaxe (-S, -G) still runs in diffcore over the whole-file change, so -L -G selects a commit whenever the pattern appears in any added or removed line of the file, even outside the tracked range. Teach -G to honor the range. diff_grep() already runs an xdiff pass and greps the +/- lines; route that pass through the line-range filter so only the tracked range's lines are grepped. Expose the filter as diff_emit_line_ranges(), an xdi_diff_outf() that emits only the tracked range's lines, thread the filepair's line_ranges through the pickaxe callback, and pass it from pickaxe_match(). Skip scoping under textconv, whose output is not in the original file's line coordinates. -G needs only a hit/no-hit answer, so the line-number concerns the filter handles for patch and check output do not apply here. -S is left matching the whole file: it counts needle occurrences per blob rather than grepping the diff, so scoping it needs a different approach, left to a follow-up. has_changes() takes the range parameter but ignores it for now. Document the resulting -L pickaxe scoping: -G is scoped to the tracked range, while -S still matches the whole file. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 5 +- diff.c | 15 ++++++ diffcore-pickaxe.c | 37 +++++++++++--- t/t4211-line-log.sh | 72 +++++++++++++++++++++++++-- xdiff-interface.h | 13 +++++ 5 files changed, 130 insertions(+), 12 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 33b4e948815aac..d619ffa6336513 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -20,6 +20,9 @@ + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, -as are pickaxe options (`-S`, `-G`) and `--diff-filter`. +as are pickaxe options (`-S`, `-G`) and `--diff-filter`. `-G` is +scoped to the tracked range; `-S` is still evaluated over the whole +file, so an `-S` query may select a commit for a change outside the +range. + include::line-range-format.adoc[] diff --git a/diff.c b/diff.c index 519c5133566032..a8f346621b9fb5 100644 --- a/diff.c +++ b/diff.c @@ -2817,6 +2817,21 @@ static int line_range_filter_diff(struct line_range_filter *filter, return ret; } +/* + * Expose the in-file line-range filter to callers outside diff.c (e.g. + * pickaxe -G); see xdiff-interface.h for the contract. + */ +int diff_emit_line_ranges(mmfile_t *one, mmfile_t *two, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + struct line_range_filter filter; + + line_range_filter_init(&filter, ranges, line_fn, cb_data); + return line_range_filter_diff(&filter, one, two, xpp, xecfg); +} + static void pprint_rename(struct strbuf *name, const char *a, const char *b) { const char *old_name = a; diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index a52d569911c48e..047b2bf7ac65d5 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -16,7 +16,8 @@ typedef int (*pickaxe_fn)(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws); + regex_t *regexp, kwset_t kws, + const struct range_set *ranges); struct diffgrep_cb { regex_t *regexp; @@ -42,7 +43,8 @@ static int diffgrep_consume(void *priv, char *line, unsigned long len) static int diff_grep(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws UNUSED) + regex_t *regexp, kwset_t kws UNUSED, + const struct range_set *ranges) { struct diffgrep_cb ecbdata; xpparam_t xpp; @@ -50,8 +52,11 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, int ret; /* - * We have both sides; need to run textual diff and see if - * the pattern appears on added/deleted lines. + * We have both sides; need to run textual diff and see if the + * pattern appears on added/deleted lines. Under -L (ranges set), + * forward only the tracked range's lines so the match is scoped. + * -G needs only a hit/no-hit answer, so the line-number bookkeeping + * the filter does for -L patch and check output is irrelevant here. */ memset(&xpp, 0, sizeof(xpp)); memset(&xecfg, 0, sizeof(xecfg)); @@ -65,8 +70,12 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, * An xdiff error might be our "data->hit" from above. See the * comment for xdiff_emit_line_fn in xdiff-interface.h */ - ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, - &ecbdata, &xpp, &xecfg); + if (ranges) + ret = diff_emit_line_ranges(one, two, ranges, diffgrep_consume, + &ecbdata, &xpp, &xecfg); + else + ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, + &ecbdata, &xpp, &xecfg); if (ecbdata.hit) return 1; if (ret) @@ -119,8 +128,13 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o UNUSED, - regex_t *regexp, kwset_t kws) + regex_t *regexp, kwset_t kws, + const struct range_set *ranges UNUSED) { + /* + * -S counts needle occurrences in each whole blob. Scoping this to + * a -L range is left to a follow-up; for now -S ignores the range. + */ unsigned int c1 = one ? contains(one, regexp, kws, 0) : 0; unsigned int c2 = two ? contains(two, regexp, kws, c1 + 1) : 0; return c1 != c2; @@ -132,6 +146,7 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, struct userdiff_driver *textconv_one = NULL; struct userdiff_driver *textconv_two = NULL; mmfile_t mf1, mf2; + const struct range_set *ranges; int ret; /* ignore unmerged */ @@ -169,7 +184,13 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, mf1.size = fill_textconv(o->repo, textconv_one, p->one, &mf1.ptr); mf2.size = fill_textconv(o->repo, textconv_two, p->two, &mf2.ptr); - ret = fn(&mf1, &mf2, o, regexp, kws); + /* + * -L scopes the search to the tracked range, but the range is in + * original-file line coordinates that do not map onto textconv + * output, so search the whole file when textconv is in play. + */ + ranges = (textconv_one || textconv_two) ? NULL : p->line_ranges; + ret = fn(&mf1, &mf2, o, regexp, kws, ranges); if (textconv_one) free(mf1.ptr); diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index 68576418f41895..bac74d44794778 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -722,9 +722,9 @@ test_expect_success '-L with -S filters to string-count changes' ' test_expect_success '-L with -G filters to diff-text matches' ' git checkout parent-oids && git log -L:func2:file.c -G "F2 [+] 2" --format= >actual && - # -G greps the whole-file diff text, not just the tracked range; - # combined with -L, this selects commits that both touch func2 - # and have "F2 + 2" in their diff. + # -G greps the diff text, and under -L only the lines in the + # tracked range (unlike -S above, which searches the whole file); + # this selects commits whose change to func2 contains "F2 + 2". test $(grep -c "^diff --git" actual) = 1 && grep "F2 + 2" actual ' @@ -1110,4 +1110,70 @@ test_expect_success '--check does not report blank-at-eof outside the range' ' test_grep ! "blank line at EOF" actual ' +test_expect_success '-L -G is scoped to the tracked range' ' + git checkout --orphan grep-scope && + git reset --hard && + cat >gp.c <<-\EOF && + int func1() + { + return ALPHA; + } + + int func2() + { + return BETA; + } + EOF + git add gp.c && + test_tick && + git commit -m "add gp.c" && + sed -e "s/ALPHA/ALPHA2/" -e "s/BETA/BETA2/" gp.c >tmp && + mv tmp gp.c && + git commit -a -m "touch both functions" && + # The commit changes ALPHA (func1) and BETA (func2). Tracking func2, + # -G BETA matches its in-range change; -G ALPHA must not, since ALPHA + # changes only outside the tracked range. + git log -L:func2:gp.c -G BETA --format=%s >actual && + test_grep "touch both functions" actual && + git log -L:func2:gp.c -G ALPHA --format=%s >actual && + test_grep ! "touch both functions" actual +' + +test_expect_success '-L -G searches the whole file under textconv' ' + git checkout --orphan grep-textconv && + git reset --hard && + cat >tc.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add tc.c && + test_tick && + git commit -m "add tc.c" && + # One commit changes func1 and func2; MAGIC lands only in the + # func2 change, outside func1. + sed -e "s/F1/F1 + 1/" -e "s/return F2/return MAGIC/" tc.c >tmp && + mv tmp tc.c && + git commit -a -m "change both funcs" && + echo "tc.c diff=tc" >.gitattributes && + + # Without a textconv driver, -G is scoped to func1, so MAGIC (only + # in the func2 change) does not select the commit. + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_must_be_empty actual && + + # A textconv driver makes the range (original-file line numbers) + # meaningless against the driver output, so -G falls back to the + # whole file and MAGIC now selects the commit. + git config diff.tc.textconv cat && + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_grep "change both funcs" actual +' + test_done diff --git a/xdiff-interface.h b/xdiff-interface.h index 51c88296ed5e68..71e5dffefb8ded 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -46,6 +46,19 @@ int xdi_diff_outf(mmfile_t *mf1, mmfile_t *mf2, xdiff_emit_line_fn line_fn, void *consume_callback_data, xpparam_t const *xpp, xdemitconf_t const *xecfg); + +struct range_set; +/* + * Like xdi_diff_outf(), but forwards only the lines within the given + * (post-image) line ranges to line_fn, as "git log -L" scopes its output. + * Returns line_fn's latched return value (so a consumer can signal a hit + * with a non-zero return), or non-zero on xdiff failure. Defined in + * diff.c (it reuses the line-range filter there). + */ +int diff_emit_line_ranges(mmfile_t *mf1, mmfile_t *mf2, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg); int read_mmfile(mmfile_t *ptr, const char *filename); void read_mmblob(mmfile_t *ptr, struct object_database *odb, const struct object_id *oid); From c6b44920e55a023e8ebad221a2b61f9dace86186 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 25 Jun 2026 10:51:56 -0700 Subject: [PATCH 08/21] gitattributes: document how external diff drivers relate to diff features The "Defining an external diff driver" section explains how to configure diff..command but not how the driver relates to the rest of Git's diff machinery. In particular, the command only replaces the textual patch: word diff, function context, color, and the like cannot apply to its output, while the summary formats, blame, and git log -L do not run it at all and keep using the builtin diff. Spell this out so the scope of an external diff driver is clear. Signed-off-by: Michael Montalbo --- Documentation/gitattributes.adoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index bd76167a45eb71..2c4fbfd7f1495c 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -784,6 +784,16 @@ with the above configuration, i.e. `j-c-diff`, with 7 parameters, just like `GIT_EXTERNAL_DIFF` program is called. See linkgit:git[1] for details. +An external diff driver replaces the patch Git would otherwise +produce for the path: Git runs the command and shows its output in +place of its own. Output features that post-process Git's diff do +not apply to it; word diff, function context (`-W`), `--color-moved`, +and coloring all act on Git's builtin diff, not the driver's output. +The driver is consulted only when Git generates a textual patch. The +summary formats (`--stat`, `--numstat`, `--shortstat`, and +`--dirstat`), `git blame`, and `git log -L` do not run it and +continue to use Git's builtin diff. + If the program is able to ignore certain changes (similar to `git diff --ignore-space-change`), then also set the option `trustExitCode` to true. It is then expected to return exit code 1 if From dfbbc2beb820b3779770726d6e9ec9da498a816a Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 17:48:34 -0700 Subject: [PATCH 09/21] diff: extract a hunk emission seam for providers blame runs its diff by loading both blobs and then calling xdi_diff() with a hunk consumer. The content load happens before the diff request exists as a value anywhere, so a component that could answer "which line ranges changed between this pair" without computing has no point to plug in, and by the time xdiff runs both blobs have been read. Extract the request into diff_provider_emit_hunks(): the caller states the diff parameters and a hunk consumer, and hands over content loading as a callback the seam invokes when it computes. Every request today is computed, so behavior is unchanged. This is the seam where hunk providers will be consulted. A provider that can answer for a pair before its content is loaded, such as a store of precomputed hunks or a long-running external process, plugs in between the request and the computation; later commits add both, and the request will grow the pair's object ids when the first provider that keys on them lands. Blame is the first consumer because it fits the shape: it diffs blob pairs it can name before reading them, so a provider hit can skip both blob reads. Blame's -C/-M split detection diffs partial buffers with no stable pair identity and stays on xdi_diff() directly. Signed-off-by: Michael Montalbo --- Makefile | 1 + blame.c | 32 ++++++++++++++++++++++++-------- diff-provider.c | 16 ++++++++++++++++ diff-provider.h | 30 ++++++++++++++++++++++++++++++ meson.build | 1 + 5 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 diff-provider.c create mode 100644 diff-provider.h diff --git a/Makefile b/Makefile index b31ecb07564a73..b3309ffd1013d0 100644 --- a/Makefile +++ b/Makefile @@ -1140,6 +1140,7 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o LIB_OBJS += diffcore-delta.o diff --git a/blame.c b/blame.c index 977cbb70974f8c..ec90d38b8a0856 100644 --- a/blame.c +++ b/blame.c @@ -23,6 +23,7 @@ #include "commit-slab.h" #include "bloom.h" #include "commit-graph.h" +#include "diff-provider.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1933,6 +1934,25 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +struct blame_diff_fill { + struct blame_scoreboard *sb; + struct blame_origin *parent, *target; + int ignore_diffs; +}; + +/* Content load for diff_provider_emit_hunks(): runs when the diff is computed. */ +static int blame_diff_fill(void *data, mmfile_t *old_file, mmfile_t *new_file) +{ + struct blame_diff_fill *f = data; + + fill_origin_blob(&f->sb->revs->diffopt, f->parent, old_file, + &f->sb->num_read_blob, f->ignore_diffs); + fill_origin_blob(&f->sb->revs->diffopt, f->target, new_file, + &f->sb->num_read_blob, f->ignore_diffs); + f->sb->num_get_patch++; + return 0; +} + /* * We are looking at the origin 'target' and aiming to pass blame * for the lines it is suspected to its parent. Run diff to find @@ -1942,9 +1962,10 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_origin *target, struct blame_origin *parent, int ignore_diffs) { - mmfile_t file_p, file_o; struct blame_chunk_cb_data d; struct blame_entry *newdest = NULL; + struct blame_diff_fill fill = { sb, parent, target, ignore_diffs }; + xpparam_t xpp = { .flags = sb->xdl_opts }; if (!target->suspects) return; /* nothing remains for this target */ @@ -1955,13 +1976,8 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; - fill_origin_blob(&sb->revs->diffopt, parent, &file_p, - &sb->num_read_blob, ignore_diffs); - fill_origin_blob(&sb->revs->diffopt, target, &file_o, - &sb->num_read_blob, ignore_diffs); - sb->num_get_patch++; - - if (diff_hunks(&file_p, &file_o, blame_chunk_cb, &d, sb->xdl_opts)) + if (diff_provider_emit_hunks(&xpp, blame_diff_fill, &fill, + blame_chunk_cb, &d)) die("unable to generate diff (%s -> %s)", oid_to_hex(&parent->commit->object.oid), oid_to_hex(&target->commit->object.oid)); diff --git a/diff-provider.c b/diff-provider.c new file mode 100644 index 00000000000000..1bac70f1cd0799 --- /dev/null +++ b/diff-provider.c @@ -0,0 +1,16 @@ +#include "git-compat-util.h" +#include "diff-provider.h" + +int diff_provider_emit_hunks(const xpparam_t *xpp, + hunk_pair_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + xdemitconf_t xecfg = { .hunk_func = hunk_cb }; + xdemitcb_t ecb = { .priv = cb_data }; + mmfile_t old_file, new_file; + + if (fill(fill_data, &old_file, &new_file) < 0) + return -1; + return xdi_diff(&old_file, &new_file, xpp, &xecfg, &ecb); +} diff --git a/diff-provider.h b/diff-provider.h new file mode 100644 index 00000000000000..b130e217907f6d --- /dev/null +++ b/diff-provider.h @@ -0,0 +1,30 @@ +#ifndef DIFF_PROVIDER_H +#define DIFF_PROVIDER_H + +#include "xdiff-interface.h" + +/* + * The seam between naming a pair of file versions to diff and + * computing their changed line ranges. Consumers that operate on + * hunk coordinates route their diff through here. + */ + +/* + * Load the pair's content. Called at most once per request, only + * when the ranges are computed rather than provided. The buffers + * borrow storage owned by the callback's owner. + */ +typedef int (*hunk_pair_fill_fn)(void *data, mmfile_t *old_file, + mmfile_t *new_file); + +/* + * Emit the exact changed ranges (context 0) for one file pair to + * hunk_cb. xpp carries the diff parameters that determine the + * ranges. Returns 0 on success, -1 on failure to load or diff. + */ +int diff_provider_emit_hunks(const xpparam_t *xpp, + hunk_pair_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +#endif /* DIFF_PROVIDER_H */ diff --git a/meson.build b/meson.build index 064fe2e2f1f4e5..a5a7ca62a4970f 100644 --- a/meson.build +++ b/meson.build @@ -328,6 +328,7 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-provider.c', 'diff.c', 'diffcore-break.c', 'diffcore-delta.c', From ff103db2d251bf48998ec44fa52fb78acfe1f677 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 20:13:14 -0700 Subject: [PATCH 10/21] diff-hunks: add the store format, library, and command Blame and "git log --stat" recover hunk coordinates by diffing blob pairs, and recompute them on every run because the results are not stored. Add a cache of those coordinates under $GIT_DIR/objects so a later run can look them up instead of decompressing the blobs and running xdiff again. The store is a single chunk-format file (see linkgit:gitformat-chunk[5]): an 8-byte header, a DHIX index of fixed-size entries sorted by key, a DHDT segment of hunk records, and a trailing hash checksum. An entry is keyed by the two blob object IDs and the xdl_opts the pair was diffed under, so a stored result is served only where that exact key recurs, independent of path. The hunks of a pair are not unique: a zero-context diff trims, which can pick a different but equally valid set than an untrimmed diff. A recording caller therefore stores a pair only when its trimmed and untrimmed diffs are identical; such an entry answers any consumer at any context, and the rare divergent pair is always computed. Identical hunk blocks are interned once and shared across keys. The library provides a reader (repo_diff_hunks_store and _replay, gated by core.diffHunks and always miss-tolerant), loaded once and cached on the object database as the commit-graph is, and a writer that accumulates entries and flushes them in one atomic pass. Writing is off by default and enabled per run by the GIT_DIFF_HUNKS_WRITE environment variable or the diffHunks.write config, the environment winning; a writer seeds from the existing store so a flush merges rather than replaces, and verifies the seed's checksum first so a corrupt store is discarded rather than rewritten with a fresh checksum that verify could no longer catch. The writer fsyncs through a new diff-hunks core.fsync component. "git diff-hunks" inspects and manages the file: "verify" checks the checksum, chunk table, sort order, and entry bounds, and "clear" removes it. Later patches wire the readers and the writer into the diff and blame paths. Signed-off-by: Michael Montalbo --- .gitignore | 1 + Documentation/Makefile | 1 + Documentation/config.adoc | 2 + Documentation/config/core.adoc | 10 +- Documentation/config/diff-hunks.adoc | 7 + Documentation/git-diff-hunks.adoc | 125 ++++ Documentation/gitformat-diff-hunks.adoc | 122 ++++ Documentation/meson.build | 2 + Makefile | 2 + builtin.h | 1 + builtin/diff-hunks.c | 49 ++ command-list.txt | 2 + diff-hunks.c | 856 ++++++++++++++++++++++++ diff-hunks.h | 113 ++++ diff-provider.c | 23 + diff-provider.h | 30 + environment.c | 1 + git.c | 1 + meson.build | 2 + odb.c | 2 + odb.h | 4 + repo-settings.c | 1 + repo-settings.h | 1 + write-or-die.h | 7 +- 24 files changed, 1362 insertions(+), 3 deletions(-) create mode 100644 Documentation/config/diff-hunks.adoc create mode 100644 Documentation/git-diff-hunks.adoc create mode 100644 Documentation/gitformat-diff-hunks.adoc create mode 100644 builtin/diff-hunks.c create mode 100644 diff-hunks.c create mode 100644 diff-hunks.h diff --git a/.gitignore b/.gitignore index 4da58c6754899e..4173111c01b2b6 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ /git-diagnose /git-diff /git-diff-files +/git-diff-hunks /git-diff-index /git-diff-pairs /git-diff-tree diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..170fcee66e23d5 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,6 +33,7 @@ MAN5_TXT += gitattributes.adoc MAN5_TXT += gitformat-bundle.adoc MAN5_TXT += gitformat-chunk.adoc MAN5_TXT += gitformat-commit-graph.adoc +MAN5_TXT += gitformat-diff-hunks.adoc MAN5_TXT += gitformat-index.adoc MAN5_TXT += gitformat-loose.adoc MAN5_TXT += gitformat-pack.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index a80e7db46d9697..7c006898c434d1 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -417,6 +417,8 @@ include::config/credential.adoc[] include::config/diff.adoc[] +include::config/diff-hunks.adoc[] + include::config/difftool.adoc[] include::config/extensions.adoc[] diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index a0ebf03e2eb050..9595619c610425 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -670,12 +670,13 @@ but risks losing recent work in the event of an unclean system shutdown. * `pack` hardens objects added to the repo in packfile form. * `pack-metadata` hardens packfile bitmaps and indexes. * `commit-graph` hardens the commit-graph file. +* `diff-hunks` hardens the diff-hunks store. * `index` hardens the index when it is modified. * `objects` is an aggregate option that is equivalent to `loose-object,pack`. * `reference` hardens references modified in the repo. * `derived-metadata` is an aggregate option that is equivalent to - `pack-metadata,commit-graph`. + `pack-metadata,commit-graph,diff-hunks`. * `committed` is an aggregate option that is currently equivalent to `objects`. This mode sacrifices some performance to ensure that work that is committed to the repository with `git commit` or similar commands @@ -750,6 +751,13 @@ core.commitGraph:: to parse the graph structure of commits. Defaults to true. See linkgit:git-commit-graph[1] for more information. +core.diffHunks:: + If true, then Git will consult the diff-hunks store (if it + exists) to skip recomputing diff hunk coordinates in commands + such as `git log --stat` and linkgit:git-blame[1]. This controls + only reading; writing the store is controlled by `diffHunks.write`. + See linkgit:git-diff-hunks[1] for more information. Defaults to true. + core.useReplaceRefs:: If set to `false`, behave as if the `--no-replace-objects` option was given on the command line. See linkgit:git[1] and diff --git a/Documentation/config/diff-hunks.adoc b/Documentation/config/diff-hunks.adoc new file mode 100644 index 00000000000000..60a4291e781755 --- /dev/null +++ b/Documentation/config/diff-hunks.adoc @@ -0,0 +1,7 @@ +diffHunks.write:: + If true, diff-producing commands (`git diff` and `git log` with a + `--stat`, `--numstat`, or `--shortstat` format) write the hunks + they compute to the diff-hunks store, filling it as a side effect. + The `GIT_DIFF_HUNKS_WRITE` environment variable overrides this for + a single invocation. Reading the store is controlled separately by + `core.diffHunks`. See linkgit:git-diff-hunks[1]. Defaults to false. diff --git a/Documentation/git-diff-hunks.adoc b/Documentation/git-diff-hunks.adoc new file mode 100644 index 00000000000000..041e8b42322248 --- /dev/null +++ b/Documentation/git-diff-hunks.adoc @@ -0,0 +1,125 @@ +git-diff-hunks(1) +================= + +NAME +---- +git-diff-hunks - Inspect and manage the precomputed diff hunk store + +SYNOPSIS +-------- +[synopsis] +git diff-hunks verify +git diff-hunks clear + +DESCRIPTION +----------- + +The diff hunk store is a cache of diff hunk coordinates, so that commands +which need them, such as linkgit:git-blame[1] and `git log` and `git diff` +with the `--stat`, `--numstat`, and `--shortstat` formats, can skip +decompressing blobs and running the diff algorithm. + +The store is a single file, `$GIT_DIR/objects/diff-hunks`. Reading is +enabled by default; writing is off by default. A `git diff`, `git log`, +`git show`, or `git diff-tree` that produces one of the stat formats +fills the store as a side effect, but only when writing is enabled for +that run (see "WARMING THE STORE" below), so ordinary reads never +modify the repository. When the store does not have the pair, holds a +different object hash, or the file is unreadable, the consumer falls +back to computing the diff. A store only speeds up these commands; it +never changes their output. + +`git diff-hunks` itself only inspects and manages the file. See +linkgit:gitformat-diff-hunks[5] for the file format. + +WARMING THE STORE +----------------- + +The store is filled by running ordinary commands with writing enabled. +Turn writing on for a single invocation with the `GIT_DIFF_HUNKS_WRITE` +environment variable, or persistently with the `diffHunks.write` +configuration; the environment variable takes precedence. A repository +owner warms the store by running the diff-producing commands they care +about with writing on, for example: + + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null + +A `--stat` walk records one entry per blob pair, which serves both +linkgit:git-blame[1] (it replays the coordinates) and the summary +formats (they sum the counts), so a single warming walk serves both. +A warm seeds from the existing store and rewrites the file with the +newly computed pairs merged in, so a later warm adds to what earlier +warms recorded rather than discarding it. + +COMMANDS +-------- + +`verify`:: + Check the integrity of the store: the trailing hash checksum, the + chunk table of contents, the sort order of the index, and the + bounds of every entry. Exits with non-zero status if the store is + corrupt. An absent store is valid. + +`clear`:: + Remove the store file. + +CORRECTNESS +----------- + +A stored result is interchangeable with a freshly computed one because an +entry is keyed by everything that determines the diff: + +* the object IDs of the old and new blob, so a result is used only for + the exact contents it was computed from; and +* the diff algorithm and ignore flags (`xdl_opts`) the hunks were + computed under. A lookup whose `xdl_opts` differ from a stored entry + misses. This is why, for example, `blame -w` and + `--diff-algorithm=` (including a per-path + `diff..algorithm`) do not reuse entries recorded under the + default settings: they change `xdl_opts`. + +The context length is not part of the key because only trim-stable +pairs are recorded: pairs whose zero-context trimmed diff and untrimmed +diff are identical, so one entry answers blame (zero context) and the +summary formats (any context) alike. The rare pair where +`trim_common_tail` picks a different but equally valid set of hunks is +never recorded and is always computed. + +Some options change the hunks in ways that are not part of the key, so +they are excluded from the store in both directions: break detection +(`-B`), `--ignore-matching-lines` (`-I`), `--anchored`, and +`--ignore-blank-lines`. linkgit:git-blame[1] additionally does not +consult the store for reverse blame, ignored revisions, or paths with a +textconv driver. + +The store carries a trailing hash checksum, but readers do not +re-checksum it on every load. As with the commit-graph and +multi-pack-index, the writer fsyncs the file (honoring `core.fsync`) and +commits it atomically, so a committed store is intact; every offset and +count is still bounds-checked as it is read. The checksum is verified by +`git diff-hunks verify`, not on the read path. + +CONFIGURATION +------------- + +`core.diffHunks`:: + Whether commands read the store. Defaults to true. See + linkgit:git-config[1]. + +`diffHunks.write`:: + Whether diff-producing commands write to the store. Defaults to + false. The `GIT_DIFF_HUNKS_WRITE` environment variable overrides it + for a single invocation. See linkgit:git-config[1]. + +Writing the store honors the `core.fsync` configuration through the +`diff-hunks` component; see linkgit:git-config[1]. + +SEE ALSO +-------- +linkgit:git-blame[1], +linkgit:git-log[1], +linkgit:gitformat-diff-hunks[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitformat-diff-hunks.adoc b/Documentation/gitformat-diff-hunks.adoc new file mode 100644 index 00000000000000..70fe2d1f6f801b --- /dev/null +++ b/Documentation/gitformat-diff-hunks.adoc @@ -0,0 +1,122 @@ +gitformat-diff-hunks(5) +======================= + +NAME +---- +gitformat-diff-hunks - Precomputed diff hunk store format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/diff-hunks + +DESCRIPTION +----------- + +The diff-hunks store memoizes diff hunk coordinates so that commands +that need them, such as `git log --stat` and linkgit:git-blame[1], can +skip decompressing blobs and running the diff algorithm. See +linkgit:git-diff-hunks[1] for how the store is filled and managed and the +configuration that controls it. + +The store is a single file, `$GIT_DIR/objects/diff-hunks`, written in +one pass and replaced atomically, so a reader sees either the old file +or the complete new one. + +Entries are keyed by the object IDs of the blob pair that was diffed +and by the diff algorithm and ignore flags (`xdl_opts`) the pair was +diffed under. A blob pair fully determines the diff input, so an entry +is valid regardless of which commits, branches, or index states the +pair was encountered in, and identical diffs performed in different +contexts share one entry. A reader whose `xdl_opts` differ from an +entry does not match it and falls back to computing the diff. + +FILE FORMAT +----------- + +All multi-byte integers are stored in network byte order. The file is an +8-byte header, the chunk table of contents and chunk data described in +linkgit:gitformat-chunk[5], and a trailing checksum. + +HEADER +~~~~~~ + +- 4-byte signature: `DHPF` (diff-hunks precomputed format) +- 1-byte version number: currently 1 +- 1-byte hash version: 1 for SHA-1, 2 for SHA-256. A store whose hash + function differs from the repository's is ignored. +- 1-byte number of chunks +- 1-byte reserved + +CHUNK LOOKUP +~~~~~~~~~~~~ + +A table of contents in the format of linkgit:gitformat-chunk[5], listing +the offset of each chunk. Both chunks below are required; a file missing +either is treated as corrupt. + +CHUNK DATA +~~~~~~~~~~ + +DHIX (index):: + A sorted sequence of fixed-size entries. Each entry is the old + blob object ID, the new blob object ID, a 4-byte `xdl_opts` + value, and a 4-byte offset into the DHDT chunk. Entries are + sorted by old object ID, then new object ID, then `xdl_opts`, + so lookups can use binary search on the full key. + +DHDT (hunk data):: + For each index entry, at its offset: a 4-byte hunk count followed + by that many 16-byte hunk records. A hunk record is four 4-byte + values: old start, old count, new start, new count, as produced + by the xdiff hunk callback. Identical hunk blocks are stored once: + distinct index entries whose recorded hunks are byte-for-byte + equal point at the same offset. + +TRAILER +~~~~~~~ + +A checksum of all preceding bytes, computed with the repository hash +function. + +CORRECTNESS +----------- + +Serving hunks from a valid store produces the same output as recomputing +the diff. The diff of a blob pair is not unique: a zero context length +triggers trim_common_tail, which can pick a different but equally valid +set of hunks than an untrimmed diff does. A pair is therefore recorded +only when its trimmed and untrimmed diffs are identical, which holds for +the overwhelming majority of pairs. Such an entry answers any consumer +at any context: git-blame replays its coordinates directly (it diffs at +zero context), and diffstat sums its per-hunk line counts, which the +context length does not change. The rare pair whose two diffs differ is +never recorded, so every consumer computes it. + +A store that cannot be used is ignored, and the consumer falls back to +computing the diff. Every offset and count read from the file is +bounds-checked, so a store that is missing, truncated, of an unknown +version, or of a different object hash does not change the diff output +(a malformed store may still produce a diagnostic on stderr). + +The store is not re-checksummed on the read path. The writer fsyncs the +file (honoring `core.fsync`) and commits it atomically, so a +committed store is intact, the same trust model the commit-graph and +multi-pack-index use. The trailing checksum is recomputed by +`git diff-hunks verify` to detect corruption. + +The checksum detects corruption but does not prove who wrote the file. A +reader trusts the coordinates in a store that passes its checks, so +anything able to write a checksum-valid file at the store path can +influence output, the same as it could by writing objects directly. + +LIMITATIONS +----------- + +- Hunk counts, offsets, and line coordinates are 32-bit, capping a + single entry at roughly four billion hunks and the hunk data at 4 GiB. + A result whose coordinates cannot be represented is not recorded. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..85f37da47e9cac 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -41,6 +41,7 @@ manpages = { 'git-describe.adoc' : 1, 'git-diagnose.adoc' : 1, 'git-diff-files.adoc' : 1, + 'git-diff-hunks.adoc' : 1, 'git-diff-index.adoc' : 1, 'git-diff-pairs.adoc' : 1, 'git-difftool.adoc' : 1, @@ -175,6 +176,7 @@ manpages = { 'gitformat-bundle.adoc' : 5, 'gitformat-chunk.adoc' : 5, 'gitformat-commit-graph.adoc' : 5, + 'gitformat-diff-hunks.adoc' : 5, 'gitformat-index.adoc' : 5, 'gitformat-loose.adoc' : 5, 'gitformat-pack.adoc' : 5, diff --git a/Makefile b/Makefile index b3309ffd1013d0..efaafe1e9348ab 100644 --- a/Makefile +++ b/Makefile @@ -1148,6 +1148,7 @@ LIB_OBJS += diffcore-order.o LIB_OBJS += diffcore-pickaxe.o LIB_OBJS += diffcore-rename.o LIB_OBJS += diffcore-rotate.o +LIB_OBJS += diff-hunks.o LIB_OBJS += dir-iterator.o LIB_OBJS += dir.o LIB_OBJS += editor.o @@ -1407,6 +1408,7 @@ BUILTIN_OBJS += builtin/credential.o BUILTIN_OBJS += builtin/describe.o BUILTIN_OBJS += builtin/diagnose.o BUILTIN_OBJS += builtin/diff-files.o +BUILTIN_OBJS += builtin/diff-hunks.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-pairs.o BUILTIN_OBJS += builtin/diff-tree.o diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..7e64da9f433836 100644 --- a/builtin.h +++ b/builtin.h @@ -175,6 +175,7 @@ int cmd_credential_store(int argc, const char **argv, const char *prefix, struct int cmd_describe(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diagnose(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_files(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_index(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_pairs(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/diff-hunks.c b/builtin/diff-hunks.c new file mode 100644 index 00000000000000..d5b1b42c3b8450 --- /dev/null +++ b/builtin/diff-hunks.c @@ -0,0 +1,49 @@ +#include "builtin.h" +#include "config.h" +#include "diff-hunks.h" +#include "gettext.h" +#include "parse-options.h" +#include "repository.h" + +static const char * const diff_hunks_usage[] = { + N_("git diff-hunks verify"), + N_("git diff-hunks clear"), + NULL +}; + +static int cmd_diff_hunks_verify(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + return diff_hunks_verify(r) ? 1 : 0; +} + +static int cmd_diff_hunks_clear(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + return diff_hunks_clear(r) ? 1 : 0; +} + +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, + struct repository *repo) +{ + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("verify", &fn, cmd_diff_hunks_verify), + OPT_SUBCOMMAND("clear", &fn, cmd_diff_hunks_clear), + OPT_END() + }; + + repo_config(repo, git_default_config, NULL); + + argc = parse_options(argc, argv, prefix, options, diff_hunks_usage, 0); + + return fn(argc, argv, prefix, repo); +} diff --git a/command-list.txt b/command-list.txt index 21b802c42026b3..e7b241e6ad8498 100644 --- a/command-list.txt +++ b/command-list.txt @@ -95,6 +95,7 @@ git-describe mainporcelain git-diagnose ancillaryinterrogators git-diff mainporcelain info git-diff-files plumbinginterrogators +git-diff-hunks plumbingmanipulators git-diff-index plumbinginterrogators git-diff-pairs plumbinginterrogators git-diff-tree plumbinginterrogators @@ -223,6 +224,7 @@ gitfaq guide gitformat-bundle developerinterfaces gitformat-chunk developerinterfaces gitformat-commit-graph developerinterfaces +gitformat-diff-hunks developerinterfaces gitformat-index developerinterfaces gitformat-pack developerinterfaces gitformat-signature developerinterfaces diff --git a/diff-hunks.c b/diff-hunks.c new file mode 100644 index 00000000000000..454950f8c0a0fe --- /dev/null +++ b/diff-hunks.c @@ -0,0 +1,856 @@ +/* + * Precomputed diff hunks, keyed by diff input. + * + * A single store at .git/objects/diff-hunks maps an (old blob, new + * blob, xdl_opts) key to the hunk coordinates of diffing the pair. + * The key determines the diff result (only trim-stable pairs are + * recorded; see diff-hunks.h), so an entry is valid in any context it + * recurs in, independent of path. Reading is always on; writing is off + * by default and enabled per run (see diff_hunks_write_enabled), so an + * ordinary command populates the store only during a warming run the + * repository owner opts into. + * + * File layout: + * Header: "DHPF"(4) + version(1) + hash_version(1) + num_chunks(1) + reserved(1) + * Table of contents (chunk-format) + * DHIX chunk: sorted entries, each + * old_blob_oid, new_blob_oid, xdl_opts(4), hdat_offset(4) + * DHDT chunk: per entry, num_hunks(4) followed by that many 16-byte hunks + * Trailing hash checksum + */ +#include "git-compat-util.h" +#include "chunk-format.h" +#include "config.h" +#include "csum-file.h" +#include "diff-hunks.h" +#include "diff-provider.h" +#include "gettext.h" +#include "hash.h" +#include "hashmap.h" +#include "lockfile.h" +#include "odb.h" +#include "path.h" +#include "repo-settings.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +#define DIFF_HUNKS_SIGNATURE 0x44485046 /* "DHPF" */ +/* + * Bump when the on-disk format changes, or when xdiff's emitted hunk + * coordinates change for a fixed (blobs, xdl_opts) key: an old store + * would otherwise serve stale hunks and change command output. + */ +#define DIFF_HUNKS_VERSION 1 +#define DIFF_HUNKS_HEADER_SIZE 8 + +#define DIFF_HUNKS_CHUNKID_INDEX 0x44484958 /* "DHIX" */ +#define DIFF_HUNKS_CHUNKID_DATA 0x44484454 /* "DHDT" */ + +/* Each hunk is 16 bytes on disk: old_start(4) old_count(4) new_start(4) new_count(4) */ +#define DIFF_HUNKS_HUNK_SIZE (4 * sizeof(uint32_t)) + +/* + * Result of a store lookup: num_hunks records encoded in the store's mmap, + * valid until the store is freed. Read them with nth_precomputed_hunk(). + */ +struct precomputed_entry { + uint32_t num_hunks; + const unsigned char *hunk_data; +}; + +/* Decode a single hunk from the raw on-disk format. */ +static inline void decode_precomputed_hunk(const unsigned char *data, + struct precomputed_hunk *h) +{ + h->old_start = get_be32(data); + h->old_count = get_be32(data + 4); + h->new_start = get_be32(data + 8); + h->new_count = get_be32(data + 12); +} + +/* Decode the nth hunk of a lookup result into *h. */ +static inline void nth_precomputed_hunk(const struct precomputed_entry *e, + uint32_t n, struct precomputed_hunk *h) +{ + decode_precomputed_hunk(e->hunk_data + (size_t)n * DIFF_HUNKS_HUNK_SIZE, h); +} + +/* Byte length of the (old_oid, new_oid, xdl_opts) lookup key. */ +static size_t store_index_key_size(const struct git_hash_algo *algo) +{ + return 2 * algo->rawsz + sizeof(uint32_t); +} + +/* Index entry: the lookup key followed by the 4-byte offset into DHDT. */ +static size_t store_index_entry_size(const struct git_hash_algo *algo) +{ + return store_index_key_size(algo) + sizeof(uint32_t); +} + +/* + * The smallest a valid store file can be: the header, a table of contents + * with one entry per chunk plus a terminating entry, and the trailing + * checksum. + */ +static size_t store_min_size(const struct git_hash_algo *algo, + uint8_t num_chunks) +{ + size_t toc_size = (num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE; + + return DIFF_HUNKS_HEADER_SIZE + toc_size + algo->rawsz; +} + +/* + * Decode an index entry's key into pointers to the two oids and the + * xdl_opts value (on-disk: old_oid, new_oid, then xdl_opts as a + * big-endian uint32). + */ +static void decode_store_index_key(const unsigned char *entry, unsigned int rawsz, + const unsigned char **old_hash, + const unsigned char **new_hash, + uint32_t *xdl_opts) +{ + *old_hash = entry; + *new_hash = entry + rawsz; + *xdl_opts = get_be32(entry + 2 * rawsz); +} + +/* The DHDT offset stored in an index entry, in the field after its key. */ +static uint32_t index_entry_hdat_offset(const unsigned char *entry, size_t keysz) +{ + return get_be32(entry + keysz); +} + +static char *store_path_ext(struct repository *r, const char *ext) +{ + struct strbuf sb = STRBUF_INIT; + + strbuf_addf(&sb, "%s/diff-hunks%s", repo_get_object_directory(r), ext); + return strbuf_detach(&sb, NULL); +} + +static char *diff_hunks_store_path(struct repository *r) +{ + return store_path_ext(r, ""); +} + +struct diff_hunks_store { + const unsigned char *data; + size_t data_len; + const struct git_hash_algo *hash_algo; + const unsigned char *index; + uint32_t num_entries; + const unsigned char *hdat; + size_t hdat_size; +}; + +static void free_store(struct diff_hunks_store *s) +{ + if (!s) + return; + if (s->data) + munmap((void *)s->data, s->data_len); + free(s); +} + +/* + * Open, mmap, and parse the store at fname. Returns the parsed store + * or NULL on any error. The diff output is unaffected either way; + * corruption is reported by verify, not treated as fatal here. + */ +static struct diff_hunks_store *load_store_at( + const struct git_hash_algo *repo_algo, const char *fname) +{ + struct diff_hunks_store *s; + struct chunkfile *cf; + int fd; + struct stat st; + void *data; + const unsigned char *p; + uint8_t num_chunks; + size_t index_size, entry_size, data_len; + + fd = git_open(fname); + if (fd < 0) + return NULL; + if (fstat(fd, &st) || st.st_size < DIFF_HUNKS_HEADER_SIZE) { + close(fd); + return NULL; + } + data_len = xsize_t(st.st_size); + data = xmmap(NULL, data_len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + p = data; + + num_chunks = p[6]; + + /* + * Reject a file that is not a readable store: wrong signature, + * version, or object hash, or too small to hold the table of + * contents that read_table_of_contents() walks (it dereferences + * each entry before range-checking its offset). + */ + if (get_be32(p) != DIFF_HUNKS_SIGNATURE || + p[4] != DIFF_HUNKS_VERSION || + p[5] != oid_version(repo_algo) || + data_len < store_min_size(repo_algo, num_chunks)) { + munmap(data, data_len); + return NULL; + } + + /* + * The trailing checksum is not verified here: the writer fsyncs + * and commits atomically, so a committed file is intact, and + * every record is bounds-checked at read (see precomputed_entry_at). + * The checksum is checked separately, by diff_hunks_verify(). + */ + + CALLOC_ARRAY(s, 1); + s->data = data; + s->data_len = data_len; + s->hash_algo = repo_algo; + + cf = init_chunkfile(NULL); + if (read_table_of_contents(cf, p, data_len, + DIFF_HUNKS_HEADER_SIZE, num_chunks, 1) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, &s->index, &index_size) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, &s->hdat, &s->hdat_size)) { + free_chunkfile(cf); + goto corrupt; + } + free_chunkfile(cf); + + entry_size = store_index_entry_size(s->hash_algo); + if (index_size % entry_size) + goto corrupt; + s->num_entries = index_size / entry_size; + return s; + +corrupt: + free_store(s); + return NULL; +} + +static struct diff_hunks_store *diff_hunks_store_load(struct repository *r) +{ + struct diff_hunks_store *s; + char *fname; + + prepare_repo_settings(r); + if (!r->settings.core_diff_hunks) + return NULL; + + fname = diff_hunks_store_path(r); + s = load_store_at(r->hash_algo, fname); + free(fname); + return s; +} + +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r) +{ + if (!r->objects) + return NULL; + if (r->objects->diff_hunks_store_attempted) + return r->objects->diff_hunks_store; + r->objects->diff_hunks_store_attempted = 1; + r->objects->diff_hunks_store = diff_hunks_store_load(r); + return r->objects->diff_hunks_store; +} + +void close_diff_hunks_store(struct object_database *o) +{ + if (!o->diff_hunks_store) + return; + free_store(o->diff_hunks_store); + o->diff_hunks_store = NULL; +} + +/* + * Fill *out with the hunk record at offset in the data chunk, and return + * 1 if the record is in bounds, 0 otherwise. The read path does not + * re-verify the checksum, and a valid checksum would not bound the count + * anyway, so a read must call this and use *out only when it returns + * non-zero. + * + * A record is a be32 hunk count followed by that many DIFF_HUNKS_HUNK_SIZE + * hunks. "remaining" tracks the bytes from offset to the end of the data + * chunk: it must hold the count, and after the count is consumed it must + * hold every hunk. The bounds are written as subtraction and division + * (never addition or multiplication) so a crafted offset or count cannot + * overflow them. + */ +static int precomputed_entry_at(const struct diff_hunks_store *s, + uint32_t offset, struct precomputed_entry *out) +{ + size_t remaining; + uint32_t num_hunks; + + if (offset >= s->hdat_size) + return 0; + remaining = s->hdat_size - offset; + if (remaining < sizeof(uint32_t)) + return 0; + + num_hunks = get_be32(s->hdat + offset); + remaining -= sizeof(uint32_t); + if (num_hunks > remaining / DIFF_HUNKS_HUNK_SIZE) + return 0; + + out->num_hunks = num_hunks; + out->hunk_data = s->hdat + offset + sizeof(uint32_t); + return 1; +} + +struct lookup_key { + const struct object_id *old_oid; + const struct object_id *new_oid; + int xdl_opts; + unsigned int rawsz; +}; + +/* + * The store's total order over (old_oid, new_oid, xdl_opts), defined + * once so the write-side sort (writer_entry_cmp) and the read-side + * search (store_bsearch_cmp) order the keys identically. + */ +static int cmp_store_index_key(const unsigned char *old_a, const unsigned char *new_a, + uint32_t opts_a, + const unsigned char *old_b, const unsigned char *new_b, + uint32_t opts_b, unsigned int rawsz) +{ + int cmp = memcmp(old_a, old_b, rawsz); + if (!cmp) + cmp = memcmp(new_a, new_b, rawsz); + if (!cmp) + cmp = (opts_a > opts_b) - (opts_a < opts_b); + return cmp; +} + +static int store_bsearch_cmp(const void *key, const void *entry_ptr) +{ + const struct lookup_key *k = key; + const unsigned char *old_hash, *new_hash; + uint32_t xdl_opts; + + decode_store_index_key(entry_ptr, k->rawsz, &old_hash, &new_hash, + &xdl_opts); + return cmp_store_index_key(k->old_oid->hash, k->new_oid->hash, + (uint32_t)k->xdl_opts, + old_hash, new_hash, xdl_opts, k->rawsz); +} + +static int store_get_one(struct diff_hunks_store *s, const struct lookup_key *key, + struct precomputed_entry *out) +{ + size_t entry_size = store_index_entry_size(s->hash_algo); + const unsigned char *found; + + found = bsearch(key, s->index, s->num_entries, entry_size, + store_bsearch_cmp); + if (!found) + return 0; + return precomputed_entry_at(s, + index_entry_hdat_offset(found, store_index_key_size(s->hash_algo)), + out); +} + +static int diff_hunks_store_get(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + struct precomputed_entry *out) +{ + struct lookup_key key; + + if (!s) + return 0; + /* The null OID names no blob and cannot key an entry. */ + if (is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + + key.old_oid = old_oid; + key.new_oid = new_oid; + key.xdl_opts = xdl_opts; + key.rawsz = s->hash_algo->rawsz; + + return store_get_one(s, &key, out); +} + +/* + * A recorded hunk sequence must satisfy the seam's well-formedness + * rule (diff_provider_hunks_check()) before it may be replayed: + * coordinates decode from be32 into long, which is 32-bit on some + * platforms, so a crafted value can decode negative or out of order. + * An entry that fails reads as a miss, so the caller recomputes. + */ +static int replayable_hunks(const struct precomputed_entry *e) +{ + struct diff_provider_hunks_check c = { 0 }; + uint32_t i; + + for (i = 0; i < e->num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(e, i, &h); + if (diff_provider_hunks_check(&c, h.old_start, h.old_count, + h.new_start, h.new_count)) + return 0; + } + return 1; +} + +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data) +{ + struct precomputed_entry e; + uint32_t i; + + if (!diff_hunks_store_get(s, old_oid, new_oid, xdl_opts, &e) || + !replayable_hunks(&e)) + return 0; + for (i = 0; i < e.num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(&e, i, &h); + hunk_func(h.old_start, h.old_count, + h.new_start, h.new_count, cb_data); + } + return 1; +} + +/* Validate one store file. Returns 0 if valid or absent, -1 if corrupt. */ +static int verify_store_at(struct repository *r, const char *fname) +{ + struct diff_hunks_store *s; + size_t entry_size; + uint32_t i; + int ret = 0; + + if (access(fname, F_OK)) + return 0; /* absent is valid */ + s = load_store_at(r->hash_algo, fname); + if (!s) + return error(_("diff-hunks store failed to load (corrupt " + "header or hash mismatch): %s"), fname); + if (!hashfile_checksum_valid(r->hash_algo, s->data, s->data_len)) { + error(_("diff-hunks store has incorrect checksum and is " + "likely corrupt: %s"), fname); + free_store(s); + return -1; + } + + entry_size = store_index_entry_size(s->hash_algo); + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + size_t keysz = store_index_key_size(s->hash_algo); + uint32_t offset = index_entry_hdat_offset(ep, keysz); + struct precomputed_entry pe; + + /* + * Keyed by (old_oid, new_oid, xdl_opts, context), increasing. + * memcmp matches cmp_store_index_key's integer comparison of + * xdl_opts and context because both are non-negative, so their + * big-endian bytes order the same as their values. + */ + if (i > 0 && memcmp(ep - entry_size, ep, keysz) >= 0) { + error(_("diff-hunks entry %u not in sorted order"), i); + ret = -1; + } + if (!precomputed_entry_at(s, offset, &pe)) { + error(_("diff-hunks entry %u has out-of-bounds hunk " + "data"), i); + ret = -1; + } + } + + free_store(s); + return ret; +} + +int diff_hunks_verify(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (verify_store_at(r, fname)) + ret = -1; + free(fname); + return ret; +} + +int diff_hunks_clear(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (unlink(fname) && errno != ENOENT) + ret = error_errno(_("unable to remove %s"), fname); + free(fname); + return ret; +} + +struct writer_entry { + struct object_id old_oid; + struct object_id new_oid; + int xdl_opts; + uint32_t hdat_offset; +}; + +struct diff_hunks_writer { + struct repository *r; + struct writer_entry *entries; + size_t nr, alloc; + size_t seed_nr; /* nr after seeding; finish skips a no-op flush */ + struct strbuf hdat; + struct hashmap dedup; /* hunk block content -> offset in hdat */ +}; + +/* A record of one distinct hunk block already present in hdat. */ +struct dedup_entry { + struct hashmap_entry ent; + uint32_t offset; + uint32_t len; +}; + +static int dedup_cmp(const void *cmp_data, + const struct hashmap_entry *a, + const struct hashmap_entry *b, + const void *keydata UNUSED) +{ + const struct diff_hunks_writer *writer = cmp_data; + const struct dedup_entry *ea = container_of(a, const struct dedup_entry, ent); + const struct dedup_entry *eb = container_of(b, const struct dedup_entry, ent); + + if (ea->len != eb->len) + return 1; + return memcmp(writer->hdat.buf + ea->offset, + writer->hdat.buf + eb->offset, ea->len); +} + +static struct diff_hunks_writer *diff_hunks_writer_new(struct repository *r) +{ + struct diff_hunks_writer *w; + + CALLOC_ARRAY(w, 1); + w->r = r; + strbuf_init(&w->hdat, 0); + hashmap_init(&w->dedup, dedup_cmp, w, 0); + return w; +} + +static void strbuf_put_be32(struct strbuf *sb, uint32_t val) +{ + unsigned char buf[4]; + put_be32(buf, val); + strbuf_add(sb, buf, 4); +} + +/* + * The hunk block just appended at `start` is deduplicated: if an + * identical block is already in hdat, this copy is dropped and the + * earlier offset returned; otherwise it is kept and remembered. + * Distinct keys that diff to the same hunks then share one block. + */ +static uint32_t intern_block(struct diff_hunks_writer *w, size_t start) +{ + size_t len = w->hdat.len - start; + struct dedup_entry key, *found, *added; + + hashmap_entry_init(&key.ent, memhash(w->hdat.buf + start, len)); + key.offset = (uint32_t)start; + key.len = (uint32_t)len; + + found = hashmap_get_entry(&w->dedup, &key, ent, NULL); + if (found) { + strbuf_setlen(&w->hdat, start); + return found->offset; + } + + added = xmalloc(sizeof(*added)); + hashmap_entry_init(&added->ent, key.ent.hash); + added->offset = key.offset; + added->len = key.len; + hashmap_add(&w->dedup, &added->ent); + return key.offset; +} + +void diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks) +{ + struct writer_entry *e; + size_t i, block_start; + + /* + * The block appended for this entry is sizeof(uint32_t) + + * nr_hunks * DIFF_HUNKS_HUNK_SIZE bytes. Bound nr_hunks so that + * length fits the uint32_t the dedup index records (and so the + * count itself fits the uint32_t written to the store). + */ + if (!nr_hunks || + nr_hunks > (UINT32_MAX - sizeof(uint32_t)) / DIFF_HUNKS_HUNK_SIZE || + is_null_oid(old_oid) || is_null_oid(new_oid)) + return; + if (w->hdat.len > UINT32_MAX) + return; + /* + * Coordinates are stored as 32-bit values; a result that cannot + * round-trip is dropped rather than silently truncated. + */ + for (i = 0; i < nr_hunks; i++) + if ((uintmax_t)hunks[i].old_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].old_count > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_count > (uintmax_t)INT32_MAX) + return; + + ALLOC_GROW(w->entries, w->nr + 1, w->alloc); + e = &w->entries[w->nr++]; + oidcpy(&e->old_oid, old_oid); + oidcpy(&e->new_oid, new_oid); + e->xdl_opts = xdl_opts; + + block_start = w->hdat.len; + strbuf_put_be32(&w->hdat, (uint32_t)nr_hunks); + for (i = 0; i < nr_hunks; i++) { + strbuf_put_be32(&w->hdat, hunks[i].old_start); + strbuf_put_be32(&w->hdat, hunks[i].old_count); + strbuf_put_be32(&w->hdat, hunks[i].new_start); + strbuf_put_be32(&w->hdat, hunks[i].new_count); + } + e->hdat_offset = intern_block(w, block_start); +} + +/* + * Seed the writer with fname's entries so a rewrite preserves them. Returns + * 0 when the file is absent or seeded, and -1 when it is present but fails + * its checksum. A rewrite re-checksums, so a corrupt source must not be + * carried forward: that would launder the corruption into a checksum-valid + * file that verify can no longer catch. This path already reads the whole + * file, so verify it here (the reader stays trust-at-read and fast) and + * discard it on mismatch: a warm then proceeds without it and rewrites + * a clean store from what it computes. + */ +static int diff_hunks_writer_seed(struct diff_hunks_writer *w, const char *fname) +{ + struct diff_hunks_store *s = load_store_at(w->r->hash_algo, fname); + unsigned int rawsz; + size_t entry_size, keysz; + struct precomputed_hunk *hunks = NULL; + size_t hunks_alloc = 0; + uint32_t i; + + if (!s) + return 0; + if (!hashfile_checksum_valid(w->r->hash_algo, s->data, s->data_len)) { + warning(_("diff-hunks store %s failed its checksum; " + "discarding it"), fname); + free_store(s); + return -1; + } + rawsz = s->hash_algo->rawsz; + entry_size = store_index_entry_size(s->hash_algo); + keysz = store_index_key_size(s->hash_algo); + + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + const unsigned char *old_hash, *new_hash; + struct object_id old_oid, new_oid; + uint32_t xdl_opts, j; + struct precomputed_entry pe; + + decode_store_index_key(ep, rawsz, &old_hash, &new_hash, + &xdl_opts); + oidread(&old_oid, old_hash, s->hash_algo); + oidread(&new_oid, new_hash, s->hash_algo); + if (!precomputed_entry_at(s, index_entry_hdat_offset(ep, keysz), &pe)) + continue; + ALLOC_GROW(hunks, pe.num_hunks, hunks_alloc); + for (j = 0; j < pe.num_hunks; j++) + nth_precomputed_hunk(&pe, j, &hunks[j]); + diff_hunks_writer_add(w, &old_oid, &new_oid, (int)xdl_opts, + hunks, pe.num_hunks); + } + free(hunks); + free_store(s); + return 0; +} + +/* + * Writing is off by default. It is enabled per invocation by the + * GIT_DIFF_HUNKS_WRITE environment variable, or persistently by the + * diffHunks.write config, with the environment variable winning when + * set. Only a warming run (a diff or log the repository owner chooses + * to run with writing on) enables it, so ordinary reads never mutate + * the store. + */ +static int diff_hunks_write_enabled(struct repository *r) +{ + const char *env = getenv("GIT_DIFF_HUNKS_WRITE"); + int val; + + if (env) { + /* + * This is a warming opt-in, so an unparseable value must not + * abort an ordinary read command: treat it as disabled. + */ + val = git_parse_maybe_bool(env); + return val < 0 ? 0 : val; + } + if (!repo_config_get_bool(r, "diffhunks.write", &val)) + return val; + return 0; +} + +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r) +{ + struct diff_hunks_writer *w; + char *fname; + + if (!diff_hunks_write_enabled(r)) + return NULL; + /* + * Seed from the existing store so a flush merges with it rather + * than replacing it: a later warm adds newly computed pairs + * without discarding what earlier warms recorded. + */ + w = diff_hunks_writer_new(r); + fname = diff_hunks_store_path(r); + diff_hunks_writer_seed(w, fname); + free(fname); + w->seed_nr = w->nr; + return w; +} + +static int writer_entry_cmp(const void *va, const void *vb, void *ctx) +{ + const struct writer_entry *a = va, *b = vb; + unsigned int rawsz = *(const unsigned int *)ctx; + return cmp_store_index_key(a->old_oid.hash, a->new_oid.hash, + (uint32_t)a->xdl_opts, + b->old_oid.hash, b->new_oid.hash, + (uint32_t)b->xdl_opts, + rawsz); +} + +struct write_ctx { + struct diff_hunks_writer *w; + unsigned int rawsz; +}; + +static int write_index_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + size_t i; + + for (i = 0; i < ctx->w->nr; i++) { + hashwrite(f, ctx->w->entries[i].old_oid.hash, ctx->rawsz); + hashwrite(f, ctx->w->entries[i].new_oid.hash, ctx->rawsz); + hashwrite_be32(f, ctx->w->entries[i].xdl_opts); + hashwrite_be32(f, ctx->w->entries[i].hdat_offset); + } + return 0; +} + +static int write_data_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + hashwrite(f, ctx->w->hdat.buf, ctx->w->hdat.len); + return 0; +} + +/* Sort, dedup, and write the accumulated entries to the file at fname. */ +static int diff_hunks_writer_flush(struct diff_hunks_writer *w, char *fname) +{ + struct lock_file lk = LOCK_INIT; + struct hashfile *f; + struct chunkfile *cf; + unsigned int rawsz = w->r->hash_algo->rawsz; + struct write_ctx ctx = { w, rawsz }; + size_t entry_size; + + QSORT_S(w->entries, w->nr, writer_entry_cmp, &rawsz); + + /* + * The same blob pair recurs across history (reverts, cherry- + * picks); identical keys carry identical hunks, so keep one of + * each. The index must stay duplicate-free for binary search. + */ + if (w->nr > 1) { + size_t kept = 1, i; + for (i = 1; i < w->nr; i++) + if (writer_entry_cmp(&w->entries[kept - 1], + &w->entries[i], &rawsz)) + w->entries[kept++] = w->entries[i]; + w->nr = kept; + } + + if (safe_create_leading_directories(w->r, fname)) { + error(_("unable to create directory for %s"), fname); + return -1; + } + if (hold_lock_file_for_update(&lk, fname, 0) < 0) { + error_errno(_("unable to lock %s"), fname); + return -1; + } + adjust_shared_perm(w->r, get_lock_file_path(&lk)); + f = hashfd(w->r->hash_algo, get_lock_file_fd(&lk), + get_lock_file_path(&lk)); + + entry_size = store_index_entry_size(w->r->hash_algo); + cf = init_chunkfile(f); + add_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, w->nr * entry_size, + write_index_chunk); + add_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, w->hdat.len, write_data_chunk); + + hashwrite_be32(f, DIFF_HUNKS_SIGNATURE); + hashwrite_u8(f, DIFF_HUNKS_VERSION); + hashwrite_u8(f, oid_version(w->r->hash_algo)); + hashwrite_u8(f, get_num_chunks(cf)); + hashwrite_u8(f, 0); /* reserved */ + + write_chunkfile(cf, &ctx); + free_chunkfile(cf); + + /* + * fsync per the user's configuration (like commit-graph and the + * multi-pack-index), then commit atomically. Readers trust the + * committed file rather than re-checksumming it; diff_hunks_verify() + * checks the checksum separately. + */ + finalize_hashfile(f, NULL, FSYNC_COMPONENT_DIFF_HUNKS, + CSUM_HASH_IN_STREAM); + if (commit_lock_file(&lk)) { + error_errno(_("unable to write %s"), fname); + return -1; + } + return 0; +} + +static void diff_hunks_writer_free(struct diff_hunks_writer *w) +{ + if (!w) + return; + hashmap_clear_and_free(&w->dedup, struct dedup_entry, ent); + free(w->entries); + strbuf_release(&w->hdat); + free(w); +} + +void diff_hunks_writer_finish(struct diff_hunks_writer *w) +{ + if (!w) + return; + /* Skip the flush when the warm recorded nothing beyond its seed. */ + if (w->nr != w->seed_nr) { + char *fname = diff_hunks_store_path(w->r); + diff_hunks_writer_flush(w, fname); + free(fname); + } + diff_hunks_writer_free(w); +} diff --git a/diff-hunks.h b/diff-hunks.h new file mode 100644 index 00000000000000..6bf39df38878fc --- /dev/null +++ b/diff-hunks.h @@ -0,0 +1,113 @@ +#ifndef DIFF_HUNKS_H +#define DIFF_HUNKS_H + +#include "hash.h" +#include "xdiff-interface.h" /* xdl_emit_hunk_consume_func_t */ + +struct object_id; +struct repository; +struct object_database; + +/* + * A persistent store of precomputed diff hunk coordinates, at + * .git/objects/diff-hunks. Entries are keyed by the two blobs diffed + * and the xdl_opts they were diffed under, so a cached result is valid + * in any context that key recurs in, independent of path. The xdl_opts + * key component mirrors the (always non-negative) diff_options field it + * projects from, and is serialized and compared as a 4-byte big-endian + * integer. + * + * The hunks a pair produces are not unique. They vary with the xdiff + * algorithm and ignore flags (xdl_opts, part of the key), and with + * whether the diff was trimmed: a zero-context diff runs + * trim_common_tail, which can pick a different but equally valid set of + * hunks than an untrimmed diff. The store holds one entry per key, so a + * pair is recorded only when its trimmed and untrimmed diffs are + * identical (the recording caller checks); such an entry serves a + * consumer at any context. The rare pair where the two diffs differ is + * never recorded and is always computed. + * + * The store is a cache: ordinary commands read it and fall back to + * computing the diff when it is absent, stale, or corrupt. It is filled + * as a side effect of diff and log runs, but only when writing is + * enabled; writing is off by default, so an ordinary command reads the + * store without recording into it. + */ + +/* + * A hunk's coordinates. The type is long to match the xdiff emit + * callback; the values are a diff's line numbers and counts, always + * within the int32 range the on-disk format stores (see + * diff_hunks_writer_add()). + */ +struct precomputed_hunk { + long old_start; + long old_count; + long new_start; + long new_count; +}; + +/* + * The repository's store, loaded once on first use and cached on the + * object database. Returns NULL when reading is disabled + * (core.diffHunks=false), the store is absent, or its object hash + * differs. The lookup functions below accept a NULL store and treat it + * as empty (every lookup misses), so callers need not check for NULL. + * The object database owns the store; callers must not free it. + */ +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r); + +/* Free the repository's cached store, at object-database teardown. */ +void close_diff_hunks_store(struct object_database *o); + +/* + * Replay the recorded hunks of an (old blob, new blob) pair diffed + * under xdl_opts through hunk_func. The sequence is validated before + * any callback runs: on a hit (return 1) every hunk is emitted, on a + * miss (return 0: absent pair, xdl_opts mismatch, or an entry that + * fails validation) nothing is emitted, so a caller may accumulate + * directly into its result. + */ +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data); + +/* + * A warming run's writer: it accumulates the hunks it computes in memory + * and flushes them to the store in one pass at finish. + */ +struct diff_hunks_writer; + +/* + * Return a writer for a warming run, or NULL when writing is disabled + * (the default). diff_hunks_writer_add() tolerates a NULL writer, so a + * caller may attach the result unconditionally. Pair with + * diff_hunks_writer_finish(). + */ +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r); + +/* + * Record a blob pair's hunks as computed under xdl_opts; a later lookup + * with a matching key is served these hunks. The caller must have + * checked that the pair's trimmed and untrimmed diffs are identical + * (see the top of this file), so the entry answers at any context. + * NULL-safe. + */ +void diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks); + +/* Flush the accumulated entries to the store and free the writer. NULL-safe. */ +void diff_hunks_writer_finish(struct diff_hunks_writer *w); + +/* Remove the store file. Returns 0 (incl. absent) or -1. */ +int diff_hunks_clear(struct repository *r); +/* Validate the store. Returns 0 if valid/absent, -1 if corrupt. */ +int diff_hunks_verify(struct repository *r); + +#endif /* DIFF_HUNKS_H */ diff --git a/diff-provider.c b/diff-provider.c index 1bac70f1cd0799..fd7e2c55994548 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -1,6 +1,29 @@ #include "git-compat-util.h" #include "diff-provider.h" +enum diff_provider_hunks_error +diff_provider_hunks_check(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count) +{ + if (old_start < 0 || old_count < 0 || + new_start < 0 || new_count < 0 || + old_start > INT32_MAX || old_count > INT32_MAX || + new_start > INT32_MAX || new_count > INT32_MAX) + return PROVIDER_HUNKS_RANGE; + if (old_start < c->prev_old_end || new_start < c->prev_new_end) + return PROVIDER_HUNKS_OVERLAP; + if (old_start - c->prev_old_end != new_start - c->prev_new_end) + return PROVIDER_HUNKS_MISALIGNED; + /* + * With each field bounded to int32 above, these sums cannot + * overflow the int64 running state even where long is 32-bit. + */ + c->prev_old_end = (int64_t)old_start + old_count; + c->prev_new_end = (int64_t)new_start + new_count; + return PROVIDER_HUNKS_OK; +} + int diff_provider_emit_hunks(const xpparam_t *xpp, hunk_pair_fill_fn fill, void *fill_data, xdl_emit_hunk_consume_func_t hunk_cb, diff --git a/diff-provider.h b/diff-provider.h index b130e217907f6d..682f58096a1681 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -9,6 +9,36 @@ * hunk coordinates route their diff through here. */ +/* + * Incremental well-formedness check for a provider-supplied hunk + * sequence, shared by every provider. Each coordinate must fit int32 + * (a consumer may truncate to int, and a provider may serialize as + * such); hunks must be in order and must not overlap; and the + * unchanged run between hunks must be the same length on both sides, + * or a consumer that walks the two files in lockstep desynchronizes. + * Every rule constrains differences between coordinates, so the check + * applies to 0-based and 1-based sequences alike. + * + * Feed the hunks in order to a zero-initialized struct; the first + * nonzero return names the violated rule, and the whole sequence must + * then be discarded unemitted. + */ +struct diff_provider_hunks_check { + int64_t prev_old_end, prev_new_end; +}; + +enum diff_provider_hunks_error { + PROVIDER_HUNKS_OK = 0, + PROVIDER_HUNKS_RANGE, /* negative or beyond int32 */ + PROVIDER_HUNKS_OVERLAP, /* out of order or overlapping */ + PROVIDER_HUNKS_MISALIGNED, /* unchanged runs differ in length */ +}; + +enum diff_provider_hunks_error +diff_provider_hunks_check(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count); + /* * Load the pair's content. Called at most once per request, only * when the ranges are computed rather than provided. The buffers diff --git a/environment.c b/environment.c index fc3ed8bb1c7a66..a28b22ae0b2c9d 100644 --- a/environment.c +++ b/environment.c @@ -233,6 +233,7 @@ static const struct fsync_component_name { { "pack", FSYNC_COMPONENT_PACK }, { "pack-metadata", FSYNC_COMPONENT_PACK_METADATA }, { "commit-graph", FSYNC_COMPONENT_COMMIT_GRAPH }, + { "diff-hunks", FSYNC_COMPONENT_DIFF_HUNKS }, { "index", FSYNC_COMPONENT_INDEX }, { "objects", FSYNC_COMPONENTS_OBJECTS }, { "reference", FSYNC_COMPONENT_REFERENCE }, diff --git a/git.c b/git.c index 36f08891ef5476..09c1ae73ac9ab7 100644 --- a/git.c +++ b/git.c @@ -566,6 +566,7 @@ static struct cmd_struct commands[] = { { "diagnose", cmd_diagnose, RUN_SETUP_GENTLY }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, + { "diff-hunks", cmd_diff_hunks, RUN_SETUP }, { "diff-index", cmd_diff_index, RUN_SETUP | NO_PARSEOPT }, { "diff-pairs", cmd_diff_pairs, RUN_SETUP | NO_PARSEOPT }, { "diff-tree", cmd_diff_tree, RUN_SETUP | NO_PARSEOPT }, diff --git a/meson.build b/meson.build index a5a7ca62a4970f..8bb39cb06d38b3 100644 --- a/meson.build +++ b/meson.build @@ -336,6 +336,7 @@ libgit_sources = [ 'diffcore-pickaxe.c', 'diffcore-rename.c', 'diffcore-rotate.c', + 'diff-hunks.c', 'dir-iterator.c', 'dir.c', 'editor.c', @@ -613,6 +614,7 @@ builtin_sources = [ 'builtin/describe.c', 'builtin/diagnose.c', 'builtin/diff-files.c', + 'builtin/diff-hunks.c', 'builtin/diff-index.c', 'builtin/diff-pairs.c', 'builtin/diff-tree.c', diff --git a/odb.c b/odb.c index 965ef68e4eca22..0c69e1cea04a91 100644 --- a/odb.c +++ b/odb.c @@ -2,6 +2,7 @@ #include "abspath.h" #include "commit-graph.h" #include "config.h" +#include "diff-hunks.h" #include "dir.h" #include "environment.h" #include "gettext.h" @@ -1033,6 +1034,7 @@ void odb_close(struct object_database *o) for (source = o->sources; source; source = source->next) odb_source_close(source); close_commit_graph(o); + close_diff_hunks_store(o); } static void odb_free_sources(struct object_database *o) diff --git a/odb.h b/odb.h index 73553ed5a7b1ea..450f087e4e2e26 100644 --- a/odb.h +++ b/odb.h @@ -8,6 +8,7 @@ #include "thread-utils.h" struct cached_object_entry; +struct diff_hunks_store; struct odb_source_inmemory; struct packed_git; struct repository; @@ -75,6 +76,9 @@ struct object_database { struct commit_graph *commit_graph; unsigned commit_graph_attempted : 1; /* if loading has been attempted */ + struct diff_hunks_store *diff_hunks_store; + unsigned diff_hunks_store_attempted : 1; /* if loading has been attempted */ + /* * This is meant to hold a *small* number of objects that you would * want odb_read_object() to be able to return, but yet you do not want diff --git a/repo-settings.c b/repo-settings.c index 208e09ff17fcee..e1282cdaa1fa56 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -77,6 +77,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "pack.usepathwalk", &r->settings.pack_use_path_walk, 0); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); + repo_cfg_bool(r, "core.diffhunks", &r->settings.core_diff_hunks, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); diff --git a/repo-settings.h b/repo-settings.h index cad9c3f0cc15f3..2b1987c02fa6a7 100644 --- a/repo-settings.h +++ b/repo-settings.h @@ -29,6 +29,7 @@ struct repo_settings { int core_commit_graph; int commit_graph_generation_version; int commit_graph_changed_paths_version; + int core_diff_hunks; int gc_write_commit_graph; int fetch_write_commit_graph; int command_requires_full_index; diff --git a/write-or-die.h b/write-or-die.h index ff0408bd849fd8..35ed324307a21b 100644 --- a/write-or-die.h +++ b/write-or-die.h @@ -22,13 +22,15 @@ enum fsync_component { FSYNC_COMPONENT_INDEX = 1 << 4, FSYNC_COMPONENT_REFERENCE = 1 << 5, FSYNC_COMPONENT_OBJECT_MAP = 1 << 6, + FSYNC_COMPONENT_DIFF_HUNKS = 1 << 7, }; #define FSYNC_COMPONENTS_OBJECTS (FSYNC_COMPONENT_LOOSE_OBJECT | \ FSYNC_COMPONENT_PACK) #define FSYNC_COMPONENTS_DERIVED_METADATA (FSYNC_COMPONENT_PACK_METADATA | \ - FSYNC_COMPONENT_COMMIT_GRAPH) + FSYNC_COMPONENT_COMMIT_GRAPH | \ + FSYNC_COMPONENT_DIFF_HUNKS) #define FSYNC_COMPONENTS_DEFAULT ((FSYNC_COMPONENTS_OBJECTS | \ FSYNC_COMPONENTS_DERIVED_METADATA) & \ @@ -46,7 +48,8 @@ enum fsync_component { FSYNC_COMPONENT_COMMIT_GRAPH | \ FSYNC_COMPONENT_INDEX | \ FSYNC_COMPONENT_REFERENCE | \ - FSYNC_COMPONENT_OBJECT_MAP) + FSYNC_COMPONENT_OBJECT_MAP | \ + FSYNC_COMPONENT_DIFF_HUNKS) #ifndef FSYNC_COMPONENTS_PLATFORM_DEFAULT #define FSYNC_COMPONENTS_PLATFORM_DEFAULT FSYNC_COMPONENTS_DEFAULT From 63ff08f263974dab976b35947757ae7e1c67868d Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 20:13:14 -0700 Subject: [PATCH 11/21] diff: record and read precomputed hunks for stat output Teach builtin_diffstat() to consult the hunk provider seam. On a hit it sums the provided hunk counts instead of decompressing the blobs and running xdiff; on a miss it computes the diff as before, and when a writer is attached it records what it computed. It records both the zero-context hunks (what blame reads) and the untrimmed hunks keyed by the configured context (what diffstat sums), so one pass serves both readers. diff_provider_query_hunks() is the seam's consult-only entry: it asks providers for the pair's ranges under the settings that key them, without loading content, and a miss leaves the consumer's own computation untouched. The diff-hunks store is the provider it consults, via diff_hunks_replay(), which validates a recorded sequence before any hunk reaches the consumer's callback; the summing callback here accumulates directly into the diffstat entry on the strength of that ordering. "git diff", "git log", "git show", and "git diff-tree" with the --stat, --numstat, and --shortstat formats consult the seam; when writing is enabled they also attach a writer, then flush it when the walk finishes. This is how the store is populated: a warming run such as GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null fills the cache as a side effect of the diff work the command already does. Reading is controlled by core.diffHunks and writing by diffHunks.write and GIT_DIFF_HUNKS_WRITE. A read-path hit is invisible in the output, so it is counted and emitted as a trace2 "read-hits" datum for tests and tuning. diff_hunks_settings_from_diffopt() is the single place that projects a diff_options down to the settings that key the store. Inputs that perturb the hunks outside the key (-B, -I, --anchored, and --ignore-blank-lines) disable both lookup and recording, so output stays identical to a store-less run. A "log -L" range-scoped stat is not the whole-pair diff the key describes, so it neither reads nor records; the line-range filter computes it as before. Add t4218 covering output parity with and without the store for the stat formats at several context lengths, the write gate (off by default, the environment overriding the config), the trim-divergent pair that is never recorded, the settings that must bypass the store, warm merging and corrupt-store discard, and verify and clear. Signed-off-by: Michael Montalbo --- builtin/diff-tree.c | 3 + builtin/diff.c | 10 + builtin/log.c | 7 + diff-provider.c | 19 ++ diff-provider.h | 31 +++ diff.c | 284 ++++++++++++++++++++++++--- diff.h | 34 ++++ t/meson.build | 1 + t/t4218-diff-hunks.sh | 379 +++++++++++++++++++++++++++++++++++++ t/t4218/trim-divergent-new | 319 +++++++++++++++++++++++++++++++ t/t4218/trim-divergent-old | 316 +++++++++++++++++++++++++++++++ 11 files changed, 1374 insertions(+), 29 deletions(-) create mode 100755 t/t4218-diff-hunks.sh create mode 100644 t/t4218/trim-divergent-new create mode 100644 t/t4218/trim-divergent-old diff --git a/builtin/diff-tree.c b/builtin/diff-tree.c index 8b8f8b54e40664..296c6a137e23c6 100644 --- a/builtin/diff-tree.c +++ b/builtin/diff-tree.c @@ -170,6 +170,8 @@ int cmd_diff_tree(int argc, opt->diffopt.rotate_to_strict = 1; + diff_hunks_attach(&opt->diffopt); + /* * NOTE! We expect "a..b" to expand to "^a b" but it is * perfectly valid for revision range parser to yield "b ^a", @@ -234,5 +236,6 @@ int cmd_diff_tree(int argc, diff_free(&opt->diffopt); } + diff_hunks_detach(&opt->diffopt); return diff_result_code(opt); } diff --git a/builtin/diff.c b/builtin/diff.c index 4b46e394cecb8d..931d9d8bbe8f39 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -568,6 +568,15 @@ int cmd_diff(int argc, } } + /* + * The hunk store is keyed by blob pair, so any diff whose + * file pairs carry known blob object IDs (tree-to-tree, + * index-to-tree) can consult the same entries that + * "git log --stat" and "git blame" use; pairs without known + * blobs bypass it at lookup time. + */ + diff_hunks_attach(&rev.diffopt); + symdiff_prepare(&rev, &sdiff); for (i = 0; i < rev.pending.nr; i++) { struct object_array_entry *entry = &rev.pending.objects[i]; @@ -644,6 +653,7 @@ int cmd_diff(int argc, result = diff_result_code(&rev); if (1 < rev.diffopt.skip_stat_unmatch) refresh_index_quietly(); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); object_array_clear(&ent); symdiff_release(&sdiff); diff --git a/builtin/log.c b/builtin/log.c index e464b30af4bcae..05aebb87b5177f 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -693,8 +693,11 @@ int cmd_show(int argc, opt.tweak = show_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + if (!rev.no_walk) { ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -767,6 +770,7 @@ int cmd_show(int argc, } rev.diffopt.no_free = 0; + diff_hunks_detach(&rev.diffopt); diff_free(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); @@ -846,8 +850,11 @@ int cmd_log(int argc, opt.tweak = log_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; diff --git a/diff-provider.c b/diff-provider.c index fd7e2c55994548..3426920397cc62 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -1,5 +1,24 @@ #include "git-compat-util.h" #include "diff-provider.h" +#include "diff-hunks.h" + +int diff_provider_active(struct repository *r) +{ + return !!repo_diff_hunks_store(r); +} + +int diff_provider_query_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + if (!old_oid || !new_oid) + return 0; + return diff_hunks_replay(repo_diff_hunks_store(r), old_oid, new_oid, + xdl_opts, hunk_cb, cb_data); +} enum diff_provider_hunks_error diff_provider_hunks_check(struct diff_provider_hunks_check *c, diff --git a/diff-provider.h b/diff-provider.h index 682f58096a1681..2424d0daea0cda 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -7,7 +7,38 @@ * The seam between naming a pair of file versions to diff and * computing their changed line ranges. Consumers that operate on * hunk coordinates route their diff through here. + * + * A hunk provider answers a consumer's request from the pair's blob + * object ids and the settings that determine the diff, before any + * content is loaded; a request no provider answers falls through to + * the consumer's own computation. The diff-hunks store (diff-hunks.h) + * is the provider consulted today. + */ + +struct object_id; +struct repository; + +/* + * Nonzero when a hunk provider is available for the repository, for + * consumers that decide up front whether consulting can pay off. + */ +int diff_provider_active(struct repository *r); + +/* + * Consult hunk providers for the changed ranges of the blob pair + * (old_oid, new_oid) diffed under xdl_opts, without loading content. + * On a hit the ranges are emitted through hunk_cb and 1 is returned; + * nothing is emitted before the provider's answer is validated, so a + * consumer may accumulate directly into its result. 0 is a miss: no + * provider answered, and the consumer computes its diff as it would + * without providers. */ +int diff_provider_query_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); /* * Incremental well-formedness check for a provider-supplied hunk diff --git a/diff.c b/diff.c index a8f346621b9fb5..88a678520e6f30 100644 --- a/diff.c +++ b/diff.c @@ -16,6 +16,8 @@ #include "revision.h" #include "quote.h" #include "diff.h" +#include "diff-hunks.h" +#include "diff-provider.h" #include "diffcore.h" #include "delta.h" #include "hex.h" @@ -34,6 +36,7 @@ #include "tmp-objdir.h" #include "graph.h" #include "oid-array.h" +#include "trace2.h" #include "packfile.h" #include "pager.h" #include "parse-options.h" @@ -2929,6 +2932,75 @@ static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat, return x; } +struct diffstat_hunk_cb_data { + struct precomputed_hunk **h; + size_t *nr, *alloc; +}; + +/* + * Hunk callback that appends each hunk's coordinates to a growable + * array, so one xdiff pass can both sum a diffstat and record hunks for + * the store. + */ +static int diffstat_hunk_cb(long start_a, long count_a, + long start_b, long count_b, + void *cb_data) +{ + struct diffstat_hunk_cb_data *d = cb_data; + + ALLOC_GROW(*d->h, *d->nr + 1, *d->alloc); + (*d->h)[*d->nr].old_start = start_a; + (*d->h)[*d->nr].old_count = count_a; + (*d->h)[*d->nr].new_start = start_b; + (*d->h)[*d->nr].new_count = count_b; + (*d->nr)++; + return 0; +} + +/* + * Collect the hunks of the two files at zero context. diff_fn chooses + * whether trimming runs: xdi_diff applies trim_common_tail, yielding the + * zero-context hunks blame reads; xdl_diff does not, yielding the + * untrimmed hunks. Both run at zero context, so the untrimmed hunks are + * not grouped the way a nonzero context would group them; diffstat only + * sums their counts, which grouping does not change. Sets *ph (caller + * frees) and *ph_nr. + */ +typedef int (*xdiff_fn)(mmfile_t *, mmfile_t *, xpparam_t const *, + xdemitconf_t const *, xdemitcb_t *); +static int collect_hunks(xdiff_fn diff_fn, mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, struct precomputed_hunk **ph, + size_t *ph_nr) +{ + size_t ph_alloc = 0; + xdemitcb_t ecb = { 0 }; + xdemitconf_t xecfg = { 0 }; + struct diffstat_hunk_cb_data cd = { ph, ph_nr, &ph_alloc }; + + *ph = NULL; + *ph_nr = 0; + xecfg.hunk_func = diffstat_hunk_cb; + ecb.priv = &cd; + return diff_fn(mf1, mf2, xpp, &xecfg, &ecb); +} + +void diff_hunks_attach(struct diff_options *o) +{ + if (!(o->output_format & + (DIFF_FORMAT_DIFFSTAT | DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_NUMSTAT))) + return; + o->hunks_writer = diff_hunks_writer_maybe_new(o->repo); +} + +void diff_hunks_detach(struct diff_options *o) +{ + if (o->hunks_read_hits) + trace2_data_intmax("diff-hunks", o->repo, "read-hits", + o->hunks_read_hits); + diff_hunks_writer_finish(o->hunks_writer); + o->hunks_writer = NULL; +} + static int diffstat_consume(void *priv, char *line, unsigned long len) { struct diffstat_t *diffstat = priv; @@ -4252,6 +4324,148 @@ static const char *get_compact_summary(const struct diff_filepair *p, int is_ren return NULL; } +/* Hunk callback for the provider seam: sum counts into a diffstat entry. */ +static int diffstat_sum_hunk_cb(long start_a UNUSED, long count_a, + long start_b UNUSED, long count_b, + void *cb_data) +{ + struct diffstat_file *data = cb_data; + + data->added += count_b; + data->deleted += count_a; + return 0; +} + +/* + * Fill data->added/deleted for a modified pair from the hunk provider seam: on + * a hit, sum the provided counts; on a warming run, compute and record them. + * Returns 1 when it produced the counts, 0 when neither a provider nor the + * writer is usable for this pair and the caller must compute the diffstat + * itself. + * + * The store is keyed by (old blob, new blob, xdl_opts), so it may only serve + * or receive pairs whose result is determined by that key alone. Inputs that + * perturb the hunks outside the key (-B, -I, --anchored) must be off. + * --ignore-blank-lines coalesces hunks differently between the emit and + * hunk-callback paths, so it is excluded to keep output identical to a + * store-less run. (--inter-hunk-context is not excluded: it only groups hunks, + * and diffstat sums their counts, which grouping does not change.) Both sides + * must be valid regular files with known blob IDs. blame applies the same rule + * to its own perturbing inputs. The -I and --anchored exclusions here, plus + * xdl_opts in the key, cover every field of xpparam_t; the assert in the + * function body checks that at compile time. + */ +static int diffstat_from_hunks(struct diff_options *o, + struct diff_filespec *one, + struct diff_filespec *two, + struct diffstat_file *data) +{ + struct precomputed_hunk *ph_trim, *ph_full, *counts; + size_t n_trim, n_full, n_counts, k; + int stable; + mmfile_t mf1, mf2; + xpparam_t xpp; + + /* + * xpparam_t is the diff algorithm's input. Its flags are the key's + * xdl_opts; ignore_regex (-I) and anchors (--anchored) are instead + * excluded by the guard below. + * + * Adding an xpparam_t field fires this assert (its size no longer + * matches the reference struct). To clear it: (1) add the field to + * the reference struct below; then (2) decide how it affects the + * key -- make it part of the key, or exclude diffs that use it in + * the guard below. The assert only tracks size: a same-size reorder + * or a changed field meaning slips past, so re-read the fields when + * it fires. + */ + (void)BUILD_ASSERT_OR_ZERO(sizeof(xpparam_t) == sizeof(struct { + unsigned long flags; + regex_t **ignore_regex; + size_t ignore_regex_nr; + char **anchors; + size_t anchors_nr; + })); + + if (!((diff_provider_active(o->repo) || o->hunks_writer) && + o->break_opt == -1 && + !o->ignore_regex_nr && + !o->anchors_nr && + !(o->xdl_opts & XDF_IGNORE_BLANK_LINES) && + DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two) && + S_ISREG(one->mode) && S_ISREG(two->mode) && + one->oid_valid && two->oid_valid)) + return 0; + + /* + * On a provider hit, sum hunk counts directly without decompressing + * blobs or running xdiff. The consult keys on the current + * diff_options, so a hit carries the hunks a provider-less run would + * have produced; the answer is validated before the first count + * reaches data. + */ + if (diff_provider_query_hunks(o->repo, &one->oid, &two->oid, + o->xdl_opts, + diffstat_sum_hunk_cb, data)) { + o->hunks_read_hits++; + return 1; + } + + /* A miss on a read-only run: let the caller compute the diffstat. */ + if (!o->hunks_writer) + return 0; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + memset(&xpp, 0, sizeof(xpp)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + + /* + * Compute the zero-context trimmed diff (what blame reads) and the + * untrimmed diff (whose counts a nonzero-context stat matches). + * xdi_diff runs first: it enforces the size limit, so the xdl_diff + * call is already bounded. + */ + if (collect_hunks(xdi_diff, &mf1, &mf2, &xpp, &ph_trim, &n_trim) || + collect_hunks(xdl_diff, &mf1, &mf2, &xpp, &ph_full, &n_full)) + die("unable to generate diffstat for %s", one->path); + + /* + * Match a store-less run: at zero context xdi_diff trims, so sum the + * trimmed diff; otherwise sum the untrimmed one. + */ + counts = o->context ? ph_full : ph_trim; + n_counts = o->context ? n_full : n_trim; + for (k = 0; k < n_counts; k++) { + data->added += counts[k].new_count; + data->deleted += counts[k].old_count; + } + + /* + * Record only a trim-stable pair, one whose trimmed and untrimmed + * diffs are identical, so the single entry answers any consumer at + * any context (see diff-hunks.h). A pair where the two differ is + * never recorded and every consumer computes it. + */ + stable = n_trim == n_full; + for (k = 0; stable && k < n_trim; k++) + stable = ph_trim[k].old_start == ph_full[k].old_start && + ph_trim[k].old_count == ph_full[k].old_count && + ph_trim[k].new_start == ph_full[k].new_start && + ph_trim[k].new_count == ph_full[k].new_count; + if (stable) + diff_hunks_writer_add(o->hunks_writer, &one->oid, &two->oid, + o->xdl_opts, ph_trim, n_trim); + free(ph_trim); + free(ph_full); + return 1; +} + static void builtin_diffstat(const char *name_a, const char *name_b, struct diff_filespec *one, struct diff_filespec *two, @@ -4303,38 +4517,50 @@ static void builtin_diffstat(const char *name_a, const char *name_b, } else if (may_differ) { - /* Crazy xdl interfaces.. */ - xpparam_t xpp; - xdemitconf_t xecfg; - - if (fill_mmfile(o->repo, &mf1, one) < 0 || - fill_mmfile(o->repo, &mf2, two) < 0) - die("unable to read files to diff"); - - memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xpp.flags = o->xdl_opts; - xpp.ignore_regex = o->ignore_regex; - xpp.ignore_regex_nr = o->ignore_regex_nr; - xpp.anchors = o->anchors; - xpp.anchors_nr = o->anchors_nr; - xecfg.ctxlen = o->context; - xecfg.interhunkctxlen = o->interhunkcontext; - xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - - if (p->line_ranges) { - struct line_range_filter lr_filter; - - line_range_filter_init(&lr_filter, p->line_ranges, - diffstat_consume, diffstat); + /* + * Serve or record via the diff-hunks store. A "log -L" + * range-scoped stat is not the whole-pair diff the store + * keys, so it neither reads nor records. Otherwise diff + * normally. + */ + if (p->line_ranges || !diffstat_from_hunks(o, one, two, data)) { + /* Crazy xdl interfaces.. */ + xpparam_t xpp; + xdemitconf_t xecfg; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + memset(&xpp, 0, sizeof(xpp)); + memset(&xecfg, 0, sizeof(xecfg)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + xecfg.ctxlen = o->context; + xecfg.interhunkctxlen = o->interhunkcontext; + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (line_range_filter_diff(&lr_filter, &mf1, &mf2, - &xpp, &xecfg)) + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, + p->line_ranges, + diffstat_consume, + diffstat); + + if (line_range_filter_diff(&lr_filter, &mf1, + &mf2, &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, + diffstat_consume, diffstat, + &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); - } else if (xdi_diff_outf(&mf1, &mf2, NULL, - diffstat_consume, diffstat, &xpp, &xecfg)) - die("unable to generate diffstat for %s", one->path); + } if (DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two)) { struct diffstat_file *file = diff --git a/diff.h b/diff.h index bb5cddaf3499e9..1098b2db082823 100644 --- a/diff.h +++ b/diff.h @@ -224,6 +224,17 @@ static inline void diff_flags_or(struct diff_flags *a, #define DIFF_WITH_ALG(opts, flag) (((opts)->xdl_opts & ~XDF_DIFF_ALGORITHM_MASK) | XDF_##flag) +/* + * The xdl_opts bits git turns on by default that a from-scratch xdl_opts + * (git blame's own option parsing) does not set, and so must OR in to match + * a store warmed at the default diff settings; a diff_options-based consumer + * (diffstat) already has them in o->xdl_opts. Today this is only the indent + * heuristic. It does NOT cover a non-default diff.algorithm: a repo that + * configures one records under that algorithm, and a consumer keying without + * it misses (a lost hit, not wrong output). + */ +#define DIFF_HUNKS_DEFAULT_XDL_OPTS XDF_INDENT_HEURISTIC + enum diff_words_type { DIFF_WORDS_NONE = 0, DIFF_WORDS_PORCELAIN, @@ -420,6 +431,18 @@ struct diff_options { */ int max_depth; int max_depth_valid; + + /* + * Precomputed diff hunks (see diff-hunks.h). diffstat consults the + * repository's store, repo_diff_hunks_store(), before running xdiff, + * keyed by each file pair's blob object IDs. When hunks_writer is set + * (a warming run), diffstat also records the hunks it computes; the + * writer is attached only for the stat output formats. hunks_read_hits + * counts store hits on the read path, emitted via trace2 so tests can + * confirm the reader is consulted. + */ + struct diff_hunks_writer *hunks_writer; + int hunks_read_hits; }; unsigned diff_filter_bit(char status); @@ -668,6 +691,17 @@ void diffcore_fix_diff_index(void); int diff_queue_is_empty(struct diff_options *o); void diff_flush(struct diff_options*); void diff_free(struct diff_options*); + +/* + * Attach a diff-hunks writer to a diff producing a stat format, so a + * warming run records the hunks it computes; a no-op when writing is off + * or for other formats. (Reads consult the repository's store directly; + * see repo_diff_hunks_store() in diff-hunks.h.) Pair with + * diff_hunks_detach() once the diff is done. + */ +void diff_hunks_attach(struct diff_options *o); +void diff_hunks_detach(struct diff_options *o); + void diff_warn_rename_limit(const char *varname, int needed, int degraded_cc); /* diff-raw status letters */ diff --git a/t/meson.build b/t/meson.build index c5832fee053561..71c219f2129f84 100644 --- a/t/meson.build +++ b/t/meson.build @@ -576,6 +576,7 @@ integration_tests = [ 't4215-log-skewed-merges.sh', 't4216-log-bloom.sh', 't4217-log-limit.sh', + 't4218-diff-hunks.sh', 't4252-am-options.sh', 't4253-am-keep-cr-dos.sh', 't4254-am-corrupt.sh', diff --git a/t/t4218-diff-hunks.sh b/t/t4218-diff-hunks.sh new file mode 100755 index 00000000000000..bb16ec0a15b582 --- /dev/null +++ b/t/t4218-diff-hunks.sh @@ -0,0 +1,379 @@ +#!/bin/sh + +test_description='precomputed diff hunks store (git diff-hunks) + +The store maps an (old blob, new blob, diff settings) key to the hunks of +diffing the pair. It is a cache: reading is always on, while writing is +off by default and enabled per run by GIT_DIFF_HUNKS_WRITE (or the +diffHunks.write config), so a diff or log warms the store only when the +owner opts in. These tests check that a warmed store never changes +output, that lookups honor the diff settings, and that a corrupt store is +read as absent while verify reports the corruption.' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +STORE=.git/objects/diff-hunks + +# Warm the store the way a repository owner would: a stat walk with +# writing enabled. A --stat walk records one entry per trim-stable blob +# pair, serving blame and the summary formats alike. Extra arguments +# (e.g. -c options) are passed to git before "log". +warm () { + GIT_DIFF_HUNKS_WRITE=1 git "$@" log --all --stat >/dev/null +} + +# Run a command with the store disabled, for ground truth. +no_store () { + git -c core.diffhunks=false "$@" +} + +test_expect_success 'setup' ' + test_commit initial file.txt "line 1" && + test_commit second file.txt "line 1 +line 2" && + test_commit third file.txt "line 1 +line 2 +line 3" && + test_commit fourth file.txt "changed line 1 +line 2 +line 3 +line 4" +' + +test_expect_success 'ordinary commands do not create the store' ' + git log --stat >/dev/null && + git blame file.txt >/dev/null && + git diff --stat second third >/dev/null && + test_path_is_missing $STORE +' + +test_expect_success 'writing is gated by env and config, env wins' ' + test_when_finished "git diff-hunks clear" && + # The diffHunks.write config enables writing. + git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_file $STORE && + git diff-hunks clear && + # GIT_DIFF_HUNKS_WRITE overrides the config: 0 disables it. + GIT_DIFF_HUNKS_WRITE=0 git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_missing $STORE && + # and enables it without any config. + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null && + test_path_is_file $STORE +' + +test_expect_success 'a warm builds a store that verifies' ' + warm && + test_path_is_file $STORE && + git diff-hunks verify +' + +test_expect_success 'a second warming run refreshes the store in place' ' + warm && + test_commit fifth file.txt "brand new line" && + warm && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'log --stat matches with and without the store' ' + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'log --numstat and --shortstat match' ' + no_store log --numstat >expect_num && + no_store log --shortstat >expect_short && + warm && + git log --numstat >actual_num && + git log --shortstat >actual_short && + test_cmp expect_num actual_num && + test_cmp expect_short actual_short +' + +# A built store must reproduce diffstat output at every context length: +# the built context is served, and an unbuilt one falls back to xdiff. +# Zero context is where trim_common_tail runs, so it is keyed apart from +# the nonzero contexts. +test_expect_success 'diffstat matches at several context lengths' ' + no_store log --stat >expect_def && + no_store log -U0 --stat >expect_u0 && + no_store log -U7 --stat >expect_u7 && + warm && + git log --stat >got_def && + git log -U0 --stat >got_u0 && + git log -U7 --stat >got_u7 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + test_cmp expect_u7 got_u7 +' + +test_expect_success 'store built at a nonzero context serves that context' ' + no_store -c diff.context=5 log --stat >expect && + warm -c diff.context=5 && + git -c diff.context=5 log --stat >actual && + test_cmp expect actual +' + +# This blob pair (a real git test file being modernized) decomposes to +# fewer changed lines at zero context, where trim_common_tail runs, than +# at nonzero context: "diff -U0" reports 9/6, "diff -U3" reports 10/7. +# Keying the contexts apart is what keeps each reader correct. The split +# is an emergent property of the whole ~300-line file and does not reduce +# to a few lines, so the pair is shipped as a fixture under t4218/. +test_expect_success 'a trim-divergent file is correct at each context' ' + cp "$TEST_DIRECTORY/t4218/trim-divergent-old" div.sh && + git add div.sh && + git commit -m divergent-old && + cp "$TEST_DIRECTORY/t4218/trim-divergent-new" div.sh && + git add div.sh && + git commit -m divergent-new && + no_store log -1 --format= --stat -- div.sh >expect_def && + no_store log -1 --format= -U0 --stat -- div.sh >expect_u0 && + warm && + git log -1 --format= --stat -- div.sh >got_def && + git log -1 --format= -U0 --stat -- div.sh >got_u0 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + # The fixture must actually diverge, or the test would pass without + # exercising the split; fail loudly if a diff change ever levels it. + ! test_cmp expect_def expect_u0 +' + +# A warming run displays the diffstat it computes. At zero context xdi_diff +# trims, so the displayed counts must be the trimmed ones (what a store-less +# run shows), not the untrimmed counts recorded for a nonzero-context reader. +test_expect_success 'warming --stat at zero context matches a store-less run' ' + git init -q warm-u0 && + ( + cd warm-u0 && + cp "$TEST_DIRECTORY/t4218/trim-divergent-old" div.sh && + git add div.sh && git commit -q -m old && + cp "$TEST_DIRECTORY/t4218/trim-divergent-new" div.sh && + git add div.sh && git commit -q -m new && + git -c core.diffhunks=false log -1 --format= -U0 --stat -- div.sh >expect && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --format= -U0 --stat -- div.sh >got && + test_cmp expect got + ) +' + +test_expect_success 'diff --stat matches with and without the store, both directions' ' + no_store diff --stat second fourth >expect_fwd && + no_store diff --stat fourth second >expect_rev && + warm && + git diff --stat second fourth >got_fwd && + git diff --stat fourth second >got_rev && + test_cmp expect_fwd got_fwd && + test_cmp expect_rev got_rev +' + +test_expect_success 'show and diff-tree --stat use the store' ' + test_when_finished "git diff-hunks clear" && + # The store attaches: a gated show/diff-tree --stat records into the + # store (without the attach there is no writer, so nothing is written). + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git show --stat fourth >/dev/null && + test_path_is_file "$STORE" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff-tree --stat fourth >/dev/null && + test_path_is_file "$STORE" && + # Reading never changes their output. + git diff-hunks clear && + no_store show --stat fourth >expect_show && + no_store diff-tree --stat fourth >expect_dt && + warm && + git show --stat fourth >got_show && + git diff-tree --stat fourth >got_dt && + test_cmp expect_show got_show && + test_cmp expect_dt got_dt +' + +test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' + no_store log -R --stat >expect && + warm && + git log -R --stat >actual && + test_cmp expect actual +' + +# The diffstat read path produces identical output on a hit or a miss, so +# it emits a trace2 "read-hits" count to prove it consulted the store. +test_expect_success 'diffstat consults the store (trace shows read hits)' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_on.json" git log --stat >/dev/null && + test_grep read-hits trace_on.json && + test_env GIT_TRACE2_EVENT="$PWD/trace_off.json" no_store log --stat >/dev/null && + test_grep ! read-hits trace_off.json +' + +# Diff settings that change hunks but are not part of the store key must +# bypass it in both directions, so output stays byte-identical to a +# store-less run. +test_expect_success 'setup ignore fixture' ' + git init ignore-repo && + ( + cd ignore-repo && + test_write_lines code keep "# c" >f && + git add f && + git commit -m c1 && + test_write_lines codeCH keep "# cX" >f && + git add f && + git commit -m c2 && + warm + ) +' + +test_expect_success '-I bypasses the store' ' + ( + cd ignore-repo && + no_store diff -I"^#" --numstat HEAD~ HEAD >expect && + git diff -I"^#" --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '-B bypasses the store' ' + git init break-repo && + ( + cd break-repo && + test_write_lines a b c d e f g h >f && + git add f && + git commit -m orig && + test_write_lines 1 2 3 4 5 6 7 8 >f && + git add f && + git commit -m rewrite && + warm && + no_store diff -B --stat HEAD~ HEAD >expect && + git diff -B --stat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '--anchored bypasses the store' ' + ( + cd ignore-repo && + no_store diff --stat --anchored=keep HEAD~ HEAD >expect && + git diff --stat --anchored=keep HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '--ignore-blank-lines output is unaffected by the store' ' + git init ibl-repo && + ( + cd ibl-repo && + printf "a\n\nx\ny\nb\n" >f && + git add f && + git commit -m v1 && + printf "a\nx\ny\nB\n" >f && + git add f && + git commit -m v2 && + warm && + no_store diff --stat --ignore-blank-lines HEAD~ HEAD >expect && + git diff --stat --ignore-blank-lines HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'a whitespace-ignoring diff is not served default entries' ' + git init ws-repo && + ( + cd ws-repo && + test_write_lines alpha beta gamma >f && + git add f && + git commit -m c1 && + test_write_lines " alpha" beta gamma delta >f && + git add f && + git commit -m c2 && + warm && + no_store diff -w --numstat HEAD~ HEAD >expect && + git diff -w --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'a driver algorithm override keeps output correct' ' + git init driver-algo && + ( + cd driver-algo && + echo "file.foo diff=foo" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + test_write_lines 1 2 3 4 5 >file.foo && + git add file.foo && + git commit -m one && + test_write_lines 1 2 X 4 5 6 >file.foo && + git add file.foo && + git commit -m two && + warm -c diff.foo.algorithm=histogram && + no_store -c diff.foo.algorithm=histogram log --stat >expect && + git -c diff.foo.algorithm=histogram log --stat >actual && + test_cmp expect actual + ) +' + +# Robustness across the shapes an object walk encounters. +test_expect_success 'binary and mode-only changes do not break the writer' ' + printf "\\000\\001\\002" >bin.dat && + git add bin.dat && + git commit -m binary-1 && + printf "\\000\\001\\003\\004" >bin.dat && + git add bin.dat && + git commit -m binary-2 && + echo "mode content" >mode.txt && + git add mode.txt && + git commit -m mode-1 && + test_chmod +x mode.txt && + git commit -m mode-2 && + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'verify succeeds on a valid store and on an absent one' ' + warm && + git diff-hunks verify && + git diff-hunks clear && + test_path_is_missing $STORE && + git diff-hunks verify +' + +test_expect_success 'verify detects a checksum mismatch' ' + test_when_finished "git diff-hunks clear" && + warm && + fsize=$(test_file_size $STORE) && + mid=$((fsize / 2)) && + printf "\\377" | dd of=$STORE bs=1 seek=$mid count=1 conv=notrunc 2>/dev/null && + test_must_fail git diff-hunks verify +' + +test_expect_success 'a warm discards a corrupt store rather than seeding from it' ' + test_when_finished "git diff-hunks clear" && + warm && + # Corrupt the checksum: the next warm must not carry the corrupt + # entries forward into a fresh checksum-valid file; it discards + # them (with a warning) and rewrites a store that verifies. + fsize=$(test_file_size $STORE) && + printf "\\377" | dd of=$STORE bs=1 seek=$((fsize / 2)) count=1 conv=notrunc 2>/dev/null && + warm 2>err && + test_grep "failed its checksum" err && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'diff-hunks clear removes the store file' ' + warm && + test_path_is_file $STORE && + git diff-hunks clear && + test_path_is_missing $STORE +' + +test_done diff --git a/t/t4218/trim-divergent-new b/t/t4218/trim-divergent-new new file mode 100644 index 00000000000000..f2de40b5ed8f14 --- /dev/null +++ b/t/t4218/trim-divergent-new @@ -0,0 +1,319 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(($_list_size - $_bisection_size * 2)) + if test "$_bisect_err" -lt 0 + then + _bisect_err=$((0 - $_bisect_err)) + fi + _bisect_err=$(($_bisect_err / 2)) ; # floor + + test_expect_success "bisection diff $_bisect_option $_head $* <= $_max_diff" ' + test $_bisect_err -le $_max_diff + ' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/t/t4218/trim-divergent-old b/t/t4218/trim-divergent-old new file mode 100644 index 00000000000000..daa009c9a1b4b6 --- /dev/null +++ b/t/t4218/trim-divergent-old @@ -0,0 +1,316 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(expr $_list_size - $_bisection_size \* 2) + test "$_bisect_err" -lt 0 && _bisect_err=$(expr 0 - $_bisect_err) + _bisect_err=$(expr $_bisect_err / 2) ; # floor + + test_expect_success \ + "bisection diff $_bisect_option $_head $* <= $_max_diff" \ + 'test $_bisect_err -le $_max_diff' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done From 0fc2016d55f92d8ed4e518149942a3894fa4babc Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 20:13:14 -0700 Subject: [PATCH 12/21] blame: read precomputed hunks Before diffing a target blob against a parent, offer the pair's identity to the hunk provider seam. diff_provider_emit_hunks() now takes the repository and the pair's blob object ids and consults providers, keyed by the ids and the request's xdiff flags at zero context, before falling back to its fill-and-compute path. A hit replays the recorded hunks through blame_chunk_cb without loading either blob; a request carrying -I patterns or anchors is outside the key and always computes. Blame withholds the identity where its diff is not the plain blob-pair diff the key describes (reverse blame, ignored revisions, and textconv paths), so those requests always compute. Whitespace and algorithm options such as -w instead change blame's xdl_opts, so the consult keys a different entry and misses a store warmed without them. Blame's default xdl_opts now come from DIFF_HUNKS_DEFAULT_XDL_OPTS, the same macro the diff side uses, so a default blame run and a default "log --stat" warming run share keys by construction. "--show-stats" reports how many pairs were served, for tests and tuning. Extend t4218 with the blame side: parity for plain, --porcelain, and --incremental output, hit accounting across warming runs, the blame inputs that must bypass or miss the store (-w, indent heuristics, --reverse, textconv, -M/-C), rename and merge handling, --contents, and reading a truncated or corrupt store as absent. Add p4218 measuring warm cost and the read speedups. Signed-off-by: Michael Montalbo --- blame.c | 44 +++++++- blame.h | 2 + builtin/blame.c | 4 +- diff-provider.c | 19 +++- diff-provider.h | 16 ++- t/perf/p4218-diff-hunks.sh | 48 +++++++++ t/t4218-diff-hunks.sh | 206 +++++++++++++++++++++++++++++++++++++ 7 files changed, 330 insertions(+), 9 deletions(-) create mode 100755 t/perf/p4218-diff-hunks.sh diff --git a/blame.c b/blame.c index ec90d38b8a0856..e7cfc6a11a158a 100644 --- a/blame.c +++ b/blame.c @@ -24,6 +24,7 @@ #include "bloom.h" #include "commit-graph.h" #include "diff-provider.h" +#include "userdiff.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1934,6 +1935,24 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +/* + * A hunk provider's key names the (old blob, new blob) pair and may only + * serve a diff whose result is determined by that pair and the xdiff + * settings. Textconv rewrites the buffers being diffed away from the + * blob contents the key names, so any origin whose path has a textconv + * driver must withhold the pair's identity. + */ +static int blame_textconv_active(struct blame_scoreboard *sb, + const char *path) +{ + struct userdiff_driver *drv; + + if (!sb->revs->diffopt.flags.allow_textconv) + return 0; + drv = userdiff_find_by_path(sb->repo->index, path); + return drv && drv->textconv; +} + struct blame_diff_fill { struct blame_scoreboard *sb; struct blame_origin *parent, *target; @@ -1966,6 +1985,7 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_entry *newdest = NULL; struct blame_diff_fill fill = { sb, parent, target, ignore_diffs }; xpparam_t xpp = { .flags = sb->xdl_opts }; + int provider_usable, served; if (!target->suspects) return; /* nothing remains for this target */ @@ -1976,11 +1996,31 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; - if (diff_provider_emit_hunks(&xpp, blame_diff_fill, &fill, - blame_chunk_cb, &d)) + /* + * Offer the pair's identity only where blame's diff is the plain + * blob-pair diff a provider's key describes; reverse blame, + * ignored revisions, and textconv paths withhold it and always + * compute. + */ + provider_usable = !sb->reverse && !ignore_diffs && + !blame_textconv_active(sb, target->path) && + !blame_textconv_active(sb, parent->path); + + served = diff_provider_emit_hunks(sb->repo, + provider_usable ? &parent->blob_oid : NULL, + provider_usable ? &target->blob_oid : NULL, + &xpp, blame_diff_fill, &fill, + blame_chunk_cb, &d); + if (served < 0) die("unable to generate diff (%s -> %s)", oid_to_hex(&parent->commit->object.oid), oid_to_hex(&target->commit->object.oid)); + if (provider_usable && diff_provider_active(sb->repo)) { + if (served) + sb->num_precomputed_hits++; + else + sb->num_precomputed_misses++; + } /* The rest are the same as the parent */ blame_chunk(&d.dstq, &d.srcq, INT_MAX, d.offset, INT_MAX, 0, parent, target, 0); diff --git a/blame.h b/blame.h index 3b34be0e5c6932..cd478ca477dc72 100644 --- a/blame.h +++ b/blame.h @@ -132,6 +132,8 @@ struct blame_scoreboard { int num_read_blob; int num_get_patch; int num_commits; + int num_precomputed_hits; + int num_precomputed_misses; /* * blame for a blame_entry with score lower than these thresholds diff --git a/builtin/blame.c b/builtin/blame.c index ffbd3ce5c5a2e3..a8ae47c8fd4fbe 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -1033,7 +1033,7 @@ int cmd_blame(int argc, parse_done: revision_opts_finish(&revs); no_whole_file_rename = !revs.diffopt.flags.follow_renames; - xdl_opts |= revs.diffopt.xdl_opts & XDF_INDENT_HEURISTIC; + xdl_opts |= revs.diffopt.xdl_opts & DIFF_HUNKS_DEFAULT_XDL_OPTS; revs.diffopt.flags.follow_renames = 0; argc = parse_options_end(&ctx); @@ -1294,6 +1294,8 @@ int cmd_blame(int argc, printf("num read blob: %d\n", sb.num_read_blob); printf("num get patch: %d\n", sb.num_get_patch); printf("num commits: %d\n", sb.num_commits); + printf("num precomputed hits: %d\n", sb.num_precomputed_hits); + printf("num precomputed misses: %d\n", sb.num_precomputed_misses); } cleanup: diff --git a/diff-provider.c b/diff-provider.c index 3426920397cc62..6e397f1895b98c 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -43,7 +43,10 @@ diff_provider_hunks_check(struct diff_provider_hunks_check *c, return PROVIDER_HUNKS_OK; } -int diff_provider_emit_hunks(const xpparam_t *xpp, +int diff_provider_emit_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, hunk_pair_fill_fn fill, void *fill_data, xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) @@ -52,7 +55,19 @@ int diff_provider_emit_hunks(const xpparam_t *xpp, xdemitcb_t ecb = { .priv = cb_data }; mmfile_t old_file, new_file; + /* + * -I patterns and anchors shape the diff but are outside the + * settings that key a provider's answer, so such a request is + * computed, never served. + */ + if (!xpp->ignore_regex_nr && !xpp->anchors_nr && + diff_provider_query_hunks(r, old_oid, new_oid, xpp->flags, + hunk_cb, cb_data)) + return 1; + if (fill(fill_data, &old_file, &new_file) < 0) return -1; - return xdi_diff(&old_file, &new_file, xpp, &xecfg, &ecb); + if (xdi_diff(&old_file, &new_file, xpp, &xecfg, &ecb) < 0) + return -1; + return 0; } diff --git a/diff-provider.h b/diff-provider.h index 2424d0daea0cda..7376a9c45272fa 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -79,11 +79,19 @@ typedef int (*hunk_pair_fill_fn)(void *data, mmfile_t *old_file, mmfile_t *new_file); /* - * Emit the exact changed ranges (context 0) for one file pair to - * hunk_cb. xpp carries the diff parameters that determine the - * ranges. Returns 0 on success, -1 on failure to load or diff. + * Emit the exact changed ranges (context 0) for the pair (old_oid, + * new_oid) to hunk_cb. Providers are consulted first, keyed by the + * object ids and xpp's flags; pass NULL object ids when the diffed + * bytes are not those blobs (or there are no blobs), which makes + * every consult miss. On a miss, fill supplies the content and + * xdiff computes the ranges from xpp. Returns 1 when a provider + * supplied the ranges, 0 when they were computed, and -1 on failure + * to load or diff. */ -int diff_provider_emit_hunks(const xpparam_t *xpp, +int diff_provider_emit_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, hunk_pair_fill_fn fill, void *fill_data, xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data); diff --git a/t/perf/p4218-diff-hunks.sh b/t/perf/p4218-diff-hunks.sh new file mode 100755 index 00000000000000..a96bbd9f6c0eba --- /dev/null +++ b/t/perf/p4218-diff-hunks.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +test_description='diff-hunks store performance' +. ./perf-lib.sh + +test_perf_default_repo + +# Pick a file to blame pseudo-randomly. The sort key is the blob +# hash, so it is stable. +test_expect_success 'select a file' ' + git ls-tree HEAD | grep ^100644 | + sort -k 3 | head -n 1 | cut -f 2 >filelist +' + +file=$(cat filelist) +export file + +# Warm the store the way an owner would: a stat walk with writing on. +test_perf 'warm the store' ' + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_expect_success 'warm the store' ' + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_perf 'log --stat -1000 (store)' ' + git log --stat -1000 >/dev/null +' + +test_perf 'log --stat -1000 (no store)' ' + git -c core.diffhunks=false log --stat -1000 >/dev/null +' + +test_perf 'blame $file (store)' ' + git blame "$file" >/dev/null +' + +test_perf 'blame $file (no store)' ' + git -c core.diffhunks=false blame "$file" >/dev/null +' + +test_expect_success 'clean up store' ' + git diff-hunks clear +' + +test_done diff --git a/t/t4218-diff-hunks.sh b/t/t4218-diff-hunks.sh index bb16ec0a15b582..5b8f376a22030d 100755 --- a/t/t4218-diff-hunks.sh +++ b/t/t4218-diff-hunks.sh @@ -80,6 +80,43 @@ test_expect_success 'a second warming run refreshes the store in place' ' test_cmp expect actual ' +test_expect_success 'core.diffhunks=false disables lookups' ' + warm && + git -c core.diffhunks=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out +' + +# Writing seeds from the current store and merges into it, so a later +# warming run keeps the entries an earlier one recorded rather than +# rebuilding. Warm one pair, then a different pair, and confirm the first +# is still served. +test_expect_success 'a later warming run preserves earlier entries' ' + git init incr && + ( + cd incr && + test_commit a1 f.txt "1" && + test_commit a2 f.txt "1 +2" && + test_commit a3 f.txt "1 +2 +3" && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a1 a2 >/dev/null && + git diff-hunks verify && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a2 a3 >/dev/null && + git diff-hunks verify && + + # Blaming as of a2 diffs the a1..a2 pair. If seeding had + # dropped it when the a2..a3 pair was warmed, this would + # report zero precomputed hits. + git blame --show-stats a2 -- f.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9]" out && + + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual + ) +' + test_expect_success 'log --stat matches with and without the store' ' no_store log --stat >expect && warm && @@ -201,6 +238,14 @@ test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' test_cmp expect actual ' +# One warm serves both diffstat and blame: the blob pairs a blame +# walks are the same parent-child pairs the diffstat warm recorded. +test_expect_success 'a single warming run serves both blame and diffstat' ' + warm && + git blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9][0-9]*" out +' + # The diffstat read path produces identical output on a hit or a miss, so # it emits a trace2 "read-hits" count to prove it consulted the store. test_expect_success 'diffstat consults the store (trace shows read hits)' ' @@ -211,6 +256,23 @@ test_expect_success 'diffstat consults the store (trace shows read hits)' ' test_grep ! read-hits trace_off.json ' +test_expect_success 'blame matches with and without the store' ' + no_store blame file.txt >expect && + warm && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame --porcelain and --incremental match' ' + no_store blame --porcelain file.txt >expect_p && + no_store blame --incremental file.txt >expect_i && + warm && + git blame --porcelain file.txt >got_p && + git blame --incremental file.txt >got_i && + test_cmp expect_p got_p && + test_cmp expect_i got_i +' + # Diff settings that change hunks but are not part of the store key must # bypass it in both directions, so output stays byte-identical to a # store-less run. @@ -297,6 +359,26 @@ test_expect_success 'a whitespace-ignoring diff is not served default entries' ' ) ' +test_expect_success 'blame -w stays correct and does not hit default entries' ' + ( + cd ws-repo && + no_store blame -w f >expect && + git blame -w --show-stats f >out 2>&1 && + test_grep "num precomputed hits: 0" out && + git blame -w f >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame with indentHeuristic off stays correct and misses' ' + warm && + git -c diff.indentHeuristic=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store -c diff.indentHeuristic=false blame file.txt >expect && + git -c diff.indentHeuristic=false blame file.txt >actual && + test_cmp expect actual +' + test_expect_success 'a driver algorithm override keeps output correct' ' git init driver-algo && ( @@ -317,6 +399,57 @@ test_expect_success 'a driver algorithm override keeps output correct' ' ) ' +test_expect_success 'blame --reverse never consults the store' ' + warm && + git blame --reverse HEAD~3..HEAD file.txt >actual 2>/dev/null && + no_store blame --reverse HEAD~3..HEAD file.txt >expect 2>/dev/null && + test_cmp expect actual +' + +test_expect_success 'blame with a textconv driver bypasses the store' ' + echo "tc.txt diff=tc" >>.gitattributes && + git add .gitattributes && + git commit -m tc-attr && + git config diff.tc.textconv "sed -e s/1/one/" && + test_commit tc1 tc.txt "line 1" && + test_commit tc2 tc.txt "line 1 +line 2" && + warm && + git blame --show-stats tc.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store blame tc.txt >expect && + git blame tc.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame -M and -C stay correct with the store' ' + warm && + no_store blame -M file.txt >expect_m && + no_store blame -C file.txt >expect_c && + git blame -M file.txt >got_m && + git blame -C file.txt >got_c && + test_cmp expect_m got_m && + test_cmp expect_c got_c +' + +# Copy-detecting (and reverse) blame still diff blob pairs through +# pass_blame_to_parent, so they must use the real blame xdl_opts. A +# whitespace-only change is invisible under -w; if -w were silently dropped +# (as it was for -C and --reverse) the -w and non-w results would coincide. +test_expect_success 'blame -C honors -w' ' + git init -q blame-cw && + ( + cd blame-cw && + printf "one\ntwo\nthree\n" >f && + git add f && git commit -q -m base && + printf "one\n two \nthree\n" >f && + git add f && git commit -q -m reindent && + git blame -C -w f >with_w && + git blame -C f >without_w && + ! test_cmp with_w without_w + ) +' + # Robustness across the shapes an object walk encounters. test_expect_success 'binary and mode-only changes do not break the writer' ' printf "\\000\\001\\002" >bin.dat && @@ -336,6 +469,79 @@ test_expect_success 'binary and mode-only changes do not break the writer' ' test_cmp expect actual ' +test_expect_success 'blame across a rename matches' ' + echo "original content" >rename-src.txt && + git add rename-src.txt && + git commit -m "add rename-src" && + echo "more" >>rename-src.txt && + git add rename-src.txt && + git commit -m "modify rename-src" && + git mv rename-src.txt rename-dst.txt && + git commit -m "rename" && + echo "post" >>rename-dst.txt && + git add rename-dst.txt && + git commit -m "modify after rename" && + no_store blame rename-dst.txt >expect && + warm && + git blame rename-dst.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame handles merge commits' ' + git checkout -b merge-side main~2 && + test_commit merge-change merge-file.txt "side content" && + git checkout main && + git merge --no-edit merge-side && + no_store blame merge-file.txt >expect && + warm && + git blame merge-file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'distinct --contents against one revision do not collide' ' + warm && + test_write_lines "line 1" "appended line" >c1 && + test_write_lines "rewritten line" >c2 && + # Ground truth without the store. + no_store blame -s --contents=c2 file.txt initial >expect && + # With the store, an intervening c1 run must not poison the c2 lookup. + git blame -s --contents=c1 file.txt initial >/dev/null && + git blame -s --contents=c2 file.txt initial >actual && + test_cmp expect actual +' + +# Integrity: a structurally broken header is read as absent (the reader +# falls back to xdiff and stays correct); a checksum mismatch is caught +# by verify, which is when integrity is checked. +test_expect_success 'a truncated store is read as absent' ' + warm && + test_copy_bytes 20 <$STORE >truncated && + mv truncated $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a corrupt signature is read as absent' ' + warm && + printf "XXXX" >corrupt && + tail -c +5 <$STORE >>corrupt && + mv corrupt $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# Byte 6 of the header is the chunk count; a value larger than the file +# can hold must be rejected before the chunk table is walked. +test_expect_success 'an over-claimed chunk count is read as absent' ' + warm && + printf "\377" | dd of=$STORE bs=1 seek=6 count=1 conv=notrunc 2>/dev/null && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + test_expect_success 'verify succeeds on a valid store and on an absent one' ' warm && git diff-hunks verify && From b1d200f2f65738898c4a91554ce0ce5995ab6fdb Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 21 May 2026 02:37:54 -0700 Subject: [PATCH 13/21] xdiff: support external hunks via xpparam_t Add two new xpparam_t fields (external_hunks, external_hunks_nr) that let callers supply pre-computed hunks. When set, xdl_diff() populates the changed[] arrays from these hunks instead of running the diff algorithm, then continues through compaction and emission as usual. Validate supplied hunks before use. Out-of-bounds line numbers, overlapping or out-of-order hunks, and misaligned unchanged runs are treated as a malformed tool response: xdl_populate_hunks_from_external() warns, returns -1, and xdl_diff() falls back to the builtin diff algorithm for that file. The run of unchanged lines between two hunks (and before the first and after the last) must be the same length on both sides; xdl_build_script() walks the two files in lockstep over unchanged lines, so a balanced total is not enough. Non-negative counts and 1-based starts are instead caller preconditions, checked with BUG(), since the caller normalizes hunks before this point. On rejection xdl_diff() frees the environment it prepared and falls through to xdl_do_diff(), which prepares a fresh one for the builtin pass. Skip trim_common_tail() in xdi_diff() when external hunks are present, since external hunks reference line numbers in the original content. The diff-hunks settings projection asserts xpparam_t's layout so that a new field forces an explicit keying decision; extend its reference struct with the two new fields. They carry a caller-supplied answer rather than a setting: the store paths build their xpparam_t locally and never populate them, so a recorded diff is always xdiff's own. Signed-off-by: Michael Montalbo --- diff.c | 7 +++- xdiff-interface.c | 7 +++- xdiff/xdiff.h | 16 +++++++++ xdiff/xdiffi.c | 84 +++++++++++++++++++++++++++++++++++++++++++++-- xdiff/xprepare.c | 10 ++++++ xdiff/xprepare.h | 1 + 6 files changed, 121 insertions(+), 4 deletions(-) diff --git a/diff.c b/diff.c index 88a678520e6f30..00026b35c1aae2 100644 --- a/diff.c +++ b/diff.c @@ -4369,7 +4369,10 @@ static int diffstat_from_hunks(struct diff_options *o, /* * xpparam_t is the diff algorithm's input. Its flags are the key's * xdl_opts; ignore_regex (-I) and anchors (--anchored) are instead - * excluded by the guard below. + * excluded by the guard below. external_hunks carries a + * caller-supplied answer rather than a setting: the store paths + * build their xpparam_t locally and never populate it, so a + * recorded diff is always xdiff's own. * * Adding an xpparam_t field fires this assert (its size no longer * matches the reference struct). To clear it: (1) add the field to @@ -4385,6 +4388,8 @@ static int diffstat_from_hunks(struct diff_options *o, size_t ignore_regex_nr; char **anchors; size_t anchors_nr; + struct xdl_hunk *external_hunks; + size_t external_hunks_nr; })); if (!((diff_provider_active(o->repo) || o->hunks_writer) && diff --git a/xdiff-interface.c b/xdiff-interface.c index 32e04630ee2ee8..afe4c12e9e9bcf 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -143,7 +143,12 @@ int xdi_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdemitconf_t co if (mf1->size > MAX_XDIFF_SIZE || mf2->size > MAX_XDIFF_SIZE) return -1; - if (!xecfg->ctxlen && !(xecfg->flags & XDL_EMIT_FUNCCONTEXT)) + /* + * External hunks reference line numbers in the original content; + * trimming the tail would change line counts and invalidate them. + */ + if (!xpp->external_hunks && + !xecfg->ctxlen && !(xecfg->flags & XDL_EMIT_FUNCCONTEXT)) trim_common_tail(&a, &b); return xdl_diff(&a, &b, xpp, xecfg, xecb); diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index dc370712e92860..4736bcdb07f16a 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -78,6 +78,18 @@ typedef struct s_mmbuffer { long size; } mmbuffer_t; +/* + * Hunk descriptor for externally computed diffs, in xdiff's own + * coordinates: line numbers are 1-based and a hunk's start is the + * first line it covers. A caller translates any external "empty side" + * idiom (such as git diff's start-0/count-0) to a 1-based start before + * handing hunks over. + */ +struct xdl_hunk { + long old_start, old_count; + long new_start, new_count; +}; + typedef struct s_xpparam { unsigned long flags; @@ -88,6 +100,10 @@ typedef struct s_xpparam { /* See Documentation/diff-options.adoc. */ char **anchors; size_t anchors_nr; + + /* Externally computed hunks: bypass the diff algorithm. Owned by caller. */ + struct xdl_hunk *external_hunks; + size_t external_hunks_nr; } xpparam_t; typedef struct s_xdemitcb { diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index c5a892f91e00c0..73a456f5dd6d28 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -1085,16 +1085,96 @@ static void xdl_mark_ignorable_regex(xdchange_t *xscr, const xdfenv_t *xe, } } +/* + * Populate the changed[] arrays from externally supplied hunks, + * bypassing the diff algorithm. The caller normalizes and validates + * the hunks first (order, overlap, and lockstep alignment), so this + * only marks lines changed after asserting the memory-safety + * preconditions it depends on: non-negative counts and 1-based starts + * (checked with BUG()), and an in-bounds range (a silent -1 so the + * caller can fall back to the builtin diff rather than index changed[] + * out of range). Keeping this diagnostic-free leaves user-facing + * messages to the git layer. + * + * Returns 0 on success, -1 if a hunk is out of range. + */ +static int xdl_populate_hunks_from_external(xdfenv_t *xe, + struct xdl_hunk *hunks, + size_t nr_hunks) +{ + size_t i; + long j; + + /* + * xdl_prepare_env() may dirty changed[] via xdl_cleanup_records(). + * Clear them so only the external hunks are marked. + */ + xdl_clear_changed(&xe->xdf1); + xdl_clear_changed(&xe->xdf2); + + for (i = 0; i < nr_hunks; i++) { + struct xdl_hunk *h = &hunks[i]; + + /* + * Non-negative counts and 1-based starts are caller + * preconditions (it normalizes hunks into xdiff coordinates + * before this point), so a violation is a bug, not a bad + * tool response. + */ + if (h->old_count < 0 || h->new_count < 0) + BUG("external hunk %"PRIuMAX": " + "negative count (old=%ld, new=%ld)", + (uintmax_t)(i + 1), + h->old_count, h->new_count); + if (h->old_start < 1 || h->new_start < 1) + BUG("external hunk %"PRIuMAX": " + "start not 1-based (old=%ld, new=%ld)", + (uintmax_t)(i + 1), + h->old_start, h->new_start); + + /* + * The caller validates ordering, overlap and lockstep + * alignment (and diagnoses a bad response). This is only a + * silent in-bounds guard so the marking loop cannot index + * changed[] out of range: start + count - 1 <= nrec, + * rewritten to avoid overflow. A count of 0 (pure + * insert/delete) allows start == nrec + 1, the position + * after the last line. On a miss, return -1 and let the + * caller fall back to the builtin diff. + */ + if (h->old_count > (long)xe->xdf1.nrec - h->old_start + 1 || + h->new_count > (long)xe->xdf2.nrec - h->new_start + 1) + return -1; + + for (j = 0; j < h->old_count; j++) + xe->xdf1.changed[h->old_start - 1 + j] = true; + for (j = 0; j < h->new_count; j++) + xe->xdf2.changed[h->new_start - 1 + j] = true; + } + + return 0; +} + int xdl_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdemitconf_t const *xecfg, xdemitcb_t *ecb) { xdchange_t *xscr; xdfenv_t xe; emit_func_t ef = xecfg->hunk_func ? xdl_call_hunk_func : xdl_emit_diff; - if (xdl_do_diff(mf1, mf2, xpp, &xe) < 0) { + if (xpp->external_hunks) { + if (xdl_prepare_env(mf1, mf2, xpp, &xe) < 0) + return -1; + if (xdl_populate_hunks_from_external(&xe, + xpp->external_hunks, + xpp->external_hunks_nr) == 0) + goto diff_done; + xdl_free_env(&xe); + } + if (xdl_do_diff(mf1, mf2, xpp, &xe) < 0) return -1; - } + +diff_done: if (xdl_change_compact(&xe.xdf1, &xe.xdf2, xpp->flags) < 0 || xdl_change_compact(&xe.xdf2, &xe.xdf1, xpp->flags) < 0 || xdl_build_script(&xe, &xscr) < 0) { diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 11bada2608a7a4..f4ab93533286da 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -471,3 +471,13 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, return 0; } + +/* + * Reset the changed[] array so that no lines are marked as changed. + * Also clears the sentinel slots at changed[-1] and changed[nrec] + * that xdl_change_compact() relies on during backward scans. + */ +void xdl_clear_changed(xdfile_t *xdf) +{ + memset(xdf->changed - 1, 0, (xdf->nrec + 2) * sizeof(bool)); +} diff --git a/xdiff/xprepare.h b/xdiff/xprepare.h index 947d9fc1bb8cf9..0413baf07bcc90 100644 --- a/xdiff/xprepare.h +++ b/xdiff/xprepare.h @@ -28,6 +28,7 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdfenv_t *xe); void xdl_free_env(xdfenv_t *xe); +void xdl_clear_changed(xdfile_t *xdf); From 896b0fee4ed0ce659d3538478eb892e7cdab5e87 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 21 May 2026 02:37:59 -0700 Subject: [PATCH 14/21] userdiff: add diff..process config Add the process field to struct userdiff_driver and teach the config parser to populate it from diff..process. Signed-off-by: Michael Montalbo --- userdiff.c | 7 +++++++ userdiff.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/userdiff.c b/userdiff.c index b5412e6bc3ecd3..7547874aa2569d 100644 --- a/userdiff.c +++ b/userdiff.c @@ -509,6 +509,13 @@ int userdiff_config(const char *k, const char *v) drv->algorithm = drv->algorithm_owned; return ret; } + if (!strcmp(type, "process")) { + int ret; + FREE_AND_NULL(drv->process_owned); + ret = git_config_string(&drv->process_owned, k, v); + drv->process = drv->process_owned; + return ret; + } return 0; } diff --git a/userdiff.h b/userdiff.h index 827361b0bc9569..51c26e0d4190e5 100644 --- a/userdiff.h +++ b/userdiff.h @@ -31,6 +31,8 @@ struct userdiff_driver { char *textconv_owned; struct notes_cache *textconv_cache; int textconv_want_cache; + const char *process; + char *process_owned; }; enum userdiff_driver_type { USERDIFF_DRIVER_TYPE_BUILTIN = 1<<0, From 4bbc6d876e5ae175a4e306be191d7e61c117b91d Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 28 May 2026 16:21:32 -0700 Subject: [PATCH 15/21] sub-process: separate process lifecycle from hashmap management subprocess_start() and subprocess_stop() couple two concerns: managing a child process (setup, handshake, teardown) and managing a hashmap that indexes running processes by command string. The hashmap suits callers like convert.c where many files may share one filter process looked up by name, but callers that manage process lifetime through their own data structures do not need it. Extract subprocess_start_command() and subprocess_stop_command() so callers can reuse the child process setup and handshake machinery without maintaining a hashmap. subprocess_start() and subprocess_stop() become thin wrappers that add hashmap operations on top. Signed-off-by: Michael Montalbo --- sub-process.c | 28 +++++++++++++++++++++++----- sub-process.h | 9 ++++++++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/sub-process.c b/sub-process.c index 83bf0a0e82e56d..5468939338e6c0 100644 --- a/sub-process.c +++ b/sub-process.c @@ -49,7 +49,7 @@ int subprocess_read_status(int fd, struct strbuf *status) return (len < 0) ? len : 0; } -void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +void subprocess_stop_command(struct subprocess_entry *entry) { if (!entry) return; @@ -57,7 +57,14 @@ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) entry->process.clean_on_exit = 0; kill(entry->process.pid, SIGTERM); finish_command(&entry->process); +} +void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +{ + if (!entry) + return; + + subprocess_stop_command(entry); hashmap_remove(hashmap, &entry->ent, NULL); } @@ -72,7 +79,7 @@ static void subprocess_exit_handler(struct child_process *process) finish_command(process); } -int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn) { int err; @@ -96,15 +103,26 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return err; } - hashmap_entry_init(&entry->ent, strhash(cmd)); - err = startfn(entry); if (err) { error("initialization for subprocess '%s' failed", cmd); - subprocess_stop(hashmap, entry); + subprocess_stop_command(entry); return err; } + return 0; +} + +int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn) +{ + int err; + + err = subprocess_start_command(entry, cmd, startfn); + if (err) + return err; + + hashmap_entry_init(&entry->ent, strhash(cmd)); hashmap_add(hashmap, &entry->ent); return 0; } diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..45f1b8e5e3212f 100644 --- a/sub-process.h +++ b/sub-process.h @@ -52,10 +52,17 @@ int cmd2process_cmp(const void *unused_cmp_data, */ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); -/* Start a subprocess and add it to the subprocess hashmap. */ +/* Start a subprocess and run the startfn (typically handshake). */ +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn); + +/* Start a subprocess, run startfn, and add it to the subprocess hashmap. */ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +/* Kill a subprocess. */ +void subprocess_stop_command(struct subprocess_entry *entry); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); From a72b4c12caef4dd1ee064d80f485865a758d63b4 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 28 May 2026 15:08:38 -0700 Subject: [PATCH 16/21] diff: add long-running diff process via diff..process Add support for external diff processes that communicate via the long-running process protocol (pkt-line over stdin/stdout). A diff process is configured per userdiff driver: [diff "cdiff"] process = /path/to/diff-tool The tool provides custom line-matching: it receives file pairs and returns hunks that reference line numbers in the content. When textconv is also configured, the tool receives the textconv-transformed content. The tool controls which lines are marked as changed while the display shows the file content. Patch output features (word diff, function context, color) work normally. A new "Which features consult the diff process" documentation section lays out which features use the tool's hunks, which compute independently, and why; the summary formats such as --stat still use the builtin diff for now. The handshake negotiates version=1 and capability=hunks. Per-file requests send command=hunks, pathname, the old and new blob object names as old-oid/new-oid, and both file contents as packetized data. The tool responds with hunk lines and a status packet (success, error, or abort). On error, Git warns and falls back to the builtin diff algorithm for that file. On abort, Git silently falls back for the current file and stops sending further requests to the tool for the remainder of the session. old-oid/new-oid name the two blobs so a tool can cache its analysis keyed on the pair. A side's oid is sent only when the content the tool receives is that raw blob: it is omitted under textconv, which rewrites the bytes, and for a working-tree side with no stored object, so an oid that is sent always names the bytes the tool receives. This is where the process protocol diverges from diff..command, which never composes with textconv (the command replaces the whole diff and always gets the raw blob). Tools ignore unknown request keys, so old tools skip them. When the tool returns no hunks followed by status=success, Git treats the file as having no changes and produces no diff output. This also means --exit-code reports no changes for that file. The subprocess is stored on the userdiff_driver struct and launched on first use. If the process fails to start, the handshake fails, or a communication error occurs mid-stream, the failure is cached on the driver to avoid retrying and re-warning on every subsequent file. Git falls back to the builtin diff (rather than consulting the tool) when an option the tool cannot honor is in effect: the whitespace-ignoring flags, --ignore-blank-lines, -I, and --anchored. The bypass keys off the effective diff parameters (xpp) rather than diffopt, so a later caller whose flags live elsewhere is covered uniformly. A change that only adds or removes the trailing newline is likewise not expressible as hunks, so it too uses the builtin diff. The hunk parser ignores unknown trailing fields on a hunk line for response forward-compatibility. Hunk accumulation is bounded by the combined byte count of the two files, so a misbehaving tool that floods hunk lines cannot grow memory without bound before validation runs. diff_process_fill_hunks() is the sole public entry point. It handles driver lookup, flag checks, subprocess management, and error reporting, returning an enum that lets callers distinguish "hunks populated" from "files equivalent" from "not applicable" from "tool failure." Helped-by: Johannes Schindelin Signed-off-by: Michael Montalbo --- Documentation/config/diff.adoc | 5 + Documentation/gitattributes.adoc | 249 +++++++++++ Makefile | 2 + diff-process.c | 499 +++++++++++++++++++++ diff-process.h | 49 ++ diff.c | 21 + diff.h | 3 + meson.build | 1 + t/helper/meson.build | 1 + t/helper/test-diff-process-backend.c | 381 ++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/meson.build | 1 + t/t4080-diff-process.sh | 645 +++++++++++++++++++++++++++ userdiff.h | 3 + 15 files changed, 1862 insertions(+) create mode 100644 diff-process.c create mode 100644 diff-process.h create mode 100644 t/helper/test-diff-process-backend.c create mode 100755 t/t4080-diff-process.sh diff --git a/Documentation/config/diff.adoc b/Documentation/config/diff.adoc index 1135a62a0ad3de..ac0635bb3bee1c 100644 --- a/Documentation/config/diff.adoc +++ b/Documentation/config/diff.adoc @@ -218,6 +218,11 @@ endif::git-diff[] Set this option to `true` to make the diff driver cache the text conversion outputs. See linkgit:gitattributes[5] for details. +`diff..process`:: + The command to run as a long-running diff process that + provides hunks to Git's diff pipeline. + See linkgit:gitattributes[5] for details. + `diff.indentHeuristic`:: Set this option to `false` to disable the default heuristics that shift diff hunk boundaries to make patches easier to read. diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index 2c4fbfd7f1495c..f4ca4a8c7e2296 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -831,6 +831,255 @@ NOTE: If `diff..command` is defined for path with the (see above), and adding `diff..algorithm` has no effect, as the algorithm is not passed to the external diff driver. +Using an external diff process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If `diff..process` is defined, Git sends the old and new file +content to an external tool and receives back a list of changed +regions (pairs of line ranges in the old and new file). Git uses +these instead of its builtin diff algorithm, but still controls +all output formatting, so features like word diff, function context, +color, and blame work normally. This is achieved by using the +long-running process protocol (described in +Documentation/technical/long-running-process-protocol.adoc). +Unlike `diff..command`, which replaces Git's output entirely, +the diff process feeds results back into the standard pipeline. If +both are configured for a path, `diff..command` takes precedence +for the patch output it replaces; the summary formats, `git blame`, +and `git log -L` never run the command and still consult the process. + +First, in `.gitattributes`, assign the `diff` attribute for paths. + +------------------------ +*.c diff=cdiff +------------------------ + +Then, define a "diff..process" configuration to specify +the diff process command. + +---------------------------------------------------------------- +[diff "cdiff"] + process = /path/to/diff-process-tool +---------------------------------------------------------------- + +When Git encounters the first file that needs to be diffed, it starts +the process and performs the handshake. In the handshake, the welcome +message sent by Git is "git-diff-client", only version 1 is supported, +and the supported capability is "hunks" (the changed regions +described below). The tool replies with "git-diff-server", the +version it supports, and the capabilities it supports. + +For each file, Git sends a list of "key=value" pairs terminated with +a flush packet, followed by the old and new file content as packetized +data, each terminated with a flush packet. The pathname is relative +to the repository root. When `diff..textconv` is also set, +the tool receives the textconv-transformed content rather than the +raw blob. Git does not send binary files to the diff process. + +----------------------- +packet: git> command=hunks +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +packet: git> OLD_CONTENT +packet: git> 0000 +packet: git> NEW_CONTENT +packet: git> 0000 +----------------------- + +The optional `old-oid` and `new-oid` keys give the object names of the +old and new blobs, so a tool can cache its analysis keyed on the pair. +A side's key is sent only when the content for that side is the raw +blob it names: it is omitted when the content is textconv-transformed, +and for a working-tree side that has no stored object. A tool that +does not recognize these keys ignores them. + +The tool is expected to respond with zero or more hunk lines, +a flush packet, and a status packet terminated with a flush packet. +Each hunk line has the form: + + `hunk ` + +where `` and `` identify a range of lines in +the old file, and `` and `` identify the +replacement range in the new file. The four fields are separated by +single spaces. Start values are 1-based and counts are non-negative. +For example, `hunk 3 2 3 4` means that 2 lines starting at line 3 in +the old file were replaced by 4 lines starting at line 3 in the new +file. An `` of 0 means no lines were removed (pure +insertion); a `` of 0 means no lines were added (pure +deletion). For a side with a count of 0 (a pure insertion or +deletion) the start is the 1-based line the change sits before, +ranging from 1 to one past the last line (the line count plus 1, to +place the change at the end of the file); like every start it must +keep the unchanged runs aligned on both sides (see below), so for a +given change it takes one specific value, not an arbitrary one. A +start of 0 is also accepted and treated as 1, matching the +empty-file-side form `git diff` emits (e.g. `hunk 0 0 1 5` for a newly +added file). A nonzero range must not extend beyond the end of the +file. Git ignores any extra +whitespace-separated tokens after ``, so a future protocol +version can append fields to a hunk line (for example a "moved" +marker) without older tools rejecting it. + +Lines are delimited by newlines. A file `"foo\nbar\n"` and a +file `"foo\nbar"` both have 2 lines. + +Hunks must be listed in order and must not overlap. Any line not +covered by a hunk is treated as unchanged and is paired, in order, +with the unchanged lines on the other side. Each run of unchanged +lines between two hunks (and the run before the first hunk and +after the last) must therefore be the same length on both sides, +not merely equal in total. For the hunks `1 3 1 5` and `10 2 12 2` +below, lines 4-9 of the old file and lines 6-11 of the new file are +both the six unchanged lines between the two hunks. A response that +balances only the total unchanged count but misaligns one of these +runs is rejected, and Git falls back to the builtin diff. + +Git does not check that the lines a hunk leaves unchanged are +byte-for-byte identical between the two sides; it pairs them by +position and shows the new side as context. A tool may therefore +report lines that differ textually (a pure reformatting, say) as +unchanged, and the diff reflects that judgment. This is +the point of a semantic backend, but it means a misbehaving tool can +produce a diff whose context does not match the old blob; as with +`git diff -w`, such a patch may not apply against the old content. + +----------------------- +packet: git< hunk 1 3 1 5 +packet: git< hunk 10 2 12 2 +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +If the tool responds with hunks and "success", Git marks those lines +as changed and feeds them into the standard diff pipeline. Git may +still slide or regroup those changes against matching context for +display, exactly as it compacts its own diffs, so the tool controls +which lines are reported as changed, not the precise hunk boundaries. +Patch output features (word diff, function context, color) work +normally. Summary formats such as `--stat` still compute their counts +with the builtin diff for now; see "Which features consult the diff +process" below for the full picture and the reasoning behind it. + +If no hunk lines precede the flush, followed by "success", Git +treats the files as having no changes: `git diff` produces no output, +`git diff --exit-code` and `--quiet` report success even though the +stored blobs differ, and `git blame` skips the commit, attributing +lines to earlier commits. +The one exception is a change that only adds or removes the file's +trailing newline: it cannot be expressed as line hunks, so when the +line content otherwise matches Git keeps the builtin diff for that +file (preserving the `\ No newline at end of file` marker) instead of +treating the two sides as equal. + +----------------------- +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +If the tool returns well-formed but invalid hunks (out of bounds, +overlapping, or with misaligned unchanged runs), Git warns and falls +back to the builtin diff for that file; the tool stays available for +subsequent files. A malformed hunk line, by contrast (bad syntax, a +nonzero count paired with a start of 0, or more hunks than the file +has lines), is a protocol violation: Git stops the process and does +not send it further requests, as described below. + +In case the tool cannot or does not want to process the content, +it is expected to respond with an "error" status. Git warns and +falls back to the builtin diff algorithm for this file, treating any +status other than "success" or "abort" the same way. The tool +remains available for subsequent files. + +----------------------- +packet: git< 0000 +packet: git< status=error +packet: git< 0000 +----------------------- + +In case the tool cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, it +is expected to respond with an "abort" status. Git silently falls +back to the builtin diff algorithm for this file and does not send +further requests to the tool. + +----------------------- +packet: git< 0000 +packet: git< status=abort +packet: git< 0000 +----------------------- + +If the tool dies during the communication or does not adhere to the +protocol then Git will stop the process and fall back to the builtin +diff algorithm. Git warns once and does not restart the process for +subsequent files. + +Tools should ignore unknown keys in the per-file request to remain +forward-compatible. Future versions of Git may send additional +`command=` values; tools that receive an unrecognized command should +respond with `status=error` rather than terminating. + +Which features consult the diff process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The diff process answers a single question: given two blobs, which +line ranges differ? Whether a particular feature consults it follows +from whether that is the question the feature is really asking. + +Features that ask "which lines changed" use the tool's hunks in place +of the builtin algorithm: + +- `git diff` patch output, together with everything layered on it: + word diff, function context (`-W`), `--color-moved`, the `@@` hunk + headers, and the `-L` line-range display. These operate on the + lines the patch step already emitted, so they reflect the tool's + hunks without any further negotiation. +- `git blame`: a commit whose change the tool reports as equivalent is + skipped, and its lines are attributed to an earlier commit. + +Features that ask a different question do not consult the process, by +design: + +- The pickaxe `-G` searches the textual diff for a pattern; it + asks "does this string appear in the diff," not "did these lines + change." (`-S` runs at an earlier stage and is likewise unaffected.) +- `git patch-id` must produce a stable hash for `git rebase` and + cherry-pick detection; deriving it from a configured tool would make + equal patches hash differently from machine to machine. +- The merge machinery (`git merge-tree`, `rerere`) computes merge + content and conflict signatures rather than display output, so the + tool's hunks must not alter its results. +- `git range-diff` diffs patch text, not source blobs, so source-file + hunks do not apply to it. +- `--check` reports whitespace errors in added lines using the builtin + diff's notion of which lines are added, not the tool's. It can + therefore flag (and exit non-zero on) a line the tool treats as + unchanged and that `git diff` shows as context. Whitespace breakage + is a property of the literal bytes, so `--check` keeps the builtin + partition deliberately; a future change could wire it to the tool if + matching `git diff` exactly became desirable. +- `--raw`, `--name-only`, and `--name-status` compare object ids at + the tree level and never run a line-level diff at all. + +Some features ask "which lines changed" but still use the builtin +algorithm for now, and may consult the process in a later change: the +summary formats (`--stat`, `--numstat`, `--shortstat`); `git log -L`'s +commit selection and parent range propagation (as distinct from its +display, which is covered above); and combined diffs (`--cc` and merge +diffs), whose protocol would have to be extended from a single old/new +pair to one comparison per merge parent. + +`--diff-algorithm` bypasses the process entirely, for every feature +listed above. The whitespace-ignoring options (`-w`, +`--ignore-space-change`, `--ignore-blank-lines`, and the like), +`-I`, and `--anchored` also bypass it for the affected files: +the tool is never told about these options, so it could not honor +them, and Git falls back to the builtin diff, which does. + Defining a custom hunk-header ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Makefile b/Makefile index efaafe1e9348ab..2e706ce58c9d0c 100644 --- a/Makefile +++ b/Makefile @@ -811,6 +811,7 @@ TEST_BUILTINS_OBJS += test-csprng.o TEST_BUILTINS_OBJS += test-date.o TEST_BUILTINS_OBJS += test-delete-gpgsig.o TEST_BUILTINS_OBJS += test-delta.o +TEST_BUILTINS_OBJS += test-diff-process-backend.o TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o TEST_BUILTINS_OBJS += test-dump-cache-tree.o @@ -1140,6 +1141,7 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-process.o LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o diff --git a/diff-process.c b/diff-process.c new file mode 100644 index 00000000000000..541bd283202303 --- /dev/null +++ b/diff-process.c @@ -0,0 +1,499 @@ +/* + * Diff process backend: communicates with a long-running external + * tool via the pkt-line protocol to obtain custom line-matching + * results. The tool controls which lines are marked as changed + * while the display shows the file content (after any textconv + * transformation, if configured). + * + * Protocol: pkt-line over stdin/stdout, following the pattern of + * the long-running filter process protocol (see convert.c). + * + * Handshake: + * git> git-diff-client / version=1 / flush + * tool< git-diff-server / version=1 / flush + * git> capability=hunks / flush + * tool< capability=hunks / flush + * + * Per-file: + * git> command=hunks / pathname= / [old-oid=] / [new-oid=] / flush + * git> / flush + * git> / flush + * tool< hunk + * tool< ... / flush + * tool< status=success / flush + * + * When the tool returns no hunks with status=success, it considers + * the files equivalent. Git will skip the diff for that file. + */ + +#include "git-compat-util.h" +#include "diff-process.h" +#include "diff.h" +#include "diff-provider.h" +#include "gettext.h" +#include "hex.h" +#include "repository.h" +#include "sigchain.h" +#include "userdiff.h" +#include "sub-process.h" +#include "pkt-line.h" +#include "strbuf.h" +#include "xdiff/xdiff.h" + +#define CAP_HUNKS (1u << 0) + +struct diff_subprocess { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_diff_process_fn(struct subprocess_entry *subprocess) +{ + static int versions[] = { 1, 0 }; + static struct subprocess_capability capabilities[] = { + { "hunks", CAP_HUNKS }, + { NULL, 0 } + }; + struct diff_subprocess *entry = + container_of(subprocess, struct diff_subprocess, subprocess); + + return subprocess_handshake(subprocess, "git-diff", + versions, NULL, + capabilities, + &entry->supported_capabilities); +} + +static struct diff_subprocess *get_or_launch_process( + struct userdiff_driver *drv) +{ + struct diff_subprocess *entry; + + if (drv->diff_subprocess) + return drv->diff_subprocess; + + entry = xcalloc(1, sizeof(*entry)); + if (subprocess_start_command(&entry->subprocess, drv->process, + start_diff_process_fn)) { + free(entry); + drv->diff_process_failed = 1; + return NULL; + } + + drv->diff_subprocess = entry; + return entry; +} + +static int send_file_content(int fd, const char *buf, long size) +{ + int ret = 0; + + if (size < 0) + return -1; + if (size > 0) + ret = write_packetized_from_buf_no_flush(buf, size, fd); + if (ret) + return ret; + return packet_flush_gently(fd); +} + +/* + * A hunk in the diff process's presentation coordinates: the line + * numbering it reports over the protocol. Kept distinct from struct + * xdl_hunk (xdiff's coordinates) so that only translated hunks ever + * reach the diff algorithm; diff_process_hunk_to_xdl() is the single + * crossing point. + */ +struct diff_process_hunk { + long old_start, old_count; + long new_start, new_count; +}; + +/* + * Parse one non-negative decimal field of a hunk line into *out and + * advance *line past it. Fields must be plain decimal with no leading + * whitespace or sign (isdigit() takes an unsigned char to stay defined + * for high-bit bytes). The first three fields are followed by a single + * space; the last (is_last) is followed by end-of-string or a space. + * Trailing space-separated tokens after the last field are allowed and + * ignored, so a future protocol version can append fields (e.g. a + * "moved" marker) without older tools rejecting the line -- mirroring + * the request-side rule that tools ignore unknown keys. + */ +static int parse_hunk_field(const char **line, long *out, int is_last) +{ + const char *p = *line; + char *end; + + if (!isdigit((unsigned char)*p)) + return -1; + errno = 0; + *out = strtol(p, &end, 10); + if (errno || end == p) + return -1; + if (is_last) { + if (*end != '\0' && *end != ' ') + return -1; + } else { + if (*end != ' ') + return -1; + end++; + } + *line = end; + return 0; +} + +static int parse_hunk_line(const char *line, + struct diff_process_hunk *presented) +{ + /* Format: "hunk " */ + if (!skip_prefix(line, "hunk ", &line)) + return -1; + if (parse_hunk_field(&line, &presented->old_start, 0) || + parse_hunk_field(&line, &presented->old_count, 0) || + parse_hunk_field(&line, &presented->new_start, 0) || + parse_hunk_field(&line, &presented->new_count, 1)) + return -1; + return 0; +} + +/* + * Translate a hunk from the diff process's presentation coordinates + * into xdiff's. + * + * Protocol starts are already 1-based positions (the line a change + * sits before), the same numbering xdiff uses, so the only adjustment + * is for an empty file side: "git diff" addresses it with a start of 0 + * and a count of 0 (e.g. "0 0 1 5" adds five lines to an empty old + * side), and since xdiff uses start-1 as an array index that 0 becomes + * 1 here. This is NOT the full inverse of xdl_emit_hunk_hdr() + * (xdiff/xutils.c): that emitter shifts a count-0 range to start-1 for + * the displayed "@@" header, but the protocol keeps the unshifted + * 1-based position for a mid-file insert or delete. This is the single + * point where presentation coordinates become xdiff coordinates, so + * xdl_populate_hunks_from_external() may assume 1-based starts. + * + * Returns -1 for a start of 0 paired with a nonzero count, which names + * no line in either coordinate system. (parse_hunk_line() already + * guarantees non-negative starts and counts.) + */ +static int diff_process_hunk_to_xdl(const struct diff_process_hunk *presented, + struct xdl_hunk *xdl) +{ + long old_start = presented->old_start; + long new_start = presented->new_start; + + if ((!old_start && presented->old_count) || + (!new_start && presented->new_count)) + return -1; + if (!old_start) + old_start = 1; + if (!new_start) + new_start = 1; + + xdl->old_start = old_start; + xdl->old_count = presented->old_count; + xdl->new_start = new_start; + xdl->new_count = presented->new_count; + return 0; +} + +static enum diff_process_result get_hunks( + struct userdiff_driver *drv, + const char *path, + const char *old_buf, long old_size, + const char *new_buf, long new_size, + const struct object_id *oid_a, + const struct object_id *oid_b, + struct xdl_hunk **hunks_out, + size_t *nr_hunks_out) +{ + struct diff_subprocess *backend; + struct child_process *process; + int fd_in, fd_out; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0; + size_t max_hunks; + int len; + char *line; + + backend = get_or_launch_process(drv); + if (!backend) + return DIFF_PROCESS_ERROR; + + if (!(backend->supported_capabilities & CAP_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&backend->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + /* Send request */ + if (packet_write_fmt_gently(fd_in, "command=hunks\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path)) + goto comm_error; + /* + * old-oid/new-oid let the tool key a cache on the blob pair. A + * side is sent only when its content is the raw blob (the caller + * passes NULL otherwise, e.g. for textconv'd content), so an oid + * that is present always names the bytes the tool receives. + */ + if (oid_a && + packet_write_fmt_gently(fd_in, "old-oid=%s\n", oid_to_hex(oid_a))) + goto comm_error; + if (oid_b && + packet_write_fmt_gently(fd_in, "new-oid=%s\n", oid_to_hex(oid_b))) + goto comm_error; + if (packet_flush_gently(fd_in)) + goto comm_error; + + /* Send old file content */ + if (send_file_content(fd_in, old_buf, old_size)) + goto comm_error; + + /* Send new file content */ + if (send_file_content(fd_in, new_buf, new_size)) + goto comm_error; + + /* + * Hunks are non-overlapping and each useful hunk covers at least + * one line, so a valid response cannot contain more hunks than the + * two files have lines, which is bounded by their byte sizes. Cap + * the accumulation accordingly so a misbehaving tool that floods + * hunk lines cannot drive unbounded memory growth before validation. + */ + max_hunks = (size_t)old_size + (size_t)new_size + 1; + + /* Read hunks until flush packet */ + while ((len = packet_read_line_gently(fd_out, NULL, &line)) >= 0 && + line) { + if (parse_hunk_line(line, &presented) < 0) + goto comm_error; + if (diff_process_hunk_to_xdl(&presented, &hunk) < 0) + goto comm_error; + if (nr_hunks >= max_hunks) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'"), drv->process, path); + goto comm_error; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + if (len < 0) + goto comm_error; + + /* Read status */ + if (subprocess_read_status(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + *hunks_out = hunks; + *nr_hunks_out = nr_hunks; + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_OK; + } + + if (!strcmp(status.buf, "abort")) { + /* + * The tool voluntarily withdrew: stop sending requests + * but do not warn (this is not a failure). + */ + backend->supported_capabilities &= ~CAP_HUNKS; + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_SKIP; + } + + /* status=error or unknown status */ + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; + +comm_error: + /* + * Communication failure (broken pipe, malformed response). + * Tear down the process and mark as failed so we do not + * retry on every subsequent file. + */ + drv->diff_process_failed = 1; + drv->diff_subprocess = NULL; + subprocess_stop_command(&backend->subprocess); + free(backend); + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} + +/* + * Whether exactly one of the two blobs ends in a newline. A change + * that only adds or removes the trailing newline is not expressible as + * line hunks, so a tool comparing lines reports the files as equal. + */ +static int eof_newline_differs(const mmfile_t *a, const mmfile_t *b) +{ + int a_nl = a->size > 0 && a->ptr[a->size - 1] == '\n'; + int b_nl = b->size > 0 && b->ptr[b->size - 1] == '\n'; + return a_nl != b_nl; +} + +/* + * Number of lines in a blob, matching xdiff's record count: one per + * newline, plus one more if the last line has no trailing newline. + */ +static long count_lines(const char *buf, long size) +{ + long lines = 0, i; + + for (i = 0; i < size; i++) + if (buf[i] == '\n') + lines++; + if (size > 0 && buf[size - 1] != '\n') + lines++; + return lines; +} + +/* + * Validate the tool's hunks (already in xdiff coordinates) before they + * bypass the diff algorithm. The content-independent rules (in-order, + * non-overlapping, lockstep-aligned, int32-bounded coordinates) are the + * seam's shared provider rule, diff_provider_hunks_check(); this + * function adds the two checks that need the blobs' line counts (a hunk + * past the end of a file, the run after the last hunk) and the + * per-rule diagnostics naming the tool. This is the git layer's job so + * xdiff stays diagnostic-free; on a bad response we warn and the caller + * falls back to the builtin diff. Returns 0 if valid, -1 (after + * warning) otherwise. + */ +static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, + long old_lines, long new_lines, + const char *process, const char *path) +{ + struct diff_provider_hunks_check c = { 0 }; + size_t i; + + for (i = 0; i < nr; i++) { + const struct xdl_hunk *h = &hunks[i]; + + if (h->old_count > old_lines - h->old_start + 1 || + h->new_count > new_lines - h->new_start + 1) { + warning(_("diff process '%s' returned a hunk past the " + "end of '%s'; using the builtin diff"), + process, path); + return -1; + } + switch (diff_provider_hunks_check(&c, h->old_start, + h->old_count, h->new_start, + h->new_count)) { + case PROVIDER_HUNKS_OK: + break; + case PROVIDER_HUNKS_RANGE: + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + process, path); + return -1; + case PROVIDER_HUNKS_OVERLAP: + warning(_("diff process '%s' returned overlapping hunks " + "for '%s'; using the builtin diff"), + process, path); + return -1; + case PROVIDER_HUNKS_MISALIGNED: + warning(_("diff process '%s' returned hunks that leave " + "'%s' misaligned; using the builtin diff"), + process, path); + return -1; + } + } + if (old_lines - c.prev_old_end != new_lines - c.prev_new_end) { + warning(_("diff process '%s' returned hunks that leave '%s' " + "misaligned; using the builtin diff"), + process, path); + return -1; + } + return 0; +} + +enum diff_process_result diff_process_fill_hunks( + struct diff_options *diffopt, + const char *path, + const mmfile_t *file_a, + const mmfile_t *file_b, + const struct object_id *oid_a, + const struct object_id *oid_b, + xpparam_t *xpp) +{ + struct userdiff_driver *drv; + struct xdl_hunk *ext_hunks = NULL; + size_t nr = 0; + enum diff_process_result res; + + if (!diffopt || !path) + return DIFF_PROCESS_SKIP; + if (diffopt->flags.no_diff_process || diffopt->ignore_driver_algorithm) + return DIFF_PROCESS_SKIP; + /* + * Whitespace-ignoring, regex-ignore (-I) and anchored options + * change which lines count as different, but the tool is never + * told about them, so its hunks could not honor them. Rather + * than silently override the user's request, fall back to the + * builtin diff, which does honor these flags. Key this off xpp + * (the parameters this diff actually runs with) rather than + * diffopt, so a caller like blame that keeps its flags outside + * diffopt is covered without a separate guard of its own. + */ + if ((xpp->flags & (XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES)) || + xpp->ignore_regex_nr || xpp->anchors_nr) + return DIFF_PROCESS_SKIP; + + drv = userdiff_find_by_path(diffopt->repo->index, path); + if (!drv || !drv->process) + return DIFF_PROCESS_SKIP; + if (drv->diff_process_failed) + return DIFF_PROCESS_SKIP; + + res = get_hunks(drv, path, + file_a->ptr, file_a->size, + file_b->ptr, file_b->size, + oid_a, oid_b, + &ext_hunks, &nr); + if (res == DIFF_PROCESS_OK) { + if (!nr) { + free(ext_hunks); + /* + * Zero hunks means the tool considers the line + * content identical, but it cannot express a + * trailing-newline-only change. When that is the + * actual difference, fall back to the builtin diff + * so the "\ No newline at end of file" marker is + * preserved instead of reporting the files equal. + */ + if (eof_newline_differs(file_a, file_b)) + return DIFF_PROCESS_SKIP; + return DIFF_PROCESS_EQUIVALENT; + } + if (validate_external_hunks(ext_hunks, nr, + count_lines(file_a->ptr, file_a->size), + count_lines(file_b->ptr, file_b->size), + drv->process, path) < 0) { + free(ext_hunks); + return DIFF_PROCESS_SKIP; + } + xpp->external_hunks = ext_hunks; + xpp->external_hunks_nr = nr; + return DIFF_PROCESS_OK; + } + if (res == DIFF_PROCESS_ERROR) { + warning(_("diff process '%s' failed for '%s'," + " falling back to builtin diff"), + drv->process, path); + return DIFF_PROCESS_ERROR; + } + return DIFF_PROCESS_SKIP; +} diff --git a/diff-process.h b/diff-process.h new file mode 100644 index 00000000000000..8d00dafe1d9eac --- /dev/null +++ b/diff-process.h @@ -0,0 +1,49 @@ +#ifndef DIFF_PROCESS_H +#define DIFF_PROCESS_H + +#include "xdiff/xdiff.h" + +struct diff_options; +struct object_id; + +enum diff_process_result { + DIFF_PROCESS_ERROR = -1, /* failed; caller falls back to builtin */ + DIFF_PROCESS_OK = 0, /* hunks populated in xpp */ + DIFF_PROCESS_SKIP, /* process did not apply: use builtin */ + DIFF_PROCESS_EQUIVALENT, /* tool says files are equivalent */ +}; + +/* + * Consult the diff process configured for 'path' and populate + * xpp->external_hunks with the returned hunks. + * + * Handles driver lookup, flag checks (--no-ext-diff, + * --diff-algorithm), subprocess management, and error reporting. + * + * Returns DIFF_PROCESS_OK when hunks are populated in xpp. + * The caller owns xpp->external_hunks and must free() it. + * + * Returns DIFF_PROCESS_EQUIVALENT when the tool returns no hunks and + * the blobs are not a trailing-newline-only change (files are + * considered identical); caller should skip diff/blame. + * Returns DIFF_PROCESS_SKIP when no process applies; caller + * should use the builtin diff algorithm. + * Returns DIFF_PROCESS_ERROR on tool failure (already warned); + * caller should fall back to the builtin diff algorithm. + * + * oid_a/oid_b, when non-NULL, are sent to the tool as old-oid/new-oid + * so it can key a cache on the blob pair. Pass NULL for a side whose + * content is not the raw blob (e.g. textconv'd) or whose object name is + * unknown, so any oid that is sent always names the bytes the tool + * receives. + */ +enum diff_process_result diff_process_fill_hunks( + struct diff_options *diffopt, + const char *path, + const mmfile_t *file_a, + const mmfile_t *file_b, + const struct object_id *oid_a, + const struct object_id *oid_b, + xpparam_t *xpp); + +#endif /* DIFF_PROCESS_H */ diff --git a/diff.c b/diff.c index 00026b35c1aae2..d36f803a1769ff 100644 --- a/diff.c +++ b/diff.c @@ -27,6 +27,7 @@ #include "utf8.h" #include "odb.h" #include "userdiff.h" +#include "diff-process.h" #include "submodule.h" #include "hashmap.h" #include "mem-pool.h" @@ -4236,6 +4237,25 @@ static void builtin_diff(const char *name_a, xpp.ignore_regex_nr = o->ignore_regex_nr; xpp.anchors = o->anchors; xpp.anchors_nr = o->anchors_nr; + + /* + * Send the blob oids only for a side whose content is the + * raw blob: textconv rewrites the bytes, and a working-tree + * side has no stored oid, so pass NULL there rather than an + * oid that would not name what the tool receives. + */ + if (diff_process_fill_hunks(o, name_a, &mf1, &mf2, + (textconv_one || !one->oid_valid) ? NULL : &one->oid, + (textconv_two || !two->oid_valid) ? NULL : &two->oid, + &xpp) + == DIFF_PROCESS_EQUIVALENT) { + if (textconv_one) + free(mf1.ptr); + if (textconv_two) + free(mf2.ptr); + goto free_ab_and_return; + } + xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; xecfg.flags = XDL_EMIT_FUNCNAMES; @@ -4280,6 +4300,7 @@ static void builtin_diff(const char *name_a, } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); + free(xpp.external_hunks); if (o->word_diff) free_diff_words_data(&ecbdata); if (textconv_one) diff --git a/diff.h b/diff.h index 1098b2db082823..83cc60e9ed2eac 100644 --- a/diff.h +++ b/diff.h @@ -173,6 +173,9 @@ struct diff_flags { */ unsigned allow_external; + /** Disables diff..process. */ + unsigned no_diff_process; + /** * For communication between the calling program and the options parser; * tell the calling program to signal the presence of difference using diff --git a/meson.build b/meson.build index 8bb39cb06d38b3..11f7a73ad2e89e 100644 --- a/meson.build +++ b/meson.build @@ -328,6 +328,7 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-process.c', 'diff-provider.c', 'diff.c', 'diffcore-break.c', diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..6abcda4afb89c0 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -12,6 +12,7 @@ test_tool_sources = [ 'test-date.c', 'test-delete-gpgsig.c', 'test-delta.c', + 'test-diff-process-backend.c', 'test-dir-iterator.c', 'test-drop-caches.c', 'test-dump-cache-tree.c', diff --git a/t/helper/test-diff-process-backend.c b/t/helper/test-diff-process-backend.c new file mode 100644 index 00000000000000..c2ec532c4a5cfa --- /dev/null +++ b/t/helper/test-diff-process-backend.c @@ -0,0 +1,381 @@ +/* + * Test backend for the long-running diff process protocol + * (see diff-process.c and Documentation/gitattributes.adoc). + * + * Usage: test-tool diff-process-backend --mode= [--log=] + * + * Implements the server side of the pkt-line handshake and a per-file + * response loop. The --mode= switch selects the response shape + * (success, error, abort, crash, malformed hunks). + * + * Per-file request from Git: + * + * packet: git> command=hunks + * packet: git> pathname= + * packet: git> [old-oid=] (omitted for textconv/worktree) + * packet: git> [new-oid=] + * packet: git> 0000 + * packet: git> OLD_CONTENT + * packet: git> 0000 + * packet: git> NEW_CONTENT + * packet: git> 0000 + * + * Response varies by --mode (default: whole-file): + * + * whole-file packet: git< hunk <1|0> <1|0> + * (start is 0 for an empty side, matching git diff) + * fixed-hunk packet: git< hunk 5 2 5 2 + * no-hunks (no hunk packets) + * bad-hunk packet: git< hunk 999 1 999 1 + * bad-parse packet: git< garbage not a hunk + * bad-sync packet: git< hunk 1 2 1 1 + * bad-gap packet: git< hunk 1 1 3 1 + * bad-start packet: git< hunk 0 1 1 1 + * multi-hunk packet: git< hunk 5 2 5 2 + * packet: git< hunk 9 2 9 2 + * insert packet: git< hunk 3 0 3 2 (mid-file count-0 insertion) + * flood packet: git< hunk 1 1 1 1 (x100000) + * overlap packet: git< hunk 1 5 1 5 + * packet: git< hunk 3 2 3 2 + * no-cap (omits capability=hunks during handshake) + * error (status=error instead of status=success) + * abort (status=abort instead of status=success) + * crash exit(1) before sending any response + * + * All success modes (not error/abort/crash) end with: + * + * packet: git< 0000 + * packet: git< status=success + * packet: git< 0000 + * + * Each request is logged to --log as: + * + * command= pathname= old-oid= new-oid= old= new= + */ + +#include "test-tool.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "strbuf.h" + +static FILE *logfile; + +enum mode { + MODE_WHOLE_FILE, + MODE_FIXED_HUNK, + MODE_NO_HUNKS, + MODE_BAD_HUNK, + MODE_BAD_PARSE, + MODE_BAD_SYNC, + MODE_BAD_GAP, + MODE_BAD_START, + MODE_MULTI_HUNK, + MODE_INSERT, + MODE_FLOOD, + MODE_OVERLAP, + MODE_NO_CAP, + MODE_ERROR, + MODE_ABORT, + MODE_CRASH, +}; + +static enum mode parse_mode(const char *s) +{ + if (!strcmp(s, "whole-file")) + return MODE_WHOLE_FILE; + if (!strcmp(s, "fixed-hunk")) + return MODE_FIXED_HUNK; + if (!strcmp(s, "no-hunks")) + return MODE_NO_HUNKS; + if (!strcmp(s, "bad-hunk")) + return MODE_BAD_HUNK; + if (!strcmp(s, "bad-parse")) + return MODE_BAD_PARSE; + if (!strcmp(s, "bad-sync")) + return MODE_BAD_SYNC; + if (!strcmp(s, "bad-gap")) + return MODE_BAD_GAP; + if (!strcmp(s, "bad-start")) + return MODE_BAD_START; + if (!strcmp(s, "multi-hunk")) + return MODE_MULTI_HUNK; + if (!strcmp(s, "insert")) + return MODE_INSERT; + if (!strcmp(s, "flood")) + return MODE_FLOOD; + if (!strcmp(s, "overlap")) + return MODE_OVERLAP; + if (!strcmp(s, "no-cap")) + return MODE_NO_CAP; + if (!strcmp(s, "error")) + return MODE_ERROR; + if (!strcmp(s, "abort")) + return MODE_ABORT; + if (!strcmp(s, "crash")) + return MODE_CRASH; + die("unknown --mode=%s", s); +} + +/* + * Read "key=value" packets up to a flush, capturing "command" and + * "pathname". Returns 1 if a request was read, 0 on EOF. + * + * The first packet uses the gentle variant so that a clean shutdown + * by Git (EOF) does not produce a spurious "the remote end hung up + * unexpectedly" on stderr. Subsequent packets use the non-gentle + * variant: once inside a request, truncation is a protocol violation + * and dying loudly is the correct response. + */ +static int read_request_header(char **command, char **pathname, + char **old_oid, char **new_oid) +{ + int first = 1; + char *line; + + *command = *pathname = *old_oid = *new_oid = NULL; + for (;;) { + const char *value; + + if (first) { + if (packet_read_line_gently(0, NULL, &line) < 0) + return 0; + first = 0; + } else { + line = packet_read_line(0, NULL); + } + if (!line) + break; + if (skip_prefix(line, "command=", &value)) + *command = xstrdup(value); + else if (skip_prefix(line, "pathname=", &value)) + *pathname = xstrdup(value); + else if (skip_prefix(line, "old-oid=", &value)) + *old_oid = xstrdup(value); + else if (skip_prefix(line, "new-oid=", &value)) + *new_oid = xstrdup(value); + } + return 1; +} + +static size_t count_lines(const struct strbuf *buf) +{ + size_t lines = 0; + + for (size_t i = 0; i < buf->len; i++) + if (buf->buf[i] == '\n') + lines++; + + return lines + (buf->len > 0 && buf->buf[buf->len - 1] != '\n'); +} + +static void send_status(const char *status) +{ + packet_flush(1); + packet_write_fmt(1, "%s\n", status); + packet_flush(1); +} + +static void respond(enum mode mode, + const struct strbuf *old_buf, + const struct strbuf *new_buf) +{ + switch (mode) { + case MODE_ERROR: + send_status("status=error"); + return; + case MODE_ABORT: + send_status("status=abort"); + return; + case MODE_CRASH: + exit(1); + case MODE_FIXED_HUNK: + packet_write_fmt(1, "hunk 5 2 5 2\n"); + break; + case MODE_BAD_HUNK: + packet_write_fmt(1, "hunk 999 1 999 1\n"); + break; + case MODE_BAD_PARSE: + packet_write_fmt(1, "garbage not a hunk\n"); + break; + case MODE_BAD_SYNC: + packet_write_fmt(1, "hunk 1 2 1 1\n"); + break; + case MODE_BAD_GAP: + /* + * Globally balanced (1 changed line on each side, so the + * total unchanged counts match) but the gap before the + * change differs between sides: old line 1 vs new line 3. + * Exercises the per-gap lockstep-alignment check. + */ + packet_write_fmt(1, "hunk 1 1 3 1\n"); + break; + case MODE_BAD_START: + /* + * A start of 0 is valid only for an empty (count 0) range; + * pairing it with a nonzero count names no line in either + * the protocol's or xdiff's coordinates, so the translation + * rejects it and git falls back to the builtin diff. + */ + packet_write_fmt(1, "hunk 0 1 1 1\n"); + break; + case MODE_MULTI_HUNK: + /* + * Two valid, non-overlapping, gap-aligned hunks. Exercises + * the accepting branch of the per-gap lockstep check with a + * non-zero previous-hunk end (the realistic two-region case). + */ + packet_write_fmt(1, "hunk 5 2 5 2\n"); + packet_write_fmt(1, "hunk 9 2 9 2\n"); + break; + case MODE_INSERT: + /* + * A mid-file pure insertion (count 0 on the old side) in the + * protocol's 1-based-position form: 2 lines inserted before + * old line 3. Exercises the count-0 path, which uses the + * unshifted position (not git diff's "-3,0" display start). + */ + packet_write_fmt(1, "hunk 3 0 3 2\n"); + break; + case MODE_FLOOD: { + /* + * Emit far more hunks than any small file has lines, so Git + * trips its accumulation cap and falls back before reading + * them all. + */ + int i; + for (i = 0; i < 100000; i++) + packet_write_fmt(1, "hunk 1 1 1 1\n"); + break; + } + case MODE_OVERLAP: + packet_write_fmt(1, "hunk 1 5 1 5\n"); + packet_write_fmt(1, "hunk 3 2 3 2\n"); + break; + case MODE_NO_HUNKS: + break; + case MODE_NO_CAP: + case MODE_WHOLE_FILE: { + size_t old_lines = count_lines(old_buf); + size_t new_lines = count_lines(new_buf); + /* + * Match git diff output: start=0 when count=0 + * (empty file side), 1 otherwise. + */ + packet_write_fmt(1, "hunk %"PRIuMAX" %"PRIuMAX + " %"PRIuMAX" %"PRIuMAX"\n", + (uintmax_t)(old_lines ? 1 : 0), + (uintmax_t)old_lines, + (uintmax_t)(new_lines ? 1 : 0), + (uintmax_t)new_lines); + break; + } + } + send_status("status=success"); +} + +static void command_loop(enum mode mode) +{ + for (;;) { + char *command = NULL, *pathname = NULL; + char *old_oid = NULL, *new_oid = NULL; + struct strbuf obuf = STRBUF_INIT; + struct strbuf nbuf = STRBUF_INIT; + + if (!read_request_header(&command, &pathname, + &old_oid, &new_oid)) + break; /* EOF: Git closed its end */ + + read_packetized_to_strbuf(0, &obuf, 0); + read_packetized_to_strbuf(0, &nbuf, 0); + + if (logfile) { + fprintf(logfile, + "command=%s pathname=%s old-oid=%s new-oid=%s" + " old=%.*s new=%.*s\n", + command ? command : "(none)", + pathname ? pathname : "(none)", + old_oid ? old_oid : "(none)", + new_oid ? new_oid : "(none)", + (int)(strchrnul(obuf.buf, '\n') - obuf.buf), + obuf.buf, + (int)(strchrnul(nbuf.buf, '\n') - nbuf.buf), + nbuf.buf); + fflush(logfile); + } + + respond(mode, &obuf, &nbuf); + + free(command); + free(pathname); + free(old_oid); + free(new_oid); + strbuf_release(&obuf); + strbuf_release(&nbuf); + } +} + +static void handshake(enum mode mode) +{ + char *line; + + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "git-diff-client")) + die("bad welcome: '%s'", line ? line : "(eof)"); + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "version=1")) + die("bad version: '%s'", line ? line : "(eof)"); + if (packet_read_line(0, NULL)) + die("expected flush after version"); + + packet_write_fmt(1, "git-diff-server\n"); + packet_write_fmt(1, "version=1\n"); + packet_flush(1); + + /* Drain capabilities advertised by Git */ + while ((line = packet_read_line(0, NULL))) + ; /* drain */ + + /* Respond with our capabilities (or none for no-cap mode) */ + if (mode != MODE_NO_CAP) + packet_write_fmt(1, "capability=hunks\n"); + packet_flush(1); +} + +static const char *const usage_str[] = { + "test-tool diff-process-backend --mode= [--log=]", + NULL +}; + +int cmd__diff_process_backend(int argc, const char **argv) +{ + const char *mode_str = NULL, *log_path = NULL; + enum mode mode = MODE_WHOLE_FILE; + struct option options[] = { + OPT_STRING(0, "mode", &mode_str, "mode", + "response shape (default whole-file);" + " see the file header for the full list of modes"), + OPT_STRING(0, "log", &log_path, "path", + "append per-request summary to this file"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, usage_str, 0); + if (argc) + usage_with_options(usage_str, options); + + if (mode_str) + mode = parse_mode(mode_str); + + if (log_path) { + logfile = fopen(log_path, "a"); + if (!logfile) + die_errno("failed to open log '%s'", log_path); + } + + handshake(mode); + command_loop(mode); + + if (logfile && fclose(logfile)) + die_errno("error closing log"); + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..3c3f95269c6279 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -22,6 +22,7 @@ static struct test_cmd cmds[] = { { "date", cmd__date }, { "delete-gpgsig", cmd__delete_gpgsig }, { "delta", cmd__delta }, + { "diff-process-backend", cmd__diff_process_backend }, { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, { "dump-cache-tree", cmd__dump_cache_tree }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..a5bb7555162c8e 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -15,6 +15,7 @@ int cmd__csprng(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); int cmd__delete_gpgsig(int argc, const char **argv); +int cmd__diff_process_backend(int argc, const char **argv); int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); diff --git a/t/meson.build b/t/meson.build index 71c219f2129f84..5f7a90047ef8c5 100644 --- a/t/meson.build +++ b/t/meson.build @@ -512,6 +512,7 @@ integration_tests = [ 't4072-diff-max-depth.sh', 't4073-diff-stat-name-width.sh', 't4074-diff-shifted-matched-group.sh', + 't4080-diff-process.sh', 't4100-apply-stat.sh', 't4101-apply-nonl.sh', 't4102-apply-rename.sh', diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh new file mode 100755 index 00000000000000..3b75df082e8def --- /dev/null +++ b/t/t4080-diff-process.sh @@ -0,0 +1,645 @@ +#!/bin/sh + +test_description='diff process via long-running process' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +# See t/helper/test-diff-process-backend.c for the backend implementation +# and available --mode= options. + +BACKEND="test-tool diff-process-backend" + +test_expect_success 'setup' ' + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + + # boundary.c: 10 lines, changes at 5-6 and 9-10. + # Used by: hunk boundaries, error fallback, crash, bad hunks, overlap. + cat >boundary.c <<-\EOF && + line1 + line2 + line3 + line4 + OLD5 + OLD6 + line7 + line8 + OLD9 + OLD10 + EOF + git add boundary.c && + + # worddiff.c: single-line function, value changes 1 -> 999. + # Used by: word-diff, --diff-algorithm, --no-ext-diff, --stat. + cat >worddiff.c <<-\EOF && + int value(void) { return 1; } + EOF + git add worddiff.c && + + # newfile.c: single-line function, value changes 42 -> 99. + # Used by: modified file, --exit-code, multiple drivers. + cat >newfile.c <<-\EOF && + int new_func(void) { return 42; } + EOF + git add newfile.c && + + # logtest.c: single-line function for log/format-patch tests. + # Needs two commits so log -1 has a diff. + cat >logtest.c <<-\EOF && + int logfunc(void) { return 1; } + EOF + git add logtest.c && + + # one.c/two.c: two-file pair for error/abort/startup-failure tests. + cat >one.c <<-\EOF && + int first(void) { return 1; } + EOF + cat >two.c <<-\EOF && + int second(void) { return 2; } + EOF + git add one.c two.c && + + git commit -m "initial" && + + # Second commit for logtest.c (so log -1 has something to show). + cat >logtest.c <<-\EOF && + int logfunc(void) { return 2; } + EOF + git add logtest.c && + git commit -m "change logtest.c" && + + # Working tree modifications (not committed). + cat >boundary.c <<-\EOF && + line1 + line2 + line3 + line4 + NEW5 + NEW6 + line7 + line8 + NEW9 + NEW10 + EOF + + cat >worddiff.c <<-\EOF && + int value(void) { return 999; } + EOF + + cat >newfile.c <<-\EOF && + int new_func(void) { return 99; } + EOF + + cat >one.c <<-\EOF && + int first(void) { return 10; } + EOF + + cat >two.c <<-\EOF + int second(void) { return 20; } + EOF +' + +# +# Core behavior: the tool controls which lines are marked as changed. +# + +test_expect_success 'diff process hunk boundaries affect output' ' + # The file has changes at lines 5-6 and 9-10, but fixed-hunk + # only reports lines 5-6 as changed. Lines 9-10 should not + # appear as changed in the output. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + diff boundary.c >actual && + test_grep "^-OLD5" actual && + test_grep "^-OLD6" actual && + test_grep "^+NEW5" actual && + test_grep "^+NEW6" actual && + test_grep ! "^-OLD9" actual && + test_grep ! "^-OLD10" actual && + test_grep ! "^+NEW9" actual && + test_grep ! "^+NEW10" actual +' + +test_expect_success 'diff process accepts valid multi-hunk output' ' + # multi-hunk reports both changed regions (5-6 and 9-10) as two + # gap-aligned hunks. This exercises the accepting branch of the + # per-gap lockstep check (non-zero previous-hunk end) and must + # produce a correct two-region diff with the lines between the + # hunks kept as context. + git -c diff.cdiff.process="$BACKEND --mode=multi-hunk" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + test_grep "^ line7" actual && + test_grep "^ line8" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process accepts a mid-file count-0 insertion' ' + # insert mode reports "hunk 3 0 3 2": a pure insertion (count 0 on + # the old side) in the protocol 1-based-position form. Exercises + # the count-0 hunk path that the other valid-hunk modes (full + # replacements, equal-count modifies) never hit. Empty stderr is + # the discriminator: a mishandled count-0 start would be rejected + # by the lockstep check and warn. + cat >insert.c <<-\EOF && + a + b + c + d + e + EOF + git add insert.c && + git commit -m "add insert.c" && + cat >insert.c <<-\EOF && + a + b + X + Y + c + d + e + EOF + git -c diff.cdiff.process="$BACKEND --mode=insert" \ + diff insert.c >actual 2>stderr && + test_grep "^+X" actual && + test_grep "^+Y" actual && + test_grep "^ c" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with modified file' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- newfile.c >actual 2>stderr && + test_grep "return 99" actual && + test_grep "pathname=newfile.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with added file (empty old side)' ' + cat >added.c <<-\EOF && + int added(void) { return 1; } + EOF + git add added.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --cached -- added.c >actual 2>stderr && + test_grep "added" actual && + test_grep "pathname=added.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with deleted file (empty new side)' ' + git add added.c && + git commit -m "commit added.c" && + git rm added.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --cached -- added.c >actual 2>stderr && + test_grep "deleted file" actual && + test_grep "pathname=added.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process skipped for binary files' ' + printf "\\0binary" >binary.c && + git add binary.c && + git commit -m "add binary" && + printf "\\0changed" >binary.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- binary.c >actual && + test_grep "Binary files" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not consulted for unmatched driver' ' + echo "not tracked by cdiff" >unmatched.txt && + git add unmatched.txt && + git commit -m "add unmatched.txt" && + + echo "modified" >unmatched.txt && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- unmatched.txt >actual && + test_grep "modified" actual && + test_path_is_missing backend.log +' + +test_expect_success 'multiple drivers use separate processes' ' + echo "*.h diff=hdiff" >>.gitattributes && + git add .gitattributes && + + cat >multi.h <<-\EOF && + int header(void) { return 1; } + EOF + git add multi.h && + git commit -m "add multi.h" && + + cat >multi.h <<-\EOF && + int header(void) { return 2; } + EOF + + test_when_finished "rm -f backend-c.log backend-h.log" && + git -c diff.cdiff.process="$BACKEND --log=backend-c.log" \ + -c diff.hdiff.process="$BACKEND --log=backend-h.log" \ + diff -- newfile.c multi.h >actual 2>stderr && + test_grep "pathname=newfile.c" backend-c.log && + test_grep "pathname=multi.h" backend-h.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works alongside textconv' ' + write_script uppercase-filter <<-\EOF && + tr "a-z" "A-Z" <"$1" + EOF + + cat >textconv.c <<-\EOF && + hello world + EOF + git add textconv.c && + git commit -m "add textconv.c" && + + cat >textconv.c <<-\EOF && + goodbye world + EOF + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.textconv="./uppercase-filter" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- textconv.c >actual 2>stderr && + # The diff process receives textconv-transformed (uppercase) content. + test_grep "pathname=textconv.c" backend.log && + test_grep "old=HELLO WORLD" backend.log && + test_grep "new=GOODBYE WORLD" backend.log && + test_must_be_empty stderr +' + +# +# Downstream features: word diff, log, equivalent files, exit code. +# + +test_expect_success 'diff process with --word-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --word-diff worddiff.c >actual 2>stderr && + test_grep "\[-1;-\]" actual && + test_grep "{+999;+}" actual && + test_grep "pathname=worddiff.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with git log -p' ' + # With no-hunks mode, the tool says the files are equivalent, + # so log -p should show the commit but no diff content. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + log -1 -p -- logtest.c >actual 2>stderr && + test_grep "change logtest.c" actual && + test_grep ! "return 2" actual && + test_grep "command=hunks pathname=logtest.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process no hunks suppresses diff output' ' + cat >nohunks.c <<-\EOF && + int zero(void) { return 0; } + EOF + git add nohunks.c && + git commit -m "add nohunks.c" && + + cat >nohunks.c <<-\EOF && + int zero(void) { return 999; } + EOF + + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff nohunks.c >actual && + test_must_be_empty actual +' + +test_expect_success 'diff process no hunks with --exit-code returns success' ' + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --exit-code nohunks.c +' + +test_expect_success 'diff process equivalent commit: --exit-code and --quiet agree' ' + # A committed blob pair (not a worktree file) whose oids differ but + # the tool reports equivalent. --exit-code and --quiet must agree + # with the shown diff (empty) and report success, not fall back to + # the byte-level "oids differ" answer. + cat >ecq.c <<-\EOF && + alpha + EOF + git add ecq.c && + git commit -m "ecq v1" && + cat >ecq.c <<-\EOF && + beta + EOF + git add ecq.c && + git commit -m "ecq v2" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --exit-code HEAD^ HEAD -- ecq.c && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --quiet HEAD^ HEAD -- ecq.c +' + +test_expect_success 'diff process falls back for trailing-newline-only change' ' + test_when_finished "rm -f backend.log" && + printf "a\nb\nc\n" >eofnl.c && + git add eofnl.c && + git commit -m "add eofnl.c" && + printf "a\nb\nc" >eofnl.c && + # Same lines, only the final newline removed. The tool reports + # no hunks (it sees identical lines), but that change is not + # expressible as hunks, so git falls back to the builtin diff + # rather than treating the files as equivalent. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff eofnl.c >actual 2>stderr && + test_grep "No newline at end of file" actual && + test_grep "pathname=eofnl.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process falls back for added file (empty old side)' ' + test_when_finished "rm -f backend.log" && + printf "x\ny\nz\n" >addnl.c && + git add addnl.c && + # The empty old side has no trailing newline while the new side + # does, so the newline fallback shows the addition rather than + # letting no-hunks suppress the whole new file. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --cached addnl.c >actual 2>stderr && + test_grep "^+x" actual && + test_grep "pathname=addnl.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process with --exit-code and hunks returns failure' ' + test_expect_code 1 git -c diff.cdiff.process="$BACKEND" \ + diff --exit-code newfile.c +' + +# +# Bypass mechanisms: flags and commands that skip the diff process. +# + +test_expect_success 'diff process bypassed by --diff-algorithm' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --diff-algorithm=patience worddiff.c >actual && + test_grep "return 999" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process bypassed under whitespace-ignoring flags' ' + test_when_finished "rm -f backend.log" && + printf "a\nb\nc\n" >wsbypass.c && + git add wsbypass.c && + git commit -m "add wsbypass.c" && + printf "a\n b \nc\n" >wsbypass.c && + # The tool is never told about these options and could not honor + # them, so git bypasses the process for each (covering the whole + # XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES mask, not just -w). + for opt in -w -b --ignore-space-at-eol --ignore-blank-lines + do + rm -f backend.log && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff $opt wsbypass.c >actual 2>stderr && + test_path_is_missing backend.log && + test_must_be_empty stderr || + return 1 + done && + # -w additionally suppresses the whitespace-only change via the + # builtin diff that now runs. + git -c diff.cdiff.process="$BACKEND" diff -w wsbypass.c >actual && + test_must_be_empty actual +' + +# +# Error handling and fallback. +# + +test_expect_success 'diff process fallback on tool error status' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=error --log=backend.log" \ + diff boundary.c >actual 2>stderr && + # Fallback produces the full builtin diff (both change regions). + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + # Tool was contacted (it replied with error, not crash). + test_grep "command=hunks pathname=boundary.c" backend.log && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process error keeps tool available for next file' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=error --log=backend.log" \ + diff -- one.c two.c >actual 2>stderr && + # Unlike abort, error keeps the tool available: both files + # are sent to the tool (and both fall back). + test_grep "pathname=one.c" backend.log && + test_grep "pathname=two.c" backend.log && + test_grep "return 10" actual && + test_grep "return 20" actual && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process abort disables for session' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=abort --log=backend.log" \ + diff -- one.c two.c >actual 2>stderr && + # Both files should still produce diff output via fallback. + test_grep "return 10" actual && + test_grep "return 20" actual && + # The tool aborts on the first file and git clears its + # capability. The second file never contacts the tool. + test_grep "pathname=one.c" backend.log && + test_grep ! "pathname=two.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process fallback on tool crash' ' + git -c diff.cdiff.process="$BACKEND --mode=crash" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + # Crash is a communication failure, so a warning is emitted. + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process startup failure only warns once' ' + git -c diff.cdiff.process="/nonexistent/tool" \ + diff -- one.c two.c >actual 2>stderr && + # Both files produce diff output via fallback. + test_grep "return 10" actual && + test_grep "return 20" actual && + # Sentinel prevents repeated warnings: only one, not one per file. + test_grep "diff process.*failed" stderr >warnings && + test_line_count = 1 warnings +' + + +test_expect_success 'diff process fallback on bad hunks' ' + git -c diff.cdiff.process="$BACKEND --mode=bad-hunk" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + test_grep "hunk past the end" stderr +' + +test_expect_success 'diff process fallback on mismatched unchanged totals' ' + cat >synctest.c <<-\EOF && + line1 + line2 + line3 + EOF + git add synctest.c && + git commit -m "add synctest.c" && + + cat >synctest.c <<-\EOF && + line1 + changed + line3 + EOF + + # bad-sync reports hunk 1 2 1 1: marks 2 old lines and 1 new + # line as changed, leaving 1 unchanged old vs 2 unchanged new. + # The synchronization invariant fails and git falls back. + git -c diff.cdiff.process="$BACKEND --mode=bad-sync" \ + diff synctest.c >actual 2>stderr && + test_grep "changed" actual && + test_grep "misaligned" stderr +' + +test_expect_success 'diff process fallback on misaligned hunk gap' ' + # bad-gap reports hunk 1 1 3 1 on boundary.c: one changed line + # on each side, so the total unchanged counts match, but the + # unchanged run before the change differs (old line 1 vs new + # line 3). A global count check would accept this and emit a + # corrupt diff; the per-gap lockstep check rejects it and git + # falls back to the builtin algorithm. + git -c diff.cdiff.process="$BACKEND --mode=bad-gap" \ + diff boundary.c >actual 2>stderr && + # The builtin fallback shows both changed regions as additions + # (a corrupt-accepted hunk would show NEW5 only as context). + test_grep "^+NEW5" actual && + test_grep "^+NEW9" actual && + test_grep "misaligned" stderr +' + +test_expect_success 'diff process fallback on overlapping hunks' ' + # boundary.c has 10 lines, so both hunks are in bounds + # but they overlap at lines 3-4, triggering the ordering check. + git -c diff.cdiff.process="$BACKEND --mode=overlap" \ + diff boundary.c >actual 2>stderr && + test_grep "NEW5" actual && + test_grep "overlapping hunks" stderr +' + +test_expect_success 'diff process fallback on malformed hunk line' ' + git -c diff.cdiff.process="$BACKEND --mode=bad-parse" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual +' + +test_expect_success 'diff process fallback on start 0 with nonzero count' ' + # bad-start reports hunk 0 1 1 1. A start of 0 is valid only for + # an empty (count 0) range, so the presentation-to-xdiff + # translation rejects it and git falls back to the builtin diff + # instead of handing xdiff an out-of-range start. + git -c diff.cdiff.process="$BACKEND --mode=bad-start" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process caps a flood of hunks and falls back' ' + # flood emits far more hunks than the file has lines. Git must + # stop accumulating and fall back to the builtin diff rather than + # grow memory without bound. + git -c diff.cdiff.process="$BACKEND --mode=flood" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "too many hunks" stderr +' + +test_expect_success 'diff process skipped when tool omits capability' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-cap --log=backend.log" \ + diff boundary.c >actual 2>stderr && + # Builtin diff runs: all changes appear, including lines 9-10 + # that a tool-provided hunk would have narrowed away. + test_grep "^-OLD5" actual && + test_grep "^-OLD9" actual && + # The process launched (creating the log) but was + # never sent a per-file request, so no hunks command is logged. + test_path_is_file backend.log && + test_grep ! "command=hunks" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process receives old-oid and new-oid for a blob pair' ' + test_when_finished "rm -f backend.log" && + cat >oidpair.c <<-\EOF && + int f(void) { return 1; } + EOF + git add oidpair.c && + git commit -m "oidpair v1" && + old=$(git rev-parse HEAD:oidpair.c) && + + cat >oidpair.c <<-\EOF && + int f(void) { return 2; } + EOF + git add oidpair.c && + git commit -m "oidpair v2" && + new=$(git rev-parse HEAD:oidpair.c) && + + # Both sides are stored blobs, so their object names are sent. + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff HEAD^ HEAD -- oidpair.c >actual 2>stderr && + test_grep "old-oid=$old new-oid=$new" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process omits old-oid and new-oid for textconv content' ' + test_when_finished "rm -f backend.log" && + write_script oidcat <<-\EOF && + cat "$1" + EOF + cat >oidtc.c <<-\EOF && + alpha + EOF + git add oidtc.c && + git commit -m "oidtc v1" && + cat >oidtc.c <<-\EOF && + beta + EOF + git add oidtc.c && + git commit -m "oidtc v2" && + + # textconv rewrites the bytes, so the raw-blob object name that + # would otherwise identify each side is omitted. + git -c diff.cdiff.textconv="./oidcat" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff HEAD^ HEAD -- oidtc.c >actual 2>stderr && + test_grep "pathname=oidtc.c" backend.log && + test_grep "old-oid=(none) new-oid=(none)" backend.log && + test_must_be_empty stderr +' + +test_done diff --git a/userdiff.h b/userdiff.h index 51c26e0d4190e5..a98eabe3770cc6 100644 --- a/userdiff.h +++ b/userdiff.h @@ -3,6 +3,7 @@ #include "notes-cache.h" +struct diff_subprocess; struct index_state; struct repository; @@ -33,6 +34,8 @@ struct userdiff_driver { int textconv_want_cache; const char *process; char *process_owned; + struct diff_subprocess *diff_subprocess; + unsigned diff_process_failed : 1; }; enum userdiff_driver_type { USERDIFF_DRIVER_TYPE_BUILTIN = 1<<0, From 20b176f732ab55d8f9f4354398eafe25d95c88a8 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Thu, 28 May 2026 15:09:25 -0700 Subject: [PATCH 17/21] diff: bypass diff process with --no-ext-diff and in format-patch Make --no-ext-diff disable diff..process in addition to diff..command. Although the two mechanisms work differently (command replaces Git's output, process feeds hunks back into the pipeline), both invoke external tools and --no-ext-diff means "no external tools." Replace the OPT_BOOL for --ext-diff with an OPT_CALLBACK that sets both allow_external and no_diff_process, so a single option controls both. Passing --ext-diff explicitly clears no_diff_process, so a later --ext-diff overrides an earlier --no-ext-diff. Disable the diff process unconditionally in format-patch so that generated patches are always based on the builtin diff algorithm and can be applied reliably by recipients who do not have the external tool. Document that --diff-algorithm also bypasses the diff process, since it forces the builtin algorithm. Signed-off-by: Michael Montalbo --- Documentation/diff-algorithm-option.adoc | 3 +++ Documentation/diff-options.adoc | 4 +++- Documentation/gitattributes.adoc | 6 +++--- builtin/log.c | 7 +++++++ diff.c | 16 ++++++++++++++-- diff.h | 5 ++++- t/t4080-diff-process.sh | 16 ++++++++++++++++ 7 files changed, 50 insertions(+), 7 deletions(-) diff --git a/Documentation/diff-algorithm-option.adoc b/Documentation/diff-algorithm-option.adoc index 8e3a0b63d784d8..4d7e2ec35f97ea 100644 --- a/Documentation/diff-algorithm-option.adoc +++ b/Documentation/diff-algorithm-option.adoc @@ -18,3 +18,6 @@ For instance, if you configured the `diff.algorithm` variable to a non-default value and want to use the default one, then you have to use `--diff-algorithm=default` option. ++ +If you explicitly choose a diff algorithm, it also bypasses +`diff..process` (see linkgit:gitattributes[5]). diff --git a/Documentation/diff-options.adoc b/Documentation/diff-options.adoc index 8a63b5e164114a..18b8b0ed242838 100644 --- a/Documentation/diff-options.adoc +++ b/Documentation/diff-options.adoc @@ -825,7 +825,9 @@ endif::git-format-patch[] to use this option with linkgit:git-log[1] and friends. `--no-ext-diff`:: - Disallow external diff drivers. + Disallow external diff helpers, including + `diff..command` and `diff..process` + (see linkgit:gitattributes[5]). `--textconv`:: `--no-textconv`:: diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index f4ca4a8c7e2296..a03fb9deb108d5 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -1073,9 +1073,9 @@ display, which is covered above); and combined diffs (`--cc` and merge diffs), whose protocol would have to be extended from a single old/new pair to one comparison per merge parent. -`--diff-algorithm` bypasses the process entirely, for every feature -listed above. The whitespace-ignoring options (`-w`, -`--ignore-space-change`, `--ignore-blank-lines`, and the like), +`--no-ext-diff` and `--diff-algorithm` bypass the process entirely, +for every feature listed above. The whitespace-ignoring options +(`-w`, `--ignore-space-change`, `--ignore-blank-lines`, and the like), `-I`, and `--anchored` also bypass it for the affected files: the tool is never told about these options, so it could not honor them, and Git falls back to the builtin diff, which does. diff --git a/builtin/log.c b/builtin/log.c index 05aebb87b5177f..c3d9cf4e88d29a 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -2224,6 +2224,13 @@ int cmd_format_patch(int argc, if (argc > 1) die(_("unrecognized argument: %s"), argv[1]); + /* + * Disable diff..process so that patches generated by + * format-patch are always based on the builtin diff algorithm + * and can be applied reliably. + */ + rev.diffopt.flags.no_diff_process = 1; + if (rev.diffopt.output_format & DIFF_FORMAT_NAME) die(_("--name-only does not make sense")); if (rev.diffopt.output_format & DIFF_FORMAT_NAME_STATUS) diff --git a/diff.c b/diff.c index d36f803a1769ff..b1c30965202b22 100644 --- a/diff.c +++ b/diff.c @@ -6302,6 +6302,17 @@ static int diff_opt_submodule(const struct option *opt, return 0; } +static int diff_opt_ext_diff(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_external = !unset; + options->flags.no_diff_process = unset; + return 0; +} + static int diff_opt_textconv(const struct option *opt, const char *arg, int unset) { @@ -6632,8 +6643,9 @@ struct option *add_diff_options(const struct option *opts, N_("exit with 1 if there were differences, 0 otherwise")), OPT_BOOL(0, "quiet", &options->flags.quick, N_("disable all output of the program")), - OPT_BOOL(0, "ext-diff", &options->flags.allow_external, - N_("allow an external diff helper to be executed")), + OPT_CALLBACK_F(0, "ext-diff", options, NULL, + N_("allow an external diff helper to be executed"), + PARSE_OPT_NOARG, diff_opt_ext_diff), OPT_CALLBACK_F(0, "textconv", options, NULL, N_("run external text conversion filters when comparing binary files"), PARSE_OPT_NOARG, diff_opt_textconv), diff --git a/diff.h b/diff.h index 83cc60e9ed2eac..7bc3184d77fbb3 100644 --- a/diff.h +++ b/diff.h @@ -173,7 +173,10 @@ struct diff_flags { */ unsigned allow_external; - /** Disables diff..process. */ + /** + * Disables diff..process. Set by --no-ext-diff and by + * format-patch. + */ unsigned no_diff_process; /** diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh index 3b75df082e8def..7e71b70ab95467 100755 --- a/t/t4080-diff-process.sh +++ b/t/t4080-diff-process.sh @@ -398,6 +398,22 @@ test_expect_success 'diff process bypassed by --diff-algorithm' ' test_path_is_missing backend.log ' +test_expect_success 'diff process bypassed by --no-ext-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --no-ext-diff worddiff.c >actual && + test_grep "return 999" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not used by format-patch' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + format-patch -1 --stdout -- logtest.c >actual && + test_grep "return 2" actual && + test_path_is_missing backend.log +' + test_expect_success 'diff process bypassed under whitespace-ignoring flags' ' test_when_finished "rm -f backend.log" && printf "a\nb\nc\n" >wsbypass.c && From 640023fc8153991622ae0861b1543ef6839ddb2c Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 19:13:50 -0700 Subject: [PATCH 18/21] blame: consult diff process for no-hunk detection When a diff process is configured via diff..process, consult it during blame's per-commit diffing. If the process returns no hunks for a commit's changes to a file, treat the commit as having no changes, causing blame to attribute lines to earlier commits. The consult rides the hunk provider seam blame already diffs through, rather than a spine of its own. diff_provider_emit_hunks() picks the producer before content is loaded, via diff_process_driver(): the entry gates of diff_process_fill_hunks() extracted into a content-free predicate. A path whose driver has a process makes the tool the producer, so the store, which holds xdiff's answer that a semantic tool may deliberately contradict, is not consulted; the seam then loads content, asks the tool, and feeds its hunks to xdiff's emission, or emits no hunks at all for a pair the tool reports equivalent. A new test pins the producer rule: a store warmed with builtin hunks does not override the tool's answer in blame. Blame's -w option is not communicated to the process and it could not honor it, so blame must fall back to the builtin diff there. Because blame keeps its whitespace flags in sb->xdl_opts rather than diffopt, the predicate keys off xpp (the flags the diff actually runs with), which covers blame without a guard of its own; such a request selects no tool, so the store may still serve it. The driver is looked up by the parent (old) path, as builtin_diff() does with name_a, so a renamed file resolves to the same driver across diff, blame, and line-log. The seam forwards the pair's blob object ids to the tool under the same gate that keys the store, so old-oid and new-oid always name the bytes the tool receives; a tool that persists a cache across invocations can key on them. The subprocess is long-running (one startup cost amortized across the blame traversal), but each commit in the file's history incurs a round-trip to the tool. Signed-off-by: Michael Montalbo --- blame.c | 8 +++ diff-process.c | 42 ++++++++---- diff-process.h | 11 +++ diff-provider.c | 38 +++++++++-- diff-provider.h | 39 ++++++++--- t/t4080-diff-process.sh | 147 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 254 insertions(+), 31 deletions(-) diff --git a/blame.c b/blame.c index e7cfc6a11a158a..5fcafae0b101e8 100644 --- a/blame.c +++ b/blame.c @@ -2006,9 +2006,17 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, !blame_textconv_active(sb, target->path) && !blame_textconv_active(sb, parent->path); + /* + * Look up the driver by the parent (old) path, as builtin_diff() + * does with name_a, so a renamed file resolves to the same driver + * across diff, blame, and line-log. A tool that reports a pair + * equivalent emits no hunks, so blame passes the whole commit + * through and looks past it. + */ served = diff_provider_emit_hunks(sb->repo, provider_usable ? &parent->blob_oid : NULL, provider_usable ? &target->blob_oid : NULL, + parent->path, &sb->revs->diffopt, &xpp, blame_diff_fill, &fill, blame_chunk_cb, &d); if (served < 0) diff --git a/diff-process.c b/diff-process.c index 541bd283202303..b812ed37127f43 100644 --- a/diff-process.c +++ b/diff-process.c @@ -420,24 +420,16 @@ static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, return 0; } -enum diff_process_result diff_process_fill_hunks( - struct diff_options *diffopt, - const char *path, - const mmfile_t *file_a, - const mmfile_t *file_b, - const struct object_id *oid_a, - const struct object_id *oid_b, - xpparam_t *xpp) +struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp) { struct userdiff_driver *drv; - struct xdl_hunk *ext_hunks = NULL; - size_t nr = 0; - enum diff_process_result res; if (!diffopt || !path) - return DIFF_PROCESS_SKIP; + return NULL; if (diffopt->flags.no_diff_process || diffopt->ignore_driver_algorithm) - return DIFF_PROCESS_SKIP; + return NULL; /* * Whitespace-ignoring, regex-ignore (-I) and anchored options * change which lines count as different, but the tool is never @@ -450,12 +442,32 @@ enum diff_process_result diff_process_fill_hunks( */ if ((xpp->flags & (XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES)) || xpp->ignore_regex_nr || xpp->anchors_nr) - return DIFF_PROCESS_SKIP; + return NULL; drv = userdiff_find_by_path(diffopt->repo->index, path); if (!drv || !drv->process) - return DIFF_PROCESS_SKIP; + return NULL; if (drv->diff_process_failed) + return NULL; + return drv; +} + +enum diff_process_result diff_process_fill_hunks( + struct diff_options *diffopt, + const char *path, + const mmfile_t *file_a, + const mmfile_t *file_b, + const struct object_id *oid_a, + const struct object_id *oid_b, + xpparam_t *xpp) +{ + struct userdiff_driver *drv; + struct xdl_hunk *ext_hunks = NULL; + size_t nr = 0; + enum diff_process_result res; + + drv = diff_process_driver(diffopt, path, xpp); + if (!drv) return DIFF_PROCESS_SKIP; res = get_hunks(drv, path, diff --git a/diff-process.h b/diff-process.h index 8d00dafe1d9eac..497031f188b5f8 100644 --- a/diff-process.h +++ b/diff-process.h @@ -5,6 +5,17 @@ struct diff_options; struct object_id; +struct userdiff_driver; + +/* + * The driver whose process a consultation for path would ask, or NULL + * when none applies (no driver, process disabled or failed, or xpp + * carries options the tool is never told about). Needs no content, so + * a caller can pick a producer before loading the blobs. + */ +struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp); enum diff_process_result { DIFF_PROCESS_ERROR = -1, /* failed; caller falls back to builtin */ diff --git a/diff-provider.c b/diff-provider.c index 6e397f1895b98c..4304d50fb05cd7 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -1,6 +1,7 @@ #include "git-compat-util.h" #include "diff-provider.h" #include "diff-hunks.h" +#include "diff-process.h" int diff_provider_active(struct repository *r) { @@ -46,28 +47,51 @@ diff_provider_hunks_check(struct diff_provider_hunks_check *c, int diff_provider_emit_hunks(struct repository *r, const struct object_id *old_oid, const struct object_id *new_oid, + const char *path, + struct diff_options *diffopt, const xpparam_t *xpp, hunk_pair_fill_fn fill, void *fill_data, xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) { + xpparam_t xpp_local = *xpp; xdemitconf_t xecfg = { .hunk_func = hunk_cb }; xdemitcb_t ecb = { .priv = cb_data }; mmfile_t old_file, new_file; + int ret; + + /* Only a tool consulted here may supply external hunks. */ + xpp_local.external_hunks = NULL; + xpp_local.external_hunks_nr = 0; /* - * -I patterns and anchors shape the diff but are outside the - * settings that key a provider's answer, so such a request is - * computed, never served. + * A process-capable driver makes the tool the producer for the + * path, so the store (which holds xdiff's answer, one a semantic + * tool may deliberately contradict) is not consulted. -I + * patterns and anchors shape the diff but are outside the + * settings that key the store, so such a request is computed, + * never served. */ - if (!xpp->ignore_regex_nr && !xpp->anchors_nr && + if (!diff_process_driver(diffopt, path, xpp) && + !xpp->ignore_regex_nr && !xpp->anchors_nr && diff_provider_query_hunks(r, old_oid, new_oid, xpp->flags, hunk_cb, cb_data)) return 1; if (fill(fill_data, &old_file, &new_file) < 0) return -1; - if (xdi_diff(&old_file, &new_file, xpp, &xecfg, &ecb) < 0) - return -1; - return 0; + + switch (diff_process_fill_hunks(diffopt, path, &old_file, &new_file, + old_oid, new_oid, &xpp_local)) { + case DIFF_PROCESS_EQUIVALENT: + /* The tool reports the pair equal: there is no hunk to emit. */ + return 0; + default: + break; /* OK: tool hunks now in xpp_local; SKIP/ERROR: builtin */ + } + + ret = xdi_diff(&old_file, &new_file, &xpp_local, &xecfg, &ecb) < 0 ? + -1 : 0; + free(xpp_local.external_hunks); + return ret; } diff --git a/diff-provider.h b/diff-provider.h index 7376a9c45272fa..4ea8855df48210 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -11,10 +11,20 @@ * A hunk provider answers a consumer's request from the pair's blob * object ids and the settings that determine the diff, before any * content is loaded; a request no provider answers falls through to - * the consumer's own computation. The diff-hunks store (diff-hunks.h) - * is the provider consulted today. + * the consumer's own computation. Two providers implement this + * interface with different authority. The diff-hunks store + * (diff-hunks.h) is in-process and not authoritative: it may only + * reproduce the builtin result, so it never asserts a pair + * equivalent, and it stands aside wherever a tool outranks it. A + * configured diff..process tool (diff-process.h) is + * authoritative for its paths: its answer may deliberately differ + * from the builtin diff, including asserting a pair equivalent. The + * consult order in diff_provider_emit_hunks() is that authority + * resolution. Whichever provider answers, its coordinates pass + * diff_provider_hunks_check() before any consumer sees them. */ +struct diff_options; struct object_id; struct repository; @@ -80,17 +90,28 @@ typedef int (*hunk_pair_fill_fn)(void *data, mmfile_t *old_file, /* * Emit the exact changed ranges (context 0) for the pair (old_oid, - * new_oid) to hunk_cb. Providers are consulted first, keyed by the - * object ids and xpp's flags; pass NULL object ids when the diffed - * bytes are not those blobs (or there are no blobs), which makes - * every consult miss. On a miss, fill supplies the content and - * xdiff computes the ranges from xpp. Returns 1 when a provider - * supplied the ranges, 0 when they were computed, and -1 on failure - * to load or diff. + * new_oid) at path to hunk_cb. + * + * The producer is picked before content is loaded. A path whose + * driver has a diff process makes the tool the producer: the store's + * entries hold xdiff's answer, which a semantic tool may deliberately + * contradict, so the identity phase is skipped, content is loaded, + * and the tool is consulted; its hunks feed xdiff's emission, and a + * tool that reports the pair equivalent emits no hunks at all. + * Otherwise the store is consulted by the object ids and xpp's flags, + * and only a miss loads content and computes. + * + * Pass NULL object ids when the diffed bytes are not those blobs (or + * there are no blobs): the identity phase then misses, and no id is + * sent to a tool. Pass a NULL diffopt or path to consult no tool. + * Returns 1 when the store supplied the ranges, 0 when they were + * emitted any other way, and -1 on failure to load or diff. */ int diff_provider_emit_hunks(struct repository *r, const struct object_id *old_oid, const struct object_id *new_oid, + const char *path, + struct diff_options *diffopt, const xpparam_t *xpp, hunk_pair_fill_fn fill, void *fill_data, xdl_emit_hunk_consume_func_t hunk_cb, diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh index 7e71b70ab95467..9806728cb074b7 100755 --- a/t/t4080-diff-process.sh +++ b/t/t4080-diff-process.sh @@ -658,4 +658,151 @@ test_expect_success 'diff process omits old-oid and new-oid for textconv content test_must_be_empty stderr ' +# +# Blame integration. +# + +test_expect_success 'blame uses tool-provided hunks' ' + cat >blame-hunk.c <<-\EOF && + line1 + line2 + line3 + line4 + original5 + original6 + line7 + line8 + line9 + line10 + EOF + git add blame-hunk.c && + git commit -m "add blame-hunk.c" && + ORIG=$(git rev-parse --short HEAD) && + + cat >blame-hunk.c <<-\EOF && + line1 + line2 + line3 + line4 + changed5 + changed6 + line7 + line8 + changed9 + changed10 + EOF + git add blame-hunk.c && + git commit -m "change blame-hunk.c" && + CHANGE=$(git rev-parse --short HEAD) && + + # With fixed-hunk mode the tool reports only lines 5-6 as changed, + # so blame should attribute lines 9-10 to the original commit + # even though the builtin diff would show them as changed. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + sed -n "5p" actual >line5 && + sed -n "6p" actual >line6 && + test_grep "$CHANGE" line5 && + test_grep "$CHANGE" line6 +' + +test_expect_success 'a warmed hunk store does not override tool hunks in blame' ' + ORIG=$(git rev-parse --short HEAD~1) && + GIT_DIFF_HUNKS_WRITE=1 git log -2 --stat -- blame-hunk.c >/dev/null && + + # Control: without a process, blame is served from the store. + git blame --show-stats blame-hunk.c >stats && + test_grep "num precomputed hits: 1" stats && + + # The store holds the builtin hunks (lines 5-6 and 9-10 changed), + # but a process-capable driver makes the tool the producer, so + # blame must reflect the tool hunks (only lines 5-6), not a store + # hit: lines 9-10 stay attributed to the original commit. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + git diff-hunks clear +' + +test_expect_success 'blame skips commits with no hunks from diff process' ' + cat >blame.c <<-\EOF && + int main(void) { + return 0; + } + EOF + git add blame.c && + git commit -m "add blame.c" && + ORIG_COMMIT=$(git rev-parse --short HEAD) && + + cat >blame.c <<-\EOF && + int main(void) + { + return 0; + } + EOF + git add blame.c && + git commit -m "reformat blame.c" && + BLAME_COMMIT=$(git rev-parse --short HEAD) && + + # Without no-hunks mode, blame attributes the change. + git blame blame.c >without && + test_grep "$BLAME_COMMIT" without && + + # With no-hunks mode, the process considers the files equivalent + # and blame skips the reformat commit, attributing to the original. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + blame blame.c >with && + test_grep ! "$BLAME_COMMIT" with && + test_grep "$ORIG_COMMIT" with +' + +test_expect_success 'blame --no-ext-diff bypasses diff process' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + blame --no-ext-diff blame.c >actual && + # Without the process, blame attributes the reformat commit normally. + test_grep "$BLAME_COMMIT" actual && + test_path_is_missing backend.log +' + +test_expect_success 'blame --no-ext-diff uses builtin hunks' ' + # fixed-hunk mode would narrow blame to lines 5-6, but + # --no-ext-diff should bypass it and use the builtin diff. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + blame --no-ext-diff blame-hunk.c >actual && + # Builtin diff attributes lines 9-10 to the change commit. + sed -n "9p" actual >line9 && + test_grep "$CHANGE" line9 && + test_path_is_missing backend.log +' + +test_expect_success 'blame -w bypasses diff process' ' + test_when_finished "rm -f backend.log" && + printf "alpha\nbeta\ngamma\n" >blamew.c && + git add blamew.c && + git commit -m "add blamew.c" && + orig=$(git rev-parse --short HEAD) && + printf "alpha\n beta \ngamma\n" >blamew.c && + git commit -am "reindent beta" && + reindent=$(git rev-parse --short HEAD) && + # blame -w must ignore the whitespace-only change and attribute + # beta to the original commit, not the reindent commit. The tool + # is never told about -w, so blame must bypass it (not let tool + # hunks override -w). + git -c diff.cdiff.process="$BACKEND --mode=whole-file --log=backend.log" \ + blame -w blamew.c >actual && + sed -n "2p" actual >line2 && + test_grep "$orig" line2 && + test_grep ! "$reindent" line2 && + test_path_is_missing backend.log +' + test_done From be9aa4aaf9b49262d5576314dc54459667f266c6 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 19:17:39 -0700 Subject: [PATCH 19/21] diff: consult diff process for --stat counts builtin_diff() already consults a configured diff..process: a file the tool reports as equivalent emits no patch, and otherwise the tool's hunks drive the output. builtin_diffstat() ran its own xdiff and ignored the process, so "git diff --stat" still counted a byte-level change for a file that "git diff" showed as unchanged. Consult diff_process_fill_hunks() before the stat xdiff, as builtin_diff() does. On DIFF_PROCESS_EQUIVALENT, skip the xdiff so the file keeps its zero inserted and deleted counts and the existing "nothing changed" pruning drops it, matching the empty patch. Otherwise the tool's hunks, or the builtin fallback, feed the counts through the shared xpparam_t. A process-capable driver also makes the tool the producer over the diff-hunks store: diffstat_from_hunks() steps aside for such a path, so neither a store read (it holds xdiff's answer) nor its own xdiff-and-record can stand in for the tool. Without this, a warming run with a tool configured would count and record builtin hunks the tool never saw. A new test pins the rule: a store warmed with builtin counts does not override the tool's counts. Under -L, route the surviving hunks through the same line-range filter builtin_diffstat() already uses for a tracked range, so a process-provided diff is scoped to that range: "git log -L --stat" counts the tool's changed lines within the range rather than the builtin line diff's. Like the builtin summary path, builtin_diffstat() does not apply textconv, so the process is consulted on the raw blob content here, unlike builtin_diff() which sends textconv'd content. This keeps "git diff --stat" counting raw lines as it does today; the asymmetry between patch output and summary counts under textconv predates this change. Because the content is the raw blob, the stat path sends the blob object names to the tool (old-oid/new-oid) for any stored blob, where the patch path omits the oid under textconv. Move the summary formats out of the "not yet wired" group of the "Which features consult the diff process" documentation and into the list of features that use the tool's hunks, noting the raw, non-textconv content they receive. Document that the line-counting --dirstat=lines follows these counts while the default --dirstat does not, and that summary formats and blame (only under --textconv) differ from patch output in whether they textconv the content the tool sees. Add tests covering counts from the tool's hunks (--numstat, --shortstat), an equivalent file producing no stat line, --stat --exit-code, the raw non-textconv content the tool receives, a multi-file mix of equivalent and changed files, a mode-only change, a range-scoped --stat under "git log -L" that reflects the tool's hunks, and the warmed-store interplay above. Signed-off-by: Michael Montalbo --- Documentation/gitattributes.adoc | 33 ++++-- diff.c | 75 +++++++++--- t/t4080-diff-process.sh | 192 +++++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 27 deletions(-) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index a03fb9deb108d5..7cdede6b218f89 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -874,7 +874,10 @@ a flush packet, followed by the old and new file content as packetized data, each terminated with a flush packet. The pathname is relative to the repository root. When `diff..textconv` is also set, the tool receives the textconv-transformed content rather than the -raw blob. Git does not send binary files to the diff process. +raw blob, matching what the consuming feature itself diffs: patch +output is textconv'd, the summary formats (noted below) are not, and +`git blame` applies textconv only under `--textconv`. Git does not +send binary files to the diff process. ----------------------- packet: git> command=hunks @@ -960,8 +963,8 @@ still slide or regroup those changes against matching context for display, exactly as it compacts its own diffs, so the tool controls which lines are reported as changed, not the precise hunk boundaries. Patch output features (word diff, function context, color) work -normally. Summary formats such as `--stat` still compute their counts -with the builtin diff for now; see "Which features consult the diff +normally, as do summary formats like `--stat`. Not every feature +consults the process, though; see "Which features consult the diff process" below for the full picture and the reasoning behind it. If no hunk lines precede the flush, followed by "success", Git @@ -1040,6 +1043,17 @@ of the builtin algorithm: hunks without any further negotiation. - `git blame`: a commit whose change the tool reports as equivalent is skipped, and its lines are attributed to an earlier commit. +- `--stat`, `--numstat`, and `--shortstat`: the inserted and deleted + counts come from the tool's hunks, so a file the tool calls + equivalent contributes no stat line, matching the empty patch that + `git diff` produces for it. These summary formats do not apply + textconv (just as the builtin summary path does not), so the tool + is consulted on the raw blob content even when a `textconv` is also + configured for patch output; this mirrors how builtin `--stat` + already counts raw lines rather than the textconv'd view. The + line-counting `--dirstat=lines` uses these same counts; the default + `--dirstat`, which weighs byte changes, is computed on its own and + does not consult the tool. Features that ask a different question do not consult the process, by design: @@ -1065,13 +1079,12 @@ design: - `--raw`, `--name-only`, and `--name-status` compare object ids at the tree level and never run a line-level diff at all. -Some features ask "which lines changed" but still use the builtin -algorithm for now, and may consult the process in a later change: the -summary formats (`--stat`, `--numstat`, `--shortstat`); `git log -L`'s -commit selection and parent range propagation (as distinct from its -display, which is covered above); and combined diffs (`--cc` and merge -diffs), whose protocol would have to be extended from a single old/new -pair to one comparison per merge parent. +Two cases ask "which lines changed" but still use the builtin +algorithm, and may consult the process in a later change: `git log +-L`'s commit selection and parent range propagation (as distinct from +its display, which is covered above), and combined diffs (`--cc` and +merge diffs), whose protocol would have to be extended from a single +old/new pair to one comparison per merge parent. `--no-ext-diff` and `--diff-algorithm` bypass the process entirely, for every feature listed above. The whitespace-ignoring options diff --git a/diff.c b/diff.c index b1c30965202b22..1326c855349053 100644 --- a/diff.c +++ b/diff.c @@ -4377,6 +4377,7 @@ static int diffstat_sum_hunk_cb(long start_a UNUSED, long count_a, * function body checks that at compile time. */ static int diffstat_from_hunks(struct diff_options *o, + const char *name_a, struct diff_filespec *one, struct diff_filespec *two, struct diffstat_file *data) @@ -4386,6 +4387,19 @@ static int diffstat_from_hunks(struct diff_options *o, int stable; mmfile_t mf1, mf2; xpparam_t xpp; + xpparam_t probe = { .flags = o->xdl_opts, + .ignore_regex_nr = o->ignore_regex_nr, + .anchors_nr = o->anchors_nr }; + + /* + * A process-capable driver makes the tool the producer for the + * path: the stat must reflect the tool's hunks, so neither a + * store read (it holds xdiff's answer) nor this function's own + * xdiff-and-record may stand in. Step aside and let the caller + * consult the tool. + */ + if (diff_process_driver(o, name_a, &probe)) + return 0; /* * xpparam_t is the diff algorithm's input. Its flags are the key's @@ -4549,7 +4563,8 @@ static void builtin_diffstat(const char *name_a, const char *name_b, * keys, so it neither reads nor records. Otherwise diff * normally. */ - if (p->line_ranges || !diffstat_from_hunks(o, one, two, data)) { + if (p->line_ranges || + !diffstat_from_hunks(o, name_a, one, two, data)) { /* Crazy xdl interfaces.. */ xpparam_t xpp; xdemitconf_t xecfg; @@ -4568,24 +4583,50 @@ static void builtin_diffstat(const char *name_a, const char *name_b, xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - - if (p->line_ranges) { - struct line_range_filter lr_filter; - - line_range_filter_init(&lr_filter, - p->line_ranges, - diffstat_consume, - diffstat); - - if (line_range_filter_diff(&lr_filter, &mf1, - &mf2, &xpp, &xecfg)) + /* + * Consult the diff process so --stat reflects the + * tool's view of which lines changed rather than the + * builtin line diff. --stat never applies textconv, + * so the tool is fed the same raw mmfiles the stat + * itself diffs (unlike builtin_diff, which consults + * the process on textconv'd content). + * When the tool reports the files as equivalent we + * skip xdiff entirely, leaving added and deleted at + * zero so the file is pruned below, just as + * builtin_diff() emits no patch for an equivalent + * file. + * + * Under -L, feed the tool's hunks through the same + * line-range filter the builtin stat uses, so a + * process-provided diff is scoped to the tracked + * range. + */ + if (diff_process_fill_hunks(o, name_a, &mf1, &mf2, + one->oid_valid ? &one->oid : NULL, + two->oid_valid ? &two->oid : NULL, + &xpp) + != DIFF_PROCESS_EQUIVALENT) { + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, + p->line_ranges, + diffstat_consume, + diffstat); + + if (line_range_filter_diff(&lr_filter, + &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, + diffstat_consume, + diffstat, + &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); - } else if (xdi_diff_outf(&mf1, &mf2, NULL, - diffstat_consume, diffstat, - &xpp, &xecfg)) - die("unable to generate diffstat for %s", - one->path); + } + free(xpp.external_hunks); } if (DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two)) { diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh index 9806728cb074b7..74e7d084b29e8e 100755 --- a/t/t4080-diff-process.sh +++ b/t/t4080-diff-process.sh @@ -282,6 +282,21 @@ test_expect_success 'diff process works alongside textconv' ' test_must_be_empty stderr ' +test_expect_success 'diff process --stat is fed raw, not textconv, content' ' + # Reuses textconv.c from the previous test (committed "hello + # world", modified to "goodbye world"). Unlike patch output, + # --stat does not apply textconv, so the tool sees raw lowercase + # content here even with a textconv configured. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.textconv="./uppercase-filter" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --stat -- textconv.c >actual 2>stderr && + test_grep "pathname=textconv.c" backend.log && + test_grep "old=hello world" backend.log && + test_grep "new=goodbye world" backend.log && + test_must_be_empty stderr +' + # # Downstream features: word diff, log, equivalent files, exit code. # @@ -386,6 +401,167 @@ test_expect_success 'diff process with --exit-code and hunks returns failure' ' diff --exit-code newfile.c ' +test_expect_success 'diff process feeds --numstat counts' ' + # fixed-hunk reports only lines 5-6 as changed, so the stat + # counts come from the tool (2/2), not the builtin diff (4/4). + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + diff --numstat boundary.c >actual 2>stderr && + printf "2\t2\tboundary.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks pathname=boundary.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process --numstat sums multi-hunk counts' ' + # multi-hunk reports both 2-line regions (5-6 and 9-10), so the + # counts add up across both hunks: 4 inserted, 4 deleted. This + # exercises the two-region hunk path through builtin_diffstat. + git -c diff.cdiff.process="$BACKEND --mode=multi-hunk" \ + diff --numstat boundary.c >actual && + printf "4\t4\tboundary.c\n" >expect && + test_cmp expect actual +' + +test_expect_success 'diff process equivalent files produce no --stat line' ' + # A file the tool calls equivalent contributes no stat line, + # matching the empty patch that git diff produces for it. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --stat worddiff.c >actual 2>stderr && + test_must_be_empty actual && + test_grep "command=hunks pathname=worddiff.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process feeds --shortstat counts' ' + # fixed-hunk reports lines 5-6 only, so the summary counts come + # from the tool (2 insertions, 2 deletions), not builtin (4/4). + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + diff --shortstat boundary.c >actual && + test_grep "2 insertions" actual && + test_grep "2 deletions" actual +' + +test_expect_success 'diff process scopes --stat to the tracked range under log -L' ' + test_when_finished "rm -f backend.log" && + cat >rangestat.c <<-\EOF && + line1 + line2 + line3 + line4 + OLD5 + OLD6 + line7 + line8 + OLD9 + OLD10 + EOF + git add rangestat.c && + git commit -m "add rangestat.c" && + + cat >rangestat.c <<-\EOF && + line1 + line2 + line3 + line4 + NEW5 + NEW6 + line7 + line8 + NEW9 + NEW10 + EOF + git add rangestat.c && + git commit -m "change rangestat.c" && + + # The file changes at lines 5-6 and 9-10, but fixed-hunk reports + # only 5-6. The builtin line diff counts both regions (4/4); the + # tool hunks flow through the same line-range filter the stat uses, + # so the range-scoped stat reflects the tool view instead (2/2). + git log --no-ext-diff -L1,10:rangestat.c --oneline --stat >builtin && + test_grep "4 insertions(+), 4 deletions(-)" builtin && + + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + log -L1,10:rangestat.c --oneline --stat >actual && + test_grep "2 insertions(+), 2 deletions(-)" actual && + test_grep ! "4 insertions" actual && + test_grep "command=hunks pathname=rangestat.c" backend.log +' + +test_expect_success 'diff process equivalent file makes --stat --exit-code succeed' ' + # The tool reports worddiff.c equivalent, so --exit-code reports + # no change (0); the builtin diff would report a change (1). + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --stat --exit-code worddiff.c && + test_expect_code 1 git diff --no-ext-diff --stat --exit-code worddiff.c +' + +test_expect_success 'diff process --numstat with mixed equivalent and changed files' ' + test_when_finished "rm -f c.log h.log" && + # Self-contained fixtures: *.c uses whole-file (changed); *.mh + # uses no-hunks (equivalent). + echo "*.mh diff=hdiff" >>.gitattributes && + git add .gitattributes && + printf "int a(void) { return 1; }\n" >mixed.c && + printf "int b(void) { return 1; }\n" >mixed.mh && + git add mixed.c mixed.mh && + git commit -m "add mixed fixtures" && + printf "int a(void) { return 2; }\n" >mixed.c && + printf "int b(void) { return 2; }\n" >mixed.mh && + git -c diff.cdiff.process="$BACKEND --mode=whole-file --log=c.log" \ + -c diff.hdiff.process="$BACKEND --mode=no-hunks --log=h.log" \ + diff --numstat mixed.c mixed.mh >actual 2>stderr && + test_grep "mixed.c" actual && + test_grep ! "mixed.mh" actual && + test_grep "pathname=mixed.c" c.log && + test_grep "pathname=mixed.mh" h.log && + test_must_be_empty stderr +' + +test_expect_success POSIXPERM 'diff process keeps mode-only change in --stat' ' + test_when_finished "rm -f backend.log" && + cat >modeonly.c <<-\EOF && + int m(void) { return 1; } + EOF + git add modeonly.c && + git commit -m "add modeonly.c" && + cat >modeonly.c <<-\EOF && + int m(void) { return 2; } + EOF + git add modeonly.c && + test_chmod +x modeonly.c && + git commit -m "edit and chmod modeonly.c" && + # Content and mode both changed, but no-hunks reports the content + # equivalent. The tool is consulted (counts are zero, not the + # builtin 1/1), yet the mode change keeps the file from being + # pruned. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --stat HEAD^ HEAD >actual 2>stderr && + test_grep "modeonly.c" actual && + test_grep "command=hunks pathname=modeonly.c" backend.log && + test_grep ! "1 insertion" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process not consulted for default --dirstat' ' + # The default (change-based) --dirstat algorithm counts via its + # own path and never contacts the tool (here --dirstat=0 just + # sets a 0% threshold), so the change is still reported even + # though no-hunks would call it equivalent. --dirstat=lines + # instead uses the process-aware stat path. + test_when_finished "rm -f backend.log" && + mkdir -p dsub && + printf "a\nb\nc\n" >dsub/d.c && + git add dsub/d.c && + git commit -m "add dsub/d.c" && + printf "a\nB\nc\n" >dsub/d.c && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --dirstat=0 dsub/d.c >actual && + test_grep "dsub" actual && + test_path_is_missing backend.log +' + # # Bypass mechanisms: flags and commands that skip the diff process. # @@ -731,6 +907,22 @@ test_expect_success 'a warmed hunk store does not override tool hunks in blame' git diff-hunks clear ' +test_expect_success 'a warmed hunk store does not override tool hunks in --stat' ' + test_when_finished "git diff-hunks clear && rm -f trace.json" && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --stat -- blame-hunk.c >/dev/null && + + # Control: without a process, the counts are served from the store. + GIT_TRACE2_EVENT="$PWD/trace.json" git log -1 --numstat -- blame-hunk.c >/dev/null && + test_grep read-hits trace.json && + + # The tool reports only lines 5-6 as changed, so the counts must + # be the tool hunks (2/2), not the stored builtin hunks (4/4). + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual +' + test_expect_success 'blame skips commits with no hunks from diff process' ' cat >blame.c <<-\EOF && int main(void) { From 4fe9aef8786b707a8971807bc7d79495264291d3 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 19:19:24 -0700 Subject: [PATCH 20/21] line-log: consult diff process for range tracking git log -L tracks line ranges by diffing each commit against its parent in collect_diff(). This pass used the builtin diff while the displayed diff (builtin_diff()) consults a configured diff..process, so the two could disagree: a reformat-only commit selected by builtin tracking was then rendered with an empty diff because the tool reported the files equivalent. Route collect_diff() through the hunk provider seam, as blame is. The caller already holds the content, so its fill callback hands the loaded mmfiles to the seam. When the tool reports the files equivalent, no ranges are collected; the tracked range then maps across unchanged and the commit drops out of the log, matching what is displayed. Like the summary formats, the tracking pass diffs raw content, so the tool is consulted on the raw blobs here. Blob oids are not threaded to this path yet, so the store is not consulted and no old-oid/new-oid is sent to the tool; a later change can supply the pair, where they would let both providers serve the range-tracking and display passes over the same commit. The driver is selected by the old (parent) path, as builtin_diff() does with name_a, so a renamed file resolves to the same driver for range tracking as for the diff that is shown. Signed-off-by: Michael Montalbo --- Documentation/gitattributes.adoc | 20 ++++++----- line-log.c | 57 ++++++++++++++++++++++++++------ t/t4080-diff-process.sh | 33 ++++++++++++++++++ 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index 7cdede6b218f89..8021dc8e399e62 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -1037,12 +1037,16 @@ Features that ask "which lines changed" use the tool's hunks in place of the builtin algorithm: - `git diff` patch output, together with everything layered on it: - word diff, function context (`-W`), `--color-moved`, the `@@` hunk - headers, and the `-L` line-range display. These operate on the - lines the patch step already emitted, so they reflect the tool's - hunks without any further negotiation. + word diff, function context (`-W`), `--color-moved`, and the `@@` + hunk headers. These operate on the lines the patch step already + emitted, so they reflect the tool's hunks without any further + negotiation. - `git blame`: a commit whose change the tool reports as equivalent is skipped, and its lines are attributed to an earlier commit. +- `git log -L`: both the line-range display and the underlying range + tracking consult the tool, so a commit it reports as equivalent is + dropped from the log (its tracked range maps across unchanged) + rather than selected and then shown with an empty diff. - `--stat`, `--numstat`, and `--shortstat`: the inserted and deleted counts come from the tool's hunks, so a file the tool calls equivalent contributes no stat line, matching the empty patch that @@ -1079,11 +1083,9 @@ design: - `--raw`, `--name-only`, and `--name-status` compare object ids at the tree level and never run a line-level diff at all. -Two cases ask "which lines changed" but still use the builtin -algorithm, and may consult the process in a later change: `git log --L`'s commit selection and parent range propagation (as distinct from -its display, which is covered above), and combined diffs (`--cc` and -merge diffs), whose protocol would have to be extended from a single +Combined diffs (`--cc` and merge diffs) ask "which lines changed" but +still use the builtin algorithm, and may consult the process in a +later change; their protocol would have to be extended from a single old/new pair to one comparison per merge parent. `--no-ext-diff` and `--diff-algorithm` bypass the process entirely, diff --git a/line-log.c b/line-log.c index 5fc75ae275e03a..75aed95f6f7b3e 100644 --- a/line-log.c +++ b/line-log.c @@ -12,6 +12,7 @@ #include "repository.h" #include "revision.h" #include "xdiff-interface.h" +#include "diff-provider.h" #include "strbuf.h" #include "line-log.h" #include "setup.h" @@ -330,22 +331,50 @@ static int collect_diff_cb(long start_a, long count_a, return 0; } -static int collect_diff(mmfile_t *parent, mmfile_t *target, struct diff_ranges *out) +struct collect_diff_fill { + mmfile_t *parent, *target; +}; + +/* The caller already holds the content; hand it to the provider seam. */ +static int collect_diff_fill(void *data, mmfile_t *old_file, + mmfile_t *new_file) +{ + struct collect_diff_fill *f = data; + + *old_file = *f->parent; + *new_file = *f->target; + return 0; +} + +static int collect_diff(struct diff_options *diffopt, const char *path, + mmfile_t *parent, mmfile_t *target, + struct diff_ranges *out) { struct collect_diff_cbdata cbdata = {NULL}; + struct collect_diff_fill fill = { parent, target }; xpparam_t xpp; - xdemitconf_t xecfg; - xdemitcb_t ecb; memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xecfg.ctxlen = xecfg.interhunkctxlen = 0; - cbdata.diff = out; - xecfg.hunk_func = collect_diff_cb; - memset(&ecb, 0, sizeof(ecb)); - ecb.priv = &cbdata; - return xdi_diff(parent, target, &xpp, &xecfg, &ecb); + + /* + * Consult the diff process (via the provider seam) so range + * tracking agrees with the diff that will be shown. When the + * tool reports the files as equivalent no ranges are collected, + * so the tracked range maps across unchanged and the commit + * drops out of the log, rather than being selected here but + * rendered with an empty diff by the process-aware + * builtin_diff(). Blob oids are not threaded to this path yet, + * so pass NULL: the store is not consulted and no + * old-oid/new-oid is sent (a later change can supply the pair, + * where they would let a tool cache across the range-tracking + * and display passes over the same commit). + */ + if (diff_provider_emit_hunks(diffopt->repo, NULL, NULL, path, + diffopt, &xpp, collect_diff_fill, &fill, + collect_diff_cb, &cbdata) < 0) + return -1; + return 0; } /* @@ -927,7 +956,13 @@ static int process_diff_filepair(struct rev_info *rev, } diff_ranges_init(&diff); - if (collect_diff(&file_parent, &file_target, &diff)) + /* + * Select the driver by the old (parent) path, as builtin_diff() does + * with name_a, so a renamed file resolves to the same driver for + * range tracking as for the diff that is shown. + */ + if (collect_diff(&rev->diffopt, pair->one->path, + &file_parent, &file_target, &diff)) die("unable to generate diff for %s", pair->one->path); /* NEEDSWORK should apply some heuristics to prevent mismatches */ diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh index 74e7d084b29e8e..e01279b174518e 100755 --- a/t/t4080-diff-process.sh +++ b/t/t4080-diff-process.sh @@ -997,4 +997,37 @@ test_expect_success 'blame -w bypasses diff process' ' test_path_is_missing backend.log ' +# +# Line-log (git log -L) range tracking. +# + +test_expect_success 'diff process drops equivalent commit from log -L' ' + test_when_finished "rm -f backend.log" && + cat >linelog.c <<-\EOF && + int tracked(void) { return 1; } + EOF + git add linelog.c && + git commit -m "add linelog.c" && + + cat >linelog.c <<-\EOF && + int tracked(void) { return 2; } + EOF + git commit -am "change tracked line" && + + # Builtin line tracking selects the change commit. + git log --no-ext-diff -L1,1:linelog.c --format="%s" >builtin && + test_grep "change tracked line" builtin && + + # With the tool reporting the change as equivalent, tracking + # drops the commit (the range maps across unchanged) instead of + # selecting it and rendering an empty diff. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + log -L1,1:linelog.c --format="%s" >actual && + test_grep ! "change tracked line" actual && + # The creating commit still appears, so the change commit was + # selectively dropped rather than the whole log going empty. + test_grep "add linelog.c" actual && + test_grep "command=hunks pathname=linelog.c" backend.log +' + test_done From 7b8410a0d2f93696e7214ab24dbaaebf60040948 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Wed, 29 Jul 2026 19:34:01 -0700 Subject: [PATCH 21/21] diff: add oid-only requests via a hunks-by-oid capability A diff process receives the full content of both sides with every request, so a tool that already knows the answer for a blob pair (a tool that keys a persistent cache on the old-oid/new-oid it receives) still costs Git the blob loading and the content transfer. Measured on git.git, that content movement is the bulk of a consultation's cost: a consult with content averages roughly 87 microseconds per pair, while a bare round-trip over the same pipe costs roughly 6, so answering from object ids alone keeps most of the benefit of not computing the diff at all. Add a negotiated "hunks-by-oid" capability. When a tool announces it, a consumer that knows both blob object ids asks with a command=hunks-by-oid request carrying only the pathname and the pair; no content sections follow. The tool answers in the usual hunk-line form, or with status=need-content, upon which Git repeats the request as a full command=hunks exchange. Because Git holds no content for the exchange, the answer is used as the tool sent it: the hunks are validated for order, overlap, and lockstep alignment (the checks that need line counts do not apply) and replayed to the consumer without passing through xdiff's compaction, and a zero-hunk success asserts equivalence outright, including the trailing-newline case the content path detects itself; a tool that cannot promise that from its cache answers need-content. diff_process_query_hunks() carries the request. The provider seam consults it in the identity phase, so a blame over a tool-driven path never loads the blobs the tool can answer for, and diffstat_from_hunks() consults it where it steps aside for a tool path, so "git log --stat" gets the same treatment. Patch output always holds content and keeps the content request. The test backend grows oid-fixed and oid-need-content modes, and the tests pin the mechanics via the backend's request log: blame and --numstat answered with no content request sent, the need-content fallback issuing both requests, and a working-tree side (no stored object id) going straight to the content request. Signed-off-by: Michael Montalbo --- Documentation/gitattributes.adoc | 24 ++++ diff-process.c | 160 ++++++++++++++++++++++++++- diff-process.h | 21 ++++ diff-provider.c | 13 +++ diff.c | 21 +++- t/helper/test-diff-process-backend.c | 42 ++++++- t/t4080-diff-process.sh | 43 +++++++ 7 files changed, 314 insertions(+), 10 deletions(-) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index 8021dc8e399e62..bc231da2cca105 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -898,6 +898,30 @@ blob it names: it is omitted when the content is textconv-transformed, and for a working-tree side that has no stored object. A tool that does not recognize these keys ignores them. +A tool that also announces the `hunks-by-oid` capability may be asked +to answer from the object names alone. Such a request uses +`command=hunks-by-oid`, always carries both keys, and is not followed +by any content section: + +----------------------- +packet: git> command=hunks-by-oid +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +----------------------- + +The tool answers in the same hunk-line form as below, or with +`status=need-content`, upon which Git repeats the request as a full +`command=hunks` exchange with the content included. Git holds no +content for an oid-only exchange, so the answer is used exactly as +sent: the hunks feed the consuming feature directly, and a +`status=success` response with zero hunks asserts that the blobs are +equivalent, including their trailing newlines. A tool that cannot +promise that from its cache answers `need-content`. Blame and the +summary formats send such requests for stored blob pairs; patch +output always sends content. + The tool is expected to respond with zero or more hunk lines, a flush packet, and a status packet terminated with a flush packet. Each hunk line has the form: diff --git a/diff-process.c b/diff-process.c index b812ed37127f43..84133eb90b8df2 100644 --- a/diff-process.c +++ b/diff-process.c @@ -24,6 +24,19 @@ * * When the tool returns no hunks with status=success, it considers * the files equivalent. Git will skip the diff for that file. + * + * A tool that also negotiates the "hunks-by-oid" capability may be + * asked without content, when both sides are stored blobs: + * git> command=hunks-by-oid / pathname= / old-oid= / new-oid= / flush + * tool< hunk ... / flush + * tool< status=success / flush + * No content sections follow such a request. The tool may instead + * answer status=need-content, and Git re-asks with a full + * command=hunks request. Because Git holds no content for this + * exchange, the answer is used as sent: the hunks are not re-run + * through xdiff's compaction, and zero hunks assert equivalence + * outright (a tool that cannot rule out a trailing-newline-only + * difference from its cache must answer need-content). */ #include "git-compat-util.h" @@ -41,6 +54,7 @@ #include "xdiff/xdiff.h" #define CAP_HUNKS (1u << 0) +#define CAP_OID_HUNKS (1u << 1) struct diff_subprocess { struct subprocess_entry subprocess; @@ -52,6 +66,7 @@ static int start_diff_process_fn(struct subprocess_entry *subprocess) static int versions[] = { 1, 0 }; static struct subprocess_capability capabilities[] = { { "hunks", CAP_HUNKS }, + { "hunks-by-oid", CAP_OID_HUNKS }, { NULL, 0 } }; struct diff_subprocess *entry = @@ -371,6 +386,9 @@ static long count_lines(const char *buf, long size) * xdiff stays diagnostic-free; on a bad response we warn and the caller * falls back to the builtin diff. Returns 0 if valid, -1 (after * warning) otherwise. + * + * An oid-only answer arrives without content: pass negative line + * counts, and the two content-dependent checks do not apply. */ static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, long old_lines, long new_lines, @@ -382,8 +400,9 @@ static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, for (i = 0; i < nr; i++) { const struct xdl_hunk *h = &hunks[i]; - if (h->old_count > old_lines - h->old_start + 1 || - h->new_count > new_lines - h->new_start + 1) { + if (old_lines >= 0 && + (h->old_count > old_lines - h->old_start + 1 || + h->new_count > new_lines - h->new_start + 1)) { warning(_("diff process '%s' returned a hunk past the " "end of '%s'; using the builtin diff"), process, path); @@ -411,7 +430,8 @@ static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, return -1; } } - if (old_lines - c.prev_old_end != new_lines - c.prev_new_end) { + if (old_lines >= 0 && + old_lines - c.prev_old_end != new_lines - c.prev_new_end) { warning(_("diff process '%s' returned hunks that leave '%s' " "misaligned; using the builtin diff"), process, path); @@ -509,3 +529,137 @@ enum diff_process_result diff_process_fill_hunks( } return DIFF_PROCESS_SKIP; } + +/* + * Without content there is no size-derived bound on a response, so cap + * accumulation at a constant instead. A pair with more changed lines + * than this is served by the content request, whose cap follows the + * file sizes. + */ +#define OID_HUNKS_MAX (1 << 20) + +enum diff_process_result diff_process_query_hunks( + struct diff_options *diffopt, + const char *path, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + struct userdiff_driver *drv; + struct diff_subprocess *backend; + struct child_process *process; + int fd_in, fd_out; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0, i; + int len; + char *line; + enum diff_process_result res; + + if (!old_oid || !new_oid) + return DIFF_PROCESS_SKIP; + drv = diff_process_driver(diffopt, path, xpp); + if (!drv) + return DIFF_PROCESS_SKIP; + + backend = get_or_launch_process(drv); + if (!backend) + return DIFF_PROCESS_ERROR; + if ((backend->supported_capabilities & (CAP_HUNKS | CAP_OID_HUNKS)) + != (CAP_HUNKS | CAP_OID_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&backend->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + if (packet_write_fmt_gently(fd_in, "command=hunks-by-oid\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path) || + packet_write_fmt_gently(fd_in, "old-oid=%s\n", oid_to_hex(old_oid)) || + packet_write_fmt_gently(fd_in, "new-oid=%s\n", oid_to_hex(new_oid)) || + packet_flush_gently(fd_in)) + goto comm_error; + + while ((len = packet_read_line_gently(fd_out, NULL, &line)) >= 0 && + line) { + if (parse_hunk_line(line, &presented) < 0) + goto comm_error; + if (diff_process_hunk_to_xdl(&presented, &hunk) < 0) + goto comm_error; + if (nr_hunks >= OID_HUNKS_MAX) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'"), drv->process, path); + goto comm_error; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + if (len < 0) + goto comm_error; + + if (subprocess_read_status(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + if (validate_external_hunks(hunks, nr_hunks, -1, -1, + drv->process, path) < 0) { + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!nr_hunks) { + res = DIFF_PROCESS_EQUIVALENT; + goto out; + } + /* + * Replay in the coordinates a hunk consumer receives from + * xdiff's emission: 0-based starts. The answer is used as + * the tool sent it; with no content in hand it cannot be + * re-run through xdiff's compaction. + */ + for (i = 0; i < nr_hunks; i++) + hunk_cb(hunks[i].old_start - 1, hunks[i].old_count, + hunks[i].new_start - 1, hunks[i].new_count, + cb_data); + res = DIFF_PROCESS_OK; + goto out; + } + if (!strcmp(status.buf, "need-content")) { + /* The tool wants the content request; the caller sends it. */ + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!strcmp(status.buf, "abort")) { + /* + * The tool withdrew from oid-only answers: stop asking, but + * keep consulting it with content. + */ + backend->supported_capabilities &= ~CAP_OID_HUNKS; + res = DIFF_PROCESS_SKIP; + goto out; + } + warning(_("diff process '%s' failed for '%s'," + " falling back to builtin diff"), + drv->process, path); + res = DIFF_PROCESS_ERROR; +out: + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return res; + +comm_error: + drv->diff_process_failed = 1; + drv->diff_subprocess = NULL; + subprocess_stop_command(&backend->subprocess); + free(backend); + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} diff --git a/diff-process.h b/diff-process.h index 497031f188b5f8..2aff886bd48183 100644 --- a/diff-process.h +++ b/diff-process.h @@ -57,4 +57,25 @@ enum diff_process_result diff_process_fill_hunks( const struct object_id *oid_b, xpparam_t *xpp); +/* + * Ask the diff process configured for 'path' to answer from the blob + * pair's object ids alone (the "hunks-by-oid" capability): no content + * is loaded or sent. On DIFF_PROCESS_OK the tool's hunks are emitted + * through hunk_cb in 0-based emission coordinates, validated for order, + * overlap, and lockstep alignment first; because Git holds no content, + * the answer is used as the tool sent it, without xdiff's compaction. + * DIFF_PROCESS_EQUIVALENT means the tool asserts the pair equal. + * DIFF_PROCESS_SKIP covers everything that should fall through to a + * content consult: no driver or capability, a missing object id, a + * status=need-content answer, or an invalid response. + */ +enum diff_process_result diff_process_query_hunks( + struct diff_options *diffopt, + const char *path, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + #endif /* DIFF_PROCESS_H */ diff --git a/diff-provider.c b/diff-provider.c index 4304d50fb05cd7..5186d1caa66f63 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -78,6 +78,19 @@ int diff_provider_emit_hunks(struct repository *r, hunk_cb, cb_data)) return 1; + /* + * A tool that negotiated hunks-by-oid answers the identity phase + * itself; only a fall-through loads content. + */ + switch (diff_process_query_hunks(diffopt, path, old_oid, new_oid, + xpp, hunk_cb, cb_data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return 0; + default: + break; + } + if (fill(fill_data, &old_file, &new_file) < 0) return -1; diff --git a/diff.c b/diff.c index 1326c855349053..5ad295b57866b7 100644 --- a/diff.c +++ b/diff.c @@ -4395,11 +4395,24 @@ static int diffstat_from_hunks(struct diff_options *o, * A process-capable driver makes the tool the producer for the * path: the stat must reflect the tool's hunks, so neither a * store read (it holds xdiff's answer) nor this function's own - * xdiff-and-record may stand in. Step aside and let the caller - * consult the tool. + * xdiff-and-record may stand in. A tool that negotiated + * hunks-by-oid can answer right here, before any blob is read; + * otherwise step aside and let the caller consult it with + * content. */ - if (diff_process_driver(o, name_a, &probe)) - return 0; + if (diff_process_driver(o, name_a, &probe)) { + switch (diff_process_query_hunks(o, name_a, + one->oid_valid ? &one->oid : NULL, + two->oid_valid ? &two->oid : NULL, + &probe, diffstat_sum_hunk_cb, + data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return 1; + default: + return 0; + } + } /* * xpparam_t is the diff algorithm's input. Its flags are the key's diff --git a/t/helper/test-diff-process-backend.c b/t/helper/test-diff-process-backend.c index c2ec532c4a5cfa..2c64d2ca4ec633 100644 --- a/t/helper/test-diff-process-backend.c +++ b/t/helper/test-diff-process-backend.c @@ -41,6 +41,12 @@ * error (status=error instead of status=success) * abort (status=abort instead of status=success) * crash exit(1) before sending any response + * oid-fixed (advertises hunks-by-oid; answers any request, + * oid-only or content, with the fixed-hunk response) + * oid-need-content + * (advertises hunks-by-oid; answers an oid-only request + * with status=need-content, a content request with the + * fixed-hunk response) * * All success modes (not error/abort/crash) end with: * @@ -77,6 +83,8 @@ enum mode { MODE_ERROR, MODE_ABORT, MODE_CRASH, + MODE_OID_FIXED, + MODE_OID_NEED_CONTENT, }; static enum mode parse_mode(const char *s) @@ -113,6 +121,10 @@ static enum mode parse_mode(const char *s) return MODE_ABORT; if (!strcmp(s, "crash")) return MODE_CRASH; + if (!strcmp(s, "oid-fixed")) + return MODE_OID_FIXED; + if (!strcmp(s, "oid-need-content")) + return MODE_OID_NEED_CONTENT; die("unknown --mode=%s", s); } @@ -189,6 +201,9 @@ static void respond(enum mode mode, case MODE_CRASH: exit(1); case MODE_FIXED_HUNK: + case MODE_OID_FIXED: + case MODE_OID_NEED_CONTENT: + /* the oid modes reach here only for a content request */ packet_write_fmt(1, "hunk 5 2 5 2\n"); break; case MODE_BAD_HUNK: @@ -280,13 +295,18 @@ static void command_loop(enum mode mode) char *old_oid = NULL, *new_oid = NULL; struct strbuf obuf = STRBUF_INIT; struct strbuf nbuf = STRBUF_INIT; + int by_oid; if (!read_request_header(&command, &pathname, &old_oid, &new_oid)) break; /* EOF: Git closed its end */ - read_packetized_to_strbuf(0, &obuf, 0); - read_packetized_to_strbuf(0, &nbuf, 0); + /* An oid-only request has no content sections to read. */ + by_oid = command && !strcmp(command, "hunks-by-oid"); + if (!by_oid) { + read_packetized_to_strbuf(0, &obuf, 0); + read_packetized_to_strbuf(0, &nbuf, 0); + } if (logfile) { fprintf(logfile, @@ -303,7 +323,21 @@ static void command_loop(enum mode mode) fflush(logfile); } - respond(mode, &obuf, &nbuf); + if (by_oid) { + if (mode == MODE_OID_FIXED) { + packet_write_fmt(1, "hunk 5 2 5 2\n"); + send_status("status=success"); + } else { + /* + * oid-need-content, and the safe default for + * a mode that never advertised the + * capability: ask for the content request. + */ + send_status("status=need-content"); + } + } else { + respond(mode, &obuf, &nbuf); + } free(command); free(pathname); @@ -338,6 +372,8 @@ static void handshake(enum mode mode) /* Respond with our capabilities (or none for no-cap mode) */ if (mode != MODE_NO_CAP) packet_write_fmt(1, "capability=hunks\n"); + if (mode == MODE_OID_FIXED || mode == MODE_OID_NEED_CONTENT) + packet_write_fmt(1, "capability=hunks-by-oid\n"); packet_flush(1); } diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh index e01279b174518e..6fba150d109a5d 100755 --- a/t/t4080-diff-process.sh +++ b/t/t4080-diff-process.sh @@ -923,6 +923,49 @@ test_expect_success 'a warmed hunk store does not override tool hunks in --stat' test_cmp expect actual ' +test_expect_success 'an oid-capable tool answers blame without content' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse --short HEAD~1) && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep ! "command=hunks pathname=" backend.log +' + +test_expect_success 'an oid-capable tool answers --stat without content' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep ! "command=hunks pathname=" backend.log +' + +test_expect_success 'need-content falls back to a content request' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content --log=backend.log" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep "command=hunks pathname=blame-hunk.c" backend.log +' + +test_expect_success 'a worktree side sends content to an oid-capable tool' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat boundary.c >actual && + printf "2\t2\tboundary.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks pathname=boundary.c" backend.log && + test_grep ! "command=hunks-by-oid" backend.log +' + test_expect_success 'blame skips commits with no hunks from diff process' ' cat >blame.c <<-\EOF && int main(void) {