diff --git a/.gitignore b/.gitignore index 4da58c6754899e..4173111c01b2b6 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ /git-diagnose /git-diff /git-diff-files +/git-diff-hunks /git-diff-index /git-diff-pairs /git-diff-tree diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..170fcee66e23d5 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,6 +33,7 @@ MAN5_TXT += gitattributes.adoc MAN5_TXT += gitformat-bundle.adoc MAN5_TXT += gitformat-chunk.adoc MAN5_TXT += gitformat-commit-graph.adoc +MAN5_TXT += gitformat-diff-hunks.adoc MAN5_TXT += gitformat-index.adoc MAN5_TXT += gitformat-loose.adoc MAN5_TXT += gitformat-pack.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index a80e7db46d9697..7c006898c434d1 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -417,6 +417,8 @@ include::config/credential.adoc[] include::config/diff.adoc[] +include::config/diff-hunks.adoc[] + include::config/difftool.adoc[] include::config/extensions.adoc[] diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index a0ebf03e2eb050..9595619c610425 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -670,12 +670,13 @@ but risks losing recent work in the event of an unclean system shutdown. * `pack` hardens objects added to the repo in packfile form. * `pack-metadata` hardens packfile bitmaps and indexes. * `commit-graph` hardens the commit-graph file. +* `diff-hunks` hardens the diff-hunks store. * `index` hardens the index when it is modified. * `objects` is an aggregate option that is equivalent to `loose-object,pack`. * `reference` hardens references modified in the repo. * `derived-metadata` is an aggregate option that is equivalent to - `pack-metadata,commit-graph`. + `pack-metadata,commit-graph,diff-hunks`. * `committed` is an aggregate option that is currently equivalent to `objects`. This mode sacrifices some performance to ensure that work that is committed to the repository with `git commit` or similar commands @@ -750,6 +751,13 @@ core.commitGraph:: to parse the graph structure of commits. Defaults to true. See linkgit:git-commit-graph[1] for more information. +core.diffHunks:: + If true, then Git will consult the diff-hunks store (if it + exists) to skip recomputing diff hunk coordinates in commands + such as `git log --stat` and linkgit:git-blame[1]. This controls + only reading; writing the store is controlled by `diffHunks.write`. + See linkgit:git-diff-hunks[1] for more information. Defaults to true. + core.useReplaceRefs:: If set to `false`, behave as if the `--no-replace-objects` option was given on the command line. See linkgit:git[1] and diff --git a/Documentation/config/diff-hunks.adoc b/Documentation/config/diff-hunks.adoc new file mode 100644 index 00000000000000..60a4291e781755 --- /dev/null +++ b/Documentation/config/diff-hunks.adoc @@ -0,0 +1,7 @@ +diffHunks.write:: + If true, diff-producing commands (`git diff` and `git log` with a + `--stat`, `--numstat`, or `--shortstat` format) write the hunks + they compute to the diff-hunks store, filling it as a side effect. + The `GIT_DIFF_HUNKS_WRITE` environment variable overrides this for + a single invocation. Reading the store is controlled separately by + `core.diffHunks`. See linkgit:git-diff-hunks[1]. Defaults to false. diff --git a/Documentation/config/diff.adoc b/Documentation/config/diff.adoc index 1135a62a0ad3de..ac0635bb3bee1c 100644 --- a/Documentation/config/diff.adoc +++ b/Documentation/config/diff.adoc @@ -218,6 +218,11 @@ endif::git-diff[] Set this option to `true` to make the diff driver cache the text conversion outputs. See linkgit:gitattributes[5] for details. +`diff..process`:: + The command to run as a long-running diff process that + provides hunks to Git's diff pipeline. + See linkgit:gitattributes[5] for details. + `diff.indentHeuristic`:: Set this option to `false` to disable the default heuristics that shift diff hunk boundaries to make patches easier to read. diff --git a/Documentation/diff-algorithm-option.adoc b/Documentation/diff-algorithm-option.adoc index 8e3a0b63d784d8..4d7e2ec35f97ea 100644 --- a/Documentation/diff-algorithm-option.adoc +++ b/Documentation/diff-algorithm-option.adoc @@ -18,3 +18,6 @@ For instance, if you configured the `diff.algorithm` variable to a non-default value and want to use the default one, then you have to use `--diff-algorithm=default` option. ++ +If you explicitly choose a diff algorithm, it also bypasses +`diff..process` (see linkgit:gitattributes[5]). diff --git a/Documentation/diff-options.adoc b/Documentation/diff-options.adoc index 8a63b5e164114a..18b8b0ed242838 100644 --- a/Documentation/diff-options.adoc +++ b/Documentation/diff-options.adoc @@ -825,7 +825,9 @@ endif::git-format-patch[] to use this option with linkgit:git-log[1] and friends. `--no-ext-diff`:: - Disallow external diff drivers. + Disallow external diff helpers, including + `diff..command` and `diff..process` + (see linkgit:gitattributes[5]). `--textconv`:: `--no-textconv`:: diff --git a/Documentation/git-diff-hunks.adoc b/Documentation/git-diff-hunks.adoc new file mode 100644 index 00000000000000..041e8b42322248 --- /dev/null +++ b/Documentation/git-diff-hunks.adoc @@ -0,0 +1,125 @@ +git-diff-hunks(1) +================= + +NAME +---- +git-diff-hunks - Inspect and manage the precomputed diff hunk store + +SYNOPSIS +-------- +[synopsis] +git diff-hunks verify +git diff-hunks clear + +DESCRIPTION +----------- + +The diff hunk store is a cache of diff hunk coordinates, so that commands +which need them, such as linkgit:git-blame[1] and `git log` and `git diff` +with the `--stat`, `--numstat`, and `--shortstat` formats, can skip +decompressing blobs and running the diff algorithm. + +The store is a single file, `$GIT_DIR/objects/diff-hunks`. Reading is +enabled by default; writing is off by default. A `git diff`, `git log`, +`git show`, or `git diff-tree` that produces one of the stat formats +fills the store as a side effect, but only when writing is enabled for +that run (see "WARMING THE STORE" below), so ordinary reads never +modify the repository. When the store does not have the pair, holds a +different object hash, or the file is unreadable, the consumer falls +back to computing the diff. A store only speeds up these commands; it +never changes their output. + +`git diff-hunks` itself only inspects and manages the file. See +linkgit:gitformat-diff-hunks[5] for the file format. + +WARMING THE STORE +----------------- + +The store is filled by running ordinary commands with writing enabled. +Turn writing on for a single invocation with the `GIT_DIFF_HUNKS_WRITE` +environment variable, or persistently with the `diffHunks.write` +configuration; the environment variable takes precedence. A repository +owner warms the store by running the diff-producing commands they care +about with writing on, for example: + + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null + +A `--stat` walk records one entry per blob pair, which serves both +linkgit:git-blame[1] (it replays the coordinates) and the summary +formats (they sum the counts), so a single warming walk serves both. +A warm seeds from the existing store and rewrites the file with the +newly computed pairs merged in, so a later warm adds to what earlier +warms recorded rather than discarding it. + +COMMANDS +-------- + +`verify`:: + Check the integrity of the store: the trailing hash checksum, the + chunk table of contents, the sort order of the index, and the + bounds of every entry. Exits with non-zero status if the store is + corrupt. An absent store is valid. + +`clear`:: + Remove the store file. + +CORRECTNESS +----------- + +A stored result is interchangeable with a freshly computed one because an +entry is keyed by everything that determines the diff: + +* the object IDs of the old and new blob, so a result is used only for + the exact contents it was computed from; and +* the diff algorithm and ignore flags (`xdl_opts`) the hunks were + computed under. A lookup whose `xdl_opts` differ from a stored entry + misses. This is why, for example, `blame -w` and + `--diff-algorithm=` (including a per-path + `diff..algorithm`) do not reuse entries recorded under the + default settings: they change `xdl_opts`. + +The context length is not part of the key because only trim-stable +pairs are recorded: pairs whose zero-context trimmed diff and untrimmed +diff are identical, so one entry answers blame (zero context) and the +summary formats (any context) alike. The rare pair where +`trim_common_tail` picks a different but equally valid set of hunks is +never recorded and is always computed. + +Some options change the hunks in ways that are not part of the key, so +they are excluded from the store in both directions: break detection +(`-B`), `--ignore-matching-lines` (`-I`), `--anchored`, and +`--ignore-blank-lines`. linkgit:git-blame[1] additionally does not +consult the store for reverse blame, ignored revisions, or paths with a +textconv driver. + +The store carries a trailing hash checksum, but readers do not +re-checksum it on every load. As with the commit-graph and +multi-pack-index, the writer fsyncs the file (honoring `core.fsync`) and +commits it atomically, so a committed store is intact; every offset and +count is still bounds-checked as it is read. The checksum is verified by +`git diff-hunks verify`, not on the read path. + +CONFIGURATION +------------- + +`core.diffHunks`:: + Whether commands read the store. Defaults to true. See + linkgit:git-config[1]. + +`diffHunks.write`:: + Whether diff-producing commands write to the store. Defaults to + false. The `GIT_DIFF_HUNKS_WRITE` environment variable overrides it + for a single invocation. See linkgit:git-config[1]. + +Writing the store honors the `core.fsync` configuration through the +`diff-hunks` component; see linkgit:git-config[1]. + +SEE ALSO +-------- +linkgit:git-blame[1], +linkgit:git-log[1], +linkgit:gitformat-diff-hunks[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index bd76167a45eb71..bc231da2cca105 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -784,6 +784,16 @@ with the above configuration, i.e. `j-c-diff`, with 7 parameters, just like `GIT_EXTERNAL_DIFF` program is called. See linkgit:git[1] for details. +An external diff driver replaces the patch Git would otherwise +produce for the path: Git runs the command and shows its output in +place of its own. Output features that post-process Git's diff do +not apply to it; word diff, function context (`-W`), `--color-moved`, +and coloring all act on Git's builtin diff, not the driver's output. +The driver is consulted only when Git generates a textual patch. The +summary formats (`--stat`, `--numstat`, `--shortstat`, and +`--dirstat`), `git blame`, and `git log -L` do not run it and +continue to use Git's builtin diff. + If the program is able to ignore certain changes (similar to `git diff --ignore-space-change`), then also set the option `trustExitCode` to true. It is then expected to return exit code 1 if @@ -821,6 +831,294 @@ NOTE: If `diff..command` is defined for path with the (see above), and adding `diff..algorithm` has no effect, as the algorithm is not passed to the external diff driver. +Using an external diff process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If `diff..process` is defined, Git sends the old and new file +content to an external tool and receives back a list of changed +regions (pairs of line ranges in the old and new file). Git uses +these instead of its builtin diff algorithm, but still controls +all output formatting, so features like word diff, function context, +color, and blame work normally. This is achieved by using the +long-running process protocol (described in +Documentation/technical/long-running-process-protocol.adoc). +Unlike `diff..command`, which replaces Git's output entirely, +the diff process feeds results back into the standard pipeline. If +both are configured for a path, `diff..command` takes precedence +for the patch output it replaces; the summary formats, `git blame`, +and `git log -L` never run the command and still consult the process. + +First, in `.gitattributes`, assign the `diff` attribute for paths. + +------------------------ +*.c diff=cdiff +------------------------ + +Then, define a "diff..process" configuration to specify +the diff process command. + +---------------------------------------------------------------- +[diff "cdiff"] + process = /path/to/diff-process-tool +---------------------------------------------------------------- + +When Git encounters the first file that needs to be diffed, it starts +the process and performs the handshake. In the handshake, the welcome +message sent by Git is "git-diff-client", only version 1 is supported, +and the supported capability is "hunks" (the changed regions +described below). The tool replies with "git-diff-server", the +version it supports, and the capabilities it supports. + +For each file, Git sends a list of "key=value" pairs terminated with +a flush packet, followed by the old and new file content as packetized +data, each terminated with a flush packet. The pathname is relative +to the repository root. When `diff..textconv` is also set, +the tool receives the textconv-transformed content rather than the +raw blob, matching what the consuming feature itself diffs: patch +output is textconv'd, the summary formats (noted below) are not, and +`git blame` applies textconv only under `--textconv`. Git does not +send binary files to the diff process. + +----------------------- +packet: git> command=hunks +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +packet: git> OLD_CONTENT +packet: git> 0000 +packet: git> NEW_CONTENT +packet: git> 0000 +----------------------- + +The optional `old-oid` and `new-oid` keys give the object names of the +old and new blobs, so a tool can cache its analysis keyed on the pair. +A side's key is sent only when the content for that side is the raw +blob it names: it is omitted when the content is textconv-transformed, +and for a working-tree side that has no stored object. A tool that +does not recognize these keys ignores them. + +A tool that also announces the `hunks-by-oid` capability may be asked +to answer from the object names alone. Such a request uses +`command=hunks-by-oid`, always carries both keys, and is not followed +by any content section: + +----------------------- +packet: git> command=hunks-by-oid +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +----------------------- + +The tool answers in the same hunk-line form as below, or with +`status=need-content`, upon which Git repeats the request as a full +`command=hunks` exchange with the content included. Git holds no +content for an oid-only exchange, so the answer is used exactly as +sent: the hunks feed the consuming feature directly, and a +`status=success` response with zero hunks asserts that the blobs are +equivalent, including their trailing newlines. A tool that cannot +promise that from its cache answers `need-content`. Blame and the +summary formats send such requests for stored blob pairs; patch +output always sends content. + +The tool is expected to respond with zero or more hunk lines, +a flush packet, and a status packet terminated with a flush packet. +Each hunk line has the form: + + `hunk ` + +where `` and `` identify a range of lines in +the old file, and `` and `` identify the +replacement range in the new file. The four fields are separated by +single spaces. Start values are 1-based and counts are non-negative. +For example, `hunk 3 2 3 4` means that 2 lines starting at line 3 in +the old file were replaced by 4 lines starting at line 3 in the new +file. An `` of 0 means no lines were removed (pure +insertion); a `` of 0 means no lines were added (pure +deletion). For a side with a count of 0 (a pure insertion or +deletion) the start is the 1-based line the change sits before, +ranging from 1 to one past the last line (the line count plus 1, to +place the change at the end of the file); like every start it must +keep the unchanged runs aligned on both sides (see below), so for a +given change it takes one specific value, not an arbitrary one. A +start of 0 is also accepted and treated as 1, matching the +empty-file-side form `git diff` emits (e.g. `hunk 0 0 1 5` for a newly +added file). A nonzero range must not extend beyond the end of the +file. Git ignores any extra +whitespace-separated tokens after ``, so a future protocol +version can append fields to a hunk line (for example a "moved" +marker) without older tools rejecting it. + +Lines are delimited by newlines. A file `"foo\nbar\n"` and a +file `"foo\nbar"` both have 2 lines. + +Hunks must be listed in order and must not overlap. Any line not +covered by a hunk is treated as unchanged and is paired, in order, +with the unchanged lines on the other side. Each run of unchanged +lines between two hunks (and the run before the first hunk and +after the last) must therefore be the same length on both sides, +not merely equal in total. For the hunks `1 3 1 5` and `10 2 12 2` +below, lines 4-9 of the old file and lines 6-11 of the new file are +both the six unchanged lines between the two hunks. A response that +balances only the total unchanged count but misaligns one of these +runs is rejected, and Git falls back to the builtin diff. + +Git does not check that the lines a hunk leaves unchanged are +byte-for-byte identical between the two sides; it pairs them by +position and shows the new side as context. A tool may therefore +report lines that differ textually (a pure reformatting, say) as +unchanged, and the diff reflects that judgment. This is +the point of a semantic backend, but it means a misbehaving tool can +produce a diff whose context does not match the old blob; as with +`git diff -w`, such a patch may not apply against the old content. + +----------------------- +packet: git< hunk 1 3 1 5 +packet: git< hunk 10 2 12 2 +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +If the tool responds with hunks and "success", Git marks those lines +as changed and feeds them into the standard diff pipeline. Git may +still slide or regroup those changes against matching context for +display, exactly as it compacts its own diffs, so the tool controls +which lines are reported as changed, not the precise hunk boundaries. +Patch output features (word diff, function context, color) work +normally, as do summary formats like `--stat`. Not every feature +consults the process, though; see "Which features consult the diff +process" below for the full picture and the reasoning behind it. + +If no hunk lines precede the flush, followed by "success", Git +treats the files as having no changes: `git diff` produces no output, +`git diff --exit-code` and `--quiet` report success even though the +stored blobs differ, and `git blame` skips the commit, attributing +lines to earlier commits. +The one exception is a change that only adds or removes the file's +trailing newline: it cannot be expressed as line hunks, so when the +line content otherwise matches Git keeps the builtin diff for that +file (preserving the `\ No newline at end of file` marker) instead of +treating the two sides as equal. + +----------------------- +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +If the tool returns well-formed but invalid hunks (out of bounds, +overlapping, or with misaligned unchanged runs), Git warns and falls +back to the builtin diff for that file; the tool stays available for +subsequent files. A malformed hunk line, by contrast (bad syntax, a +nonzero count paired with a start of 0, or more hunks than the file +has lines), is a protocol violation: Git stops the process and does +not send it further requests, as described below. + +In case the tool cannot or does not want to process the content, +it is expected to respond with an "error" status. Git warns and +falls back to the builtin diff algorithm for this file, treating any +status other than "success" or "abort" the same way. The tool +remains available for subsequent files. + +----------------------- +packet: git< 0000 +packet: git< status=error +packet: git< 0000 +----------------------- + +In case the tool cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, it +is expected to respond with an "abort" status. Git silently falls +back to the builtin diff algorithm for this file and does not send +further requests to the tool. + +----------------------- +packet: git< 0000 +packet: git< status=abort +packet: git< 0000 +----------------------- + +If the tool dies during the communication or does not adhere to the +protocol then Git will stop the process and fall back to the builtin +diff algorithm. Git warns once and does not restart the process for +subsequent files. + +Tools should ignore unknown keys in the per-file request to remain +forward-compatible. Future versions of Git may send additional +`command=` values; tools that receive an unrecognized command should +respond with `status=error` rather than terminating. + +Which features consult the diff process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The diff process answers a single question: given two blobs, which +line ranges differ? Whether a particular feature consults it follows +from whether that is the question the feature is really asking. + +Features that ask "which lines changed" use the tool's hunks in place +of the builtin algorithm: + +- `git diff` patch output, together with everything layered on it: + word diff, function context (`-W`), `--color-moved`, and the `@@` + hunk headers. These operate on the lines the patch step already + emitted, so they reflect the tool's hunks without any further + negotiation. +- `git blame`: a commit whose change the tool reports as equivalent is + skipped, and its lines are attributed to an earlier commit. +- `git log -L`: both the line-range display and the underlying range + tracking consult the tool, so a commit it reports as equivalent is + dropped from the log (its tracked range maps across unchanged) + rather than selected and then shown with an empty diff. +- `--stat`, `--numstat`, and `--shortstat`: the inserted and deleted + counts come from the tool's hunks, so a file the tool calls + equivalent contributes no stat line, matching the empty patch that + `git diff` produces for it. These summary formats do not apply + textconv (just as the builtin summary path does not), so the tool + is consulted on the raw blob content even when a `textconv` is also + configured for patch output; this mirrors how builtin `--stat` + already counts raw lines rather than the textconv'd view. The + line-counting `--dirstat=lines` uses these same counts; the default + `--dirstat`, which weighs byte changes, is computed on its own and + does not consult the tool. + +Features that ask a different question do not consult the process, by +design: + +- The pickaxe `-G` searches the textual diff for a pattern; it + asks "does this string appear in the diff," not "did these lines + change." (`-S` runs at an earlier stage and is likewise unaffected.) +- `git patch-id` must produce a stable hash for `git rebase` and + cherry-pick detection; deriving it from a configured tool would make + equal patches hash differently from machine to machine. +- The merge machinery (`git merge-tree`, `rerere`) computes merge + content and conflict signatures rather than display output, so the + tool's hunks must not alter its results. +- `git range-diff` diffs patch text, not source blobs, so source-file + hunks do not apply to it. +- `--check` reports whitespace errors in added lines using the builtin + diff's notion of which lines are added, not the tool's. It can + therefore flag (and exit non-zero on) a line the tool treats as + unchanged and that `git diff` shows as context. Whitespace breakage + is a property of the literal bytes, so `--check` keeps the builtin + partition deliberately; a future change could wire it to the tool if + matching `git diff` exactly became desirable. +- `--raw`, `--name-only`, and `--name-status` compare object ids at + the tree level and never run a line-level diff at all. + +Combined diffs (`--cc` and merge diffs) ask "which lines changed" but +still use the builtin algorithm, and may consult the process in a +later change; their protocol would have to be extended from a single +old/new pair to one comparison per merge parent. + +`--no-ext-diff` and `--diff-algorithm` bypass the process entirely, +for every feature listed above. The whitespace-ignoring options +(`-w`, `--ignore-space-change`, `--ignore-blank-lines`, and the like), +`-I`, and `--anchored` also bypass it for the affected files: +the tool is never told about these options, so it could not honor +them, and Git falls back to the builtin diff, which does. + Defining a custom hunk-header ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Documentation/gitformat-diff-hunks.adoc b/Documentation/gitformat-diff-hunks.adoc new file mode 100644 index 00000000000000..70fe2d1f6f801b --- /dev/null +++ b/Documentation/gitformat-diff-hunks.adoc @@ -0,0 +1,122 @@ +gitformat-diff-hunks(5) +======================= + +NAME +---- +gitformat-diff-hunks - Precomputed diff hunk store format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/diff-hunks + +DESCRIPTION +----------- + +The diff-hunks store memoizes diff hunk coordinates so that commands +that need them, such as `git log --stat` and linkgit:git-blame[1], can +skip decompressing blobs and running the diff algorithm. See +linkgit:git-diff-hunks[1] for how the store is filled and managed and the +configuration that controls it. + +The store is a single file, `$GIT_DIR/objects/diff-hunks`, written in +one pass and replaced atomically, so a reader sees either the old file +or the complete new one. + +Entries are keyed by the object IDs of the blob pair that was diffed +and by the diff algorithm and ignore flags (`xdl_opts`) the pair was +diffed under. A blob pair fully determines the diff input, so an entry +is valid regardless of which commits, branches, or index states the +pair was encountered in, and identical diffs performed in different +contexts share one entry. A reader whose `xdl_opts` differ from an +entry does not match it and falls back to computing the diff. + +FILE FORMAT +----------- + +All multi-byte integers are stored in network byte order. The file is an +8-byte header, the chunk table of contents and chunk data described in +linkgit:gitformat-chunk[5], and a trailing checksum. + +HEADER +~~~~~~ + +- 4-byte signature: `DHPF` (diff-hunks precomputed format) +- 1-byte version number: currently 1 +- 1-byte hash version: 1 for SHA-1, 2 for SHA-256. A store whose hash + function differs from the repository's is ignored. +- 1-byte number of chunks +- 1-byte reserved + +CHUNK LOOKUP +~~~~~~~~~~~~ + +A table of contents in the format of linkgit:gitformat-chunk[5], listing +the offset of each chunk. Both chunks below are required; a file missing +either is treated as corrupt. + +CHUNK DATA +~~~~~~~~~~ + +DHIX (index):: + A sorted sequence of fixed-size entries. Each entry is the old + blob object ID, the new blob object ID, a 4-byte `xdl_opts` + value, and a 4-byte offset into the DHDT chunk. Entries are + sorted by old object ID, then new object ID, then `xdl_opts`, + so lookups can use binary search on the full key. + +DHDT (hunk data):: + For each index entry, at its offset: a 4-byte hunk count followed + by that many 16-byte hunk records. A hunk record is four 4-byte + values: old start, old count, new start, new count, as produced + by the xdiff hunk callback. Identical hunk blocks are stored once: + distinct index entries whose recorded hunks are byte-for-byte + equal point at the same offset. + +TRAILER +~~~~~~~ + +A checksum of all preceding bytes, computed with the repository hash +function. + +CORRECTNESS +----------- + +Serving hunks from a valid store produces the same output as recomputing +the diff. The diff of a blob pair is not unique: a zero context length +triggers trim_common_tail, which can pick a different but equally valid +set of hunks than an untrimmed diff does. A pair is therefore recorded +only when its trimmed and untrimmed diffs are identical, which holds for +the overwhelming majority of pairs. Such an entry answers any consumer +at any context: git-blame replays its coordinates directly (it diffs at +zero context), and diffstat sums its per-hunk line counts, which the +context length does not change. The rare pair whose two diffs differ is +never recorded, so every consumer computes it. + +A store that cannot be used is ignored, and the consumer falls back to +computing the diff. Every offset and count read from the file is +bounds-checked, so a store that is missing, truncated, of an unknown +version, or of a different object hash does not change the diff output +(a malformed store may still produce a diagnostic on stderr). + +The store is not re-checksummed on the read path. The writer fsyncs the +file (honoring `core.fsync`) and commits it atomically, so a +committed store is intact, the same trust model the commit-graph and +multi-pack-index use. The trailing checksum is recomputed by +`git diff-hunks verify` to detect corruption. + +The checksum detects corruption but does not prove who wrote the file. A +reader trusts the coordinates in a store that passes its checks, so +anything able to write a checksum-valid file at the store path can +influence output, the same as it could by writing objects directly. + +LIMITATIONS +----------- + +- Hunk counts, offsets, and line coordinates are 32-bit, capping a + single entry at roughly four billion hunks and the hunk data at 4 GiB. + A result whose coordinates cannot be represented is not recorded. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 72f639b5e79ea4..d619ffa6336513 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -9,13 +9,20 @@ __ and __ (or __) must exist in the starting revision. You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. - Non-patch diff formats `--raw`, `--name-only`, `--name-status`, - and `--summary` are supported. Diff stat formats - (`--stat`, `--numstat`, `--shortstat`, `--dirstat`) are not - currently implemented. + The following non-patch diff formats are supported: `--raw`, + `--name-only`, `--name-status`, `--summary`, `--check`, + `--stat`, `--numstat`, and `--shortstat`. + The stat formats count only lines within the tracked range. + `--dirstat` is not supported + with `-L`: it summarizes change as each directory's share of + the total churn, not as counts for the tracked lines. Use + `--numstat` for exact per-file counts within the range. + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, -as are pickaxe options (`-S`, `-G`) and `--diff-filter`. +as are pickaxe options (`-S`, `-G`) and `--diff-filter`. `-G` is +scoped to the tracked range; `-S` is still evaluated over the whole +file, so an `-S` query may select a commit for a change outside the +range. + include::line-range-format.adoc[] diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..85f37da47e9cac 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -41,6 +41,7 @@ manpages = { 'git-describe.adoc' : 1, 'git-diagnose.adoc' : 1, 'git-diff-files.adoc' : 1, + 'git-diff-hunks.adoc' : 1, 'git-diff-index.adoc' : 1, 'git-diff-pairs.adoc' : 1, 'git-difftool.adoc' : 1, @@ -175,6 +176,7 @@ manpages = { 'gitformat-bundle.adoc' : 5, 'gitformat-chunk.adoc' : 5, 'gitformat-commit-graph.adoc' : 5, + 'gitformat-diff-hunks.adoc' : 5, 'gitformat-index.adoc' : 5, 'gitformat-loose.adoc' : 5, 'gitformat-pack.adoc' : 5, diff --git a/Makefile b/Makefile index b31ecb07564a73..2e706ce58c9d0c 100644 --- a/Makefile +++ b/Makefile @@ -811,6 +811,7 @@ TEST_BUILTINS_OBJS += test-csprng.o TEST_BUILTINS_OBJS += test-date.o TEST_BUILTINS_OBJS += test-delete-gpgsig.o TEST_BUILTINS_OBJS += test-delta.o +TEST_BUILTINS_OBJS += test-diff-process-backend.o TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o TEST_BUILTINS_OBJS += test-dump-cache-tree.o @@ -1140,6 +1141,8 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-process.o +LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o LIB_OBJS += diffcore-delta.o @@ -1147,6 +1150,7 @@ LIB_OBJS += diffcore-order.o LIB_OBJS += diffcore-pickaxe.o LIB_OBJS += diffcore-rename.o LIB_OBJS += diffcore-rotate.o +LIB_OBJS += diff-hunks.o LIB_OBJS += dir-iterator.o LIB_OBJS += dir.o LIB_OBJS += editor.o @@ -1406,6 +1410,7 @@ BUILTIN_OBJS += builtin/credential.o BUILTIN_OBJS += builtin/describe.o BUILTIN_OBJS += builtin/diagnose.o BUILTIN_OBJS += builtin/diff-files.o +BUILTIN_OBJS += builtin/diff-hunks.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-pairs.o BUILTIN_OBJS += builtin/diff-tree.o diff --git a/blame.c b/blame.c index 977cbb70974f8c..5fcafae0b101e8 100644 --- a/blame.c +++ b/blame.c @@ -23,6 +23,8 @@ #include "commit-slab.h" #include "bloom.h" #include "commit-graph.h" +#include "diff-provider.h" +#include "userdiff.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1933,6 +1935,43 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +/* + * A hunk provider's key names the (old blob, new blob) pair and may only + * serve a diff whose result is determined by that pair and the xdiff + * settings. Textconv rewrites the buffers being diffed away from the + * blob contents the key names, so any origin whose path has a textconv + * driver must withhold the pair's identity. + */ +static int blame_textconv_active(struct blame_scoreboard *sb, + const char *path) +{ + struct userdiff_driver *drv; + + if (!sb->revs->diffopt.flags.allow_textconv) + return 0; + drv = userdiff_find_by_path(sb->repo->index, path); + return drv && drv->textconv; +} + +struct blame_diff_fill { + struct blame_scoreboard *sb; + struct blame_origin *parent, *target; + int ignore_diffs; +}; + +/* Content load for diff_provider_emit_hunks(): runs when the diff is computed. */ +static int blame_diff_fill(void *data, mmfile_t *old_file, mmfile_t *new_file) +{ + struct blame_diff_fill *f = data; + + fill_origin_blob(&f->sb->revs->diffopt, f->parent, old_file, + &f->sb->num_read_blob, f->ignore_diffs); + fill_origin_blob(&f->sb->revs->diffopt, f->target, new_file, + &f->sb->num_read_blob, f->ignore_diffs); + f->sb->num_get_patch++; + return 0; +} + /* * We are looking at the origin 'target' and aiming to pass blame * for the lines it is suspected to its parent. Run diff to find @@ -1942,9 +1981,11 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_origin *target, struct blame_origin *parent, int ignore_diffs) { - mmfile_t file_p, file_o; struct blame_chunk_cb_data d; struct blame_entry *newdest = NULL; + struct blame_diff_fill fill = { sb, parent, target, ignore_diffs }; + xpparam_t xpp = { .flags = sb->xdl_opts }; + int provider_usable, served; if (!target->suspects) return; /* nothing remains for this target */ @@ -1955,16 +1996,39 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; - fill_origin_blob(&sb->revs->diffopt, parent, &file_p, - &sb->num_read_blob, ignore_diffs); - fill_origin_blob(&sb->revs->diffopt, target, &file_o, - &sb->num_read_blob, ignore_diffs); - sb->num_get_patch++; + /* + * Offer the pair's identity only where blame's diff is the plain + * blob-pair diff a provider's key describes; reverse blame, + * ignored revisions, and textconv paths withhold it and always + * compute. + */ + provider_usable = !sb->reverse && !ignore_diffs && + !blame_textconv_active(sb, target->path) && + !blame_textconv_active(sb, parent->path); - if (diff_hunks(&file_p, &file_o, blame_chunk_cb, &d, sb->xdl_opts)) + /* + * Look up the driver by the parent (old) path, as builtin_diff() + * does with name_a, so a renamed file resolves to the same driver + * across diff, blame, and line-log. A tool that reports a pair + * equivalent emits no hunks, so blame passes the whole commit + * through and looks past it. + */ + served = diff_provider_emit_hunks(sb->repo, + provider_usable ? &parent->blob_oid : NULL, + provider_usable ? &target->blob_oid : NULL, + parent->path, &sb->revs->diffopt, + &xpp, blame_diff_fill, &fill, + blame_chunk_cb, &d); + if (served < 0) die("unable to generate diff (%s -> %s)", oid_to_hex(&parent->commit->object.oid), oid_to_hex(&target->commit->object.oid)); + if (provider_usable && diff_provider_active(sb->repo)) { + if (served) + sb->num_precomputed_hits++; + else + sb->num_precomputed_misses++; + } /* The rest are the same as the parent */ blame_chunk(&d.dstq, &d.srcq, INT_MAX, d.offset, INT_MAX, 0, parent, target, 0); diff --git a/blame.h b/blame.h index 3b34be0e5c6932..cd478ca477dc72 100644 --- a/blame.h +++ b/blame.h @@ -132,6 +132,8 @@ struct blame_scoreboard { int num_read_blob; int num_get_patch; int num_commits; + int num_precomputed_hits; + int num_precomputed_misses; /* * blame for a blame_entry with score lower than these thresholds diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..7e64da9f433836 100644 --- a/builtin.h +++ b/builtin.h @@ -175,6 +175,7 @@ int cmd_credential_store(int argc, const char **argv, const char *prefix, struct int cmd_describe(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diagnose(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_files(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_index(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_pairs(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/blame.c b/builtin/blame.c index ffbd3ce5c5a2e3..a8ae47c8fd4fbe 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -1033,7 +1033,7 @@ int cmd_blame(int argc, parse_done: revision_opts_finish(&revs); no_whole_file_rename = !revs.diffopt.flags.follow_renames; - xdl_opts |= revs.diffopt.xdl_opts & XDF_INDENT_HEURISTIC; + xdl_opts |= revs.diffopt.xdl_opts & DIFF_HUNKS_DEFAULT_XDL_OPTS; revs.diffopt.flags.follow_renames = 0; argc = parse_options_end(&ctx); @@ -1294,6 +1294,8 @@ int cmd_blame(int argc, printf("num read blob: %d\n", sb.num_read_blob); printf("num get patch: %d\n", sb.num_get_patch); printf("num commits: %d\n", sb.num_commits); + printf("num precomputed hits: %d\n", sb.num_precomputed_hits); + printf("num precomputed misses: %d\n", sb.num_precomputed_misses); } cleanup: diff --git a/builtin/diff-hunks.c b/builtin/diff-hunks.c new file mode 100644 index 00000000000000..d5b1b42c3b8450 --- /dev/null +++ b/builtin/diff-hunks.c @@ -0,0 +1,49 @@ +#include "builtin.h" +#include "config.h" +#include "diff-hunks.h" +#include "gettext.h" +#include "parse-options.h" +#include "repository.h" + +static const char * const diff_hunks_usage[] = { + N_("git diff-hunks verify"), + N_("git diff-hunks clear"), + NULL +}; + +static int cmd_diff_hunks_verify(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + return diff_hunks_verify(r) ? 1 : 0; +} + +static int cmd_diff_hunks_clear(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + return diff_hunks_clear(r) ? 1 : 0; +} + +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, + struct repository *repo) +{ + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("verify", &fn, cmd_diff_hunks_verify), + OPT_SUBCOMMAND("clear", &fn, cmd_diff_hunks_clear), + OPT_END() + }; + + repo_config(repo, git_default_config, NULL); + + argc = parse_options(argc, argv, prefix, options, diff_hunks_usage, 0); + + return fn(argc, argv, prefix, repo); +} diff --git a/builtin/diff-tree.c b/builtin/diff-tree.c index 8b8f8b54e40664..296c6a137e23c6 100644 --- a/builtin/diff-tree.c +++ b/builtin/diff-tree.c @@ -170,6 +170,8 @@ int cmd_diff_tree(int argc, opt->diffopt.rotate_to_strict = 1; + diff_hunks_attach(&opt->diffopt); + /* * NOTE! We expect "a..b" to expand to "^a b" but it is * perfectly valid for revision range parser to yield "b ^a", @@ -234,5 +236,6 @@ int cmd_diff_tree(int argc, diff_free(&opt->diffopt); } + diff_hunks_detach(&opt->diffopt); return diff_result_code(opt); } diff --git a/builtin/diff.c b/builtin/diff.c index 4b46e394cecb8d..931d9d8bbe8f39 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -568,6 +568,15 @@ int cmd_diff(int argc, } } + /* + * The hunk store is keyed by blob pair, so any diff whose + * file pairs carry known blob object IDs (tree-to-tree, + * index-to-tree) can consult the same entries that + * "git log --stat" and "git blame" use; pairs without known + * blobs bypass it at lookup time. + */ + diff_hunks_attach(&rev.diffopt); + symdiff_prepare(&rev, &sdiff); for (i = 0; i < rev.pending.nr; i++) { struct object_array_entry *entry = &rev.pending.objects[i]; @@ -644,6 +653,7 @@ int cmd_diff(int argc, result = diff_result_code(&rev); if (1 < rev.diffopt.skip_stat_unmatch) refresh_index_quietly(); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); object_array_clear(&ent); symdiff_release(&sdiff); diff --git a/builtin/log.c b/builtin/log.c index e464b30af4bcae..c3d9cf4e88d29a 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -693,8 +693,11 @@ int cmd_show(int argc, opt.tweak = show_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + if (!rev.no_walk) { ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -767,6 +770,7 @@ int cmd_show(int argc, } rev.diffopt.no_free = 0; + diff_hunks_detach(&rev.diffopt); diff_free(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); @@ -846,8 +850,11 @@ int cmd_log(int argc, opt.tweak = log_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -2217,6 +2224,13 @@ int cmd_format_patch(int argc, if (argc > 1) die(_("unrecognized argument: %s"), argv[1]); + /* + * Disable diff..process so that patches generated by + * format-patch are always based on the builtin diff algorithm + * and can be applied reliably. + */ + rev.diffopt.flags.no_diff_process = 1; + if (rev.diffopt.output_format & DIFF_FORMAT_NAME) die(_("--name-only does not make sense")); if (rev.diffopt.output_format & DIFF_FORMAT_NAME_STATUS) diff --git a/command-list.txt b/command-list.txt index 21b802c42026b3..e7b241e6ad8498 100644 --- a/command-list.txt +++ b/command-list.txt @@ -95,6 +95,7 @@ git-describe mainporcelain git-diagnose ancillaryinterrogators git-diff mainporcelain info git-diff-files plumbinginterrogators +git-diff-hunks plumbingmanipulators git-diff-index plumbinginterrogators git-diff-pairs plumbinginterrogators git-diff-tree plumbinginterrogators @@ -223,6 +224,7 @@ gitfaq guide gitformat-bundle developerinterfaces gitformat-chunk developerinterfaces gitformat-commit-graph developerinterfaces +gitformat-diff-hunks developerinterfaces gitformat-index developerinterfaces gitformat-pack developerinterfaces gitformat-signature developerinterfaces diff --git a/diff-hunks.c b/diff-hunks.c new file mode 100644 index 00000000000000..454950f8c0a0fe --- /dev/null +++ b/diff-hunks.c @@ -0,0 +1,856 @@ +/* + * Precomputed diff hunks, keyed by diff input. + * + * A single store at .git/objects/diff-hunks maps an (old blob, new + * blob, xdl_opts) key to the hunk coordinates of diffing the pair. + * The key determines the diff result (only trim-stable pairs are + * recorded; see diff-hunks.h), so an entry is valid in any context it + * recurs in, independent of path. Reading is always on; writing is off + * by default and enabled per run (see diff_hunks_write_enabled), so an + * ordinary command populates the store only during a warming run the + * repository owner opts into. + * + * File layout: + * Header: "DHPF"(4) + version(1) + hash_version(1) + num_chunks(1) + reserved(1) + * Table of contents (chunk-format) + * DHIX chunk: sorted entries, each + * old_blob_oid, new_blob_oid, xdl_opts(4), hdat_offset(4) + * DHDT chunk: per entry, num_hunks(4) followed by that many 16-byte hunks + * Trailing hash checksum + */ +#include "git-compat-util.h" +#include "chunk-format.h" +#include "config.h" +#include "csum-file.h" +#include "diff-hunks.h" +#include "diff-provider.h" +#include "gettext.h" +#include "hash.h" +#include "hashmap.h" +#include "lockfile.h" +#include "odb.h" +#include "path.h" +#include "repo-settings.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +#define DIFF_HUNKS_SIGNATURE 0x44485046 /* "DHPF" */ +/* + * Bump when the on-disk format changes, or when xdiff's emitted hunk + * coordinates change for a fixed (blobs, xdl_opts) key: an old store + * would otherwise serve stale hunks and change command output. + */ +#define DIFF_HUNKS_VERSION 1 +#define DIFF_HUNKS_HEADER_SIZE 8 + +#define DIFF_HUNKS_CHUNKID_INDEX 0x44484958 /* "DHIX" */ +#define DIFF_HUNKS_CHUNKID_DATA 0x44484454 /* "DHDT" */ + +/* Each hunk is 16 bytes on disk: old_start(4) old_count(4) new_start(4) new_count(4) */ +#define DIFF_HUNKS_HUNK_SIZE (4 * sizeof(uint32_t)) + +/* + * Result of a store lookup: num_hunks records encoded in the store's mmap, + * valid until the store is freed. Read them with nth_precomputed_hunk(). + */ +struct precomputed_entry { + uint32_t num_hunks; + const unsigned char *hunk_data; +}; + +/* Decode a single hunk from the raw on-disk format. */ +static inline void decode_precomputed_hunk(const unsigned char *data, + struct precomputed_hunk *h) +{ + h->old_start = get_be32(data); + h->old_count = get_be32(data + 4); + h->new_start = get_be32(data + 8); + h->new_count = get_be32(data + 12); +} + +/* Decode the nth hunk of a lookup result into *h. */ +static inline void nth_precomputed_hunk(const struct precomputed_entry *e, + uint32_t n, struct precomputed_hunk *h) +{ + decode_precomputed_hunk(e->hunk_data + (size_t)n * DIFF_HUNKS_HUNK_SIZE, h); +} + +/* Byte length of the (old_oid, new_oid, xdl_opts) lookup key. */ +static size_t store_index_key_size(const struct git_hash_algo *algo) +{ + return 2 * algo->rawsz + sizeof(uint32_t); +} + +/* Index entry: the lookup key followed by the 4-byte offset into DHDT. */ +static size_t store_index_entry_size(const struct git_hash_algo *algo) +{ + return store_index_key_size(algo) + sizeof(uint32_t); +} + +/* + * The smallest a valid store file can be: the header, a table of contents + * with one entry per chunk plus a terminating entry, and the trailing + * checksum. + */ +static size_t store_min_size(const struct git_hash_algo *algo, + uint8_t num_chunks) +{ + size_t toc_size = (num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE; + + return DIFF_HUNKS_HEADER_SIZE + toc_size + algo->rawsz; +} + +/* + * Decode an index entry's key into pointers to the two oids and the + * xdl_opts value (on-disk: old_oid, new_oid, then xdl_opts as a + * big-endian uint32). + */ +static void decode_store_index_key(const unsigned char *entry, unsigned int rawsz, + const unsigned char **old_hash, + const unsigned char **new_hash, + uint32_t *xdl_opts) +{ + *old_hash = entry; + *new_hash = entry + rawsz; + *xdl_opts = get_be32(entry + 2 * rawsz); +} + +/* The DHDT offset stored in an index entry, in the field after its key. */ +static uint32_t index_entry_hdat_offset(const unsigned char *entry, size_t keysz) +{ + return get_be32(entry + keysz); +} + +static char *store_path_ext(struct repository *r, const char *ext) +{ + struct strbuf sb = STRBUF_INIT; + + strbuf_addf(&sb, "%s/diff-hunks%s", repo_get_object_directory(r), ext); + return strbuf_detach(&sb, NULL); +} + +static char *diff_hunks_store_path(struct repository *r) +{ + return store_path_ext(r, ""); +} + +struct diff_hunks_store { + const unsigned char *data; + size_t data_len; + const struct git_hash_algo *hash_algo; + const unsigned char *index; + uint32_t num_entries; + const unsigned char *hdat; + size_t hdat_size; +}; + +static void free_store(struct diff_hunks_store *s) +{ + if (!s) + return; + if (s->data) + munmap((void *)s->data, s->data_len); + free(s); +} + +/* + * Open, mmap, and parse the store at fname. Returns the parsed store + * or NULL on any error. The diff output is unaffected either way; + * corruption is reported by verify, not treated as fatal here. + */ +static struct diff_hunks_store *load_store_at( + const struct git_hash_algo *repo_algo, const char *fname) +{ + struct diff_hunks_store *s; + struct chunkfile *cf; + int fd; + struct stat st; + void *data; + const unsigned char *p; + uint8_t num_chunks; + size_t index_size, entry_size, data_len; + + fd = git_open(fname); + if (fd < 0) + return NULL; + if (fstat(fd, &st) || st.st_size < DIFF_HUNKS_HEADER_SIZE) { + close(fd); + return NULL; + } + data_len = xsize_t(st.st_size); + data = xmmap(NULL, data_len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + p = data; + + num_chunks = p[6]; + + /* + * Reject a file that is not a readable store: wrong signature, + * version, or object hash, or too small to hold the table of + * contents that read_table_of_contents() walks (it dereferences + * each entry before range-checking its offset). + */ + if (get_be32(p) != DIFF_HUNKS_SIGNATURE || + p[4] != DIFF_HUNKS_VERSION || + p[5] != oid_version(repo_algo) || + data_len < store_min_size(repo_algo, num_chunks)) { + munmap(data, data_len); + return NULL; + } + + /* + * The trailing checksum is not verified here: the writer fsyncs + * and commits atomically, so a committed file is intact, and + * every record is bounds-checked at read (see precomputed_entry_at). + * The checksum is checked separately, by diff_hunks_verify(). + */ + + CALLOC_ARRAY(s, 1); + s->data = data; + s->data_len = data_len; + s->hash_algo = repo_algo; + + cf = init_chunkfile(NULL); + if (read_table_of_contents(cf, p, data_len, + DIFF_HUNKS_HEADER_SIZE, num_chunks, 1) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, &s->index, &index_size) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, &s->hdat, &s->hdat_size)) { + free_chunkfile(cf); + goto corrupt; + } + free_chunkfile(cf); + + entry_size = store_index_entry_size(s->hash_algo); + if (index_size % entry_size) + goto corrupt; + s->num_entries = index_size / entry_size; + return s; + +corrupt: + free_store(s); + return NULL; +} + +static struct diff_hunks_store *diff_hunks_store_load(struct repository *r) +{ + struct diff_hunks_store *s; + char *fname; + + prepare_repo_settings(r); + if (!r->settings.core_diff_hunks) + return NULL; + + fname = diff_hunks_store_path(r); + s = load_store_at(r->hash_algo, fname); + free(fname); + return s; +} + +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r) +{ + if (!r->objects) + return NULL; + if (r->objects->diff_hunks_store_attempted) + return r->objects->diff_hunks_store; + r->objects->diff_hunks_store_attempted = 1; + r->objects->diff_hunks_store = diff_hunks_store_load(r); + return r->objects->diff_hunks_store; +} + +void close_diff_hunks_store(struct object_database *o) +{ + if (!o->diff_hunks_store) + return; + free_store(o->diff_hunks_store); + o->diff_hunks_store = NULL; +} + +/* + * Fill *out with the hunk record at offset in the data chunk, and return + * 1 if the record is in bounds, 0 otherwise. The read path does not + * re-verify the checksum, and a valid checksum would not bound the count + * anyway, so a read must call this and use *out only when it returns + * non-zero. + * + * A record is a be32 hunk count followed by that many DIFF_HUNKS_HUNK_SIZE + * hunks. "remaining" tracks the bytes from offset to the end of the data + * chunk: it must hold the count, and after the count is consumed it must + * hold every hunk. The bounds are written as subtraction and division + * (never addition or multiplication) so a crafted offset or count cannot + * overflow them. + */ +static int precomputed_entry_at(const struct diff_hunks_store *s, + uint32_t offset, struct precomputed_entry *out) +{ + size_t remaining; + uint32_t num_hunks; + + if (offset >= s->hdat_size) + return 0; + remaining = s->hdat_size - offset; + if (remaining < sizeof(uint32_t)) + return 0; + + num_hunks = get_be32(s->hdat + offset); + remaining -= sizeof(uint32_t); + if (num_hunks > remaining / DIFF_HUNKS_HUNK_SIZE) + return 0; + + out->num_hunks = num_hunks; + out->hunk_data = s->hdat + offset + sizeof(uint32_t); + return 1; +} + +struct lookup_key { + const struct object_id *old_oid; + const struct object_id *new_oid; + int xdl_opts; + unsigned int rawsz; +}; + +/* + * The store's total order over (old_oid, new_oid, xdl_opts), defined + * once so the write-side sort (writer_entry_cmp) and the read-side + * search (store_bsearch_cmp) order the keys identically. + */ +static int cmp_store_index_key(const unsigned char *old_a, const unsigned char *new_a, + uint32_t opts_a, + const unsigned char *old_b, const unsigned char *new_b, + uint32_t opts_b, unsigned int rawsz) +{ + int cmp = memcmp(old_a, old_b, rawsz); + if (!cmp) + cmp = memcmp(new_a, new_b, rawsz); + if (!cmp) + cmp = (opts_a > opts_b) - (opts_a < opts_b); + return cmp; +} + +static int store_bsearch_cmp(const void *key, const void *entry_ptr) +{ + const struct lookup_key *k = key; + const unsigned char *old_hash, *new_hash; + uint32_t xdl_opts; + + decode_store_index_key(entry_ptr, k->rawsz, &old_hash, &new_hash, + &xdl_opts); + return cmp_store_index_key(k->old_oid->hash, k->new_oid->hash, + (uint32_t)k->xdl_opts, + old_hash, new_hash, xdl_opts, k->rawsz); +} + +static int store_get_one(struct diff_hunks_store *s, const struct lookup_key *key, + struct precomputed_entry *out) +{ + size_t entry_size = store_index_entry_size(s->hash_algo); + const unsigned char *found; + + found = bsearch(key, s->index, s->num_entries, entry_size, + store_bsearch_cmp); + if (!found) + return 0; + return precomputed_entry_at(s, + index_entry_hdat_offset(found, store_index_key_size(s->hash_algo)), + out); +} + +static int diff_hunks_store_get(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + struct precomputed_entry *out) +{ + struct lookup_key key; + + if (!s) + return 0; + /* The null OID names no blob and cannot key an entry. */ + if (is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + + key.old_oid = old_oid; + key.new_oid = new_oid; + key.xdl_opts = xdl_opts; + key.rawsz = s->hash_algo->rawsz; + + return store_get_one(s, &key, out); +} + +/* + * A recorded hunk sequence must satisfy the seam's well-formedness + * rule (diff_provider_hunks_check()) before it may be replayed: + * coordinates decode from be32 into long, which is 32-bit on some + * platforms, so a crafted value can decode negative or out of order. + * An entry that fails reads as a miss, so the caller recomputes. + */ +static int replayable_hunks(const struct precomputed_entry *e) +{ + struct diff_provider_hunks_check c = { 0 }; + uint32_t i; + + for (i = 0; i < e->num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(e, i, &h); + if (diff_provider_hunks_check(&c, h.old_start, h.old_count, + h.new_start, h.new_count)) + return 0; + } + return 1; +} + +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data) +{ + struct precomputed_entry e; + uint32_t i; + + if (!diff_hunks_store_get(s, old_oid, new_oid, xdl_opts, &e) || + !replayable_hunks(&e)) + return 0; + for (i = 0; i < e.num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(&e, i, &h); + hunk_func(h.old_start, h.old_count, + h.new_start, h.new_count, cb_data); + } + return 1; +} + +/* Validate one store file. Returns 0 if valid or absent, -1 if corrupt. */ +static int verify_store_at(struct repository *r, const char *fname) +{ + struct diff_hunks_store *s; + size_t entry_size; + uint32_t i; + int ret = 0; + + if (access(fname, F_OK)) + return 0; /* absent is valid */ + s = load_store_at(r->hash_algo, fname); + if (!s) + return error(_("diff-hunks store failed to load (corrupt " + "header or hash mismatch): %s"), fname); + if (!hashfile_checksum_valid(r->hash_algo, s->data, s->data_len)) { + error(_("diff-hunks store has incorrect checksum and is " + "likely corrupt: %s"), fname); + free_store(s); + return -1; + } + + entry_size = store_index_entry_size(s->hash_algo); + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + size_t keysz = store_index_key_size(s->hash_algo); + uint32_t offset = index_entry_hdat_offset(ep, keysz); + struct precomputed_entry pe; + + /* + * Keyed by (old_oid, new_oid, xdl_opts, context), increasing. + * memcmp matches cmp_store_index_key's integer comparison of + * xdl_opts and context because both are non-negative, so their + * big-endian bytes order the same as their values. + */ + if (i > 0 && memcmp(ep - entry_size, ep, keysz) >= 0) { + error(_("diff-hunks entry %u not in sorted order"), i); + ret = -1; + } + if (!precomputed_entry_at(s, offset, &pe)) { + error(_("diff-hunks entry %u has out-of-bounds hunk " + "data"), i); + ret = -1; + } + } + + free_store(s); + return ret; +} + +int diff_hunks_verify(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (verify_store_at(r, fname)) + ret = -1; + free(fname); + return ret; +} + +int diff_hunks_clear(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (unlink(fname) && errno != ENOENT) + ret = error_errno(_("unable to remove %s"), fname); + free(fname); + return ret; +} + +struct writer_entry { + struct object_id old_oid; + struct object_id new_oid; + int xdl_opts; + uint32_t hdat_offset; +}; + +struct diff_hunks_writer { + struct repository *r; + struct writer_entry *entries; + size_t nr, alloc; + size_t seed_nr; /* nr after seeding; finish skips a no-op flush */ + struct strbuf hdat; + struct hashmap dedup; /* hunk block content -> offset in hdat */ +}; + +/* A record of one distinct hunk block already present in hdat. */ +struct dedup_entry { + struct hashmap_entry ent; + uint32_t offset; + uint32_t len; +}; + +static int dedup_cmp(const void *cmp_data, + const struct hashmap_entry *a, + const struct hashmap_entry *b, + const void *keydata UNUSED) +{ + const struct diff_hunks_writer *writer = cmp_data; + const struct dedup_entry *ea = container_of(a, const struct dedup_entry, ent); + const struct dedup_entry *eb = container_of(b, const struct dedup_entry, ent); + + if (ea->len != eb->len) + return 1; + return memcmp(writer->hdat.buf + ea->offset, + writer->hdat.buf + eb->offset, ea->len); +} + +static struct diff_hunks_writer *diff_hunks_writer_new(struct repository *r) +{ + struct diff_hunks_writer *w; + + CALLOC_ARRAY(w, 1); + w->r = r; + strbuf_init(&w->hdat, 0); + hashmap_init(&w->dedup, dedup_cmp, w, 0); + return w; +} + +static void strbuf_put_be32(struct strbuf *sb, uint32_t val) +{ + unsigned char buf[4]; + put_be32(buf, val); + strbuf_add(sb, buf, 4); +} + +/* + * The hunk block just appended at `start` is deduplicated: if an + * identical block is already in hdat, this copy is dropped and the + * earlier offset returned; otherwise it is kept and remembered. + * Distinct keys that diff to the same hunks then share one block. + */ +static uint32_t intern_block(struct diff_hunks_writer *w, size_t start) +{ + size_t len = w->hdat.len - start; + struct dedup_entry key, *found, *added; + + hashmap_entry_init(&key.ent, memhash(w->hdat.buf + start, len)); + key.offset = (uint32_t)start; + key.len = (uint32_t)len; + + found = hashmap_get_entry(&w->dedup, &key, ent, NULL); + if (found) { + strbuf_setlen(&w->hdat, start); + return found->offset; + } + + added = xmalloc(sizeof(*added)); + hashmap_entry_init(&added->ent, key.ent.hash); + added->offset = key.offset; + added->len = key.len; + hashmap_add(&w->dedup, &added->ent); + return key.offset; +} + +void diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks) +{ + struct writer_entry *e; + size_t i, block_start; + + /* + * The block appended for this entry is sizeof(uint32_t) + + * nr_hunks * DIFF_HUNKS_HUNK_SIZE bytes. Bound nr_hunks so that + * length fits the uint32_t the dedup index records (and so the + * count itself fits the uint32_t written to the store). + */ + if (!nr_hunks || + nr_hunks > (UINT32_MAX - sizeof(uint32_t)) / DIFF_HUNKS_HUNK_SIZE || + is_null_oid(old_oid) || is_null_oid(new_oid)) + return; + if (w->hdat.len > UINT32_MAX) + return; + /* + * Coordinates are stored as 32-bit values; a result that cannot + * round-trip is dropped rather than silently truncated. + */ + for (i = 0; i < nr_hunks; i++) + if ((uintmax_t)hunks[i].old_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].old_count > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_count > (uintmax_t)INT32_MAX) + return; + + ALLOC_GROW(w->entries, w->nr + 1, w->alloc); + e = &w->entries[w->nr++]; + oidcpy(&e->old_oid, old_oid); + oidcpy(&e->new_oid, new_oid); + e->xdl_opts = xdl_opts; + + block_start = w->hdat.len; + strbuf_put_be32(&w->hdat, (uint32_t)nr_hunks); + for (i = 0; i < nr_hunks; i++) { + strbuf_put_be32(&w->hdat, hunks[i].old_start); + strbuf_put_be32(&w->hdat, hunks[i].old_count); + strbuf_put_be32(&w->hdat, hunks[i].new_start); + strbuf_put_be32(&w->hdat, hunks[i].new_count); + } + e->hdat_offset = intern_block(w, block_start); +} + +/* + * Seed the writer with fname's entries so a rewrite preserves them. Returns + * 0 when the file is absent or seeded, and -1 when it is present but fails + * its checksum. A rewrite re-checksums, so a corrupt source must not be + * carried forward: that would launder the corruption into a checksum-valid + * file that verify can no longer catch. This path already reads the whole + * file, so verify it here (the reader stays trust-at-read and fast) and + * discard it on mismatch: a warm then proceeds without it and rewrites + * a clean store from what it computes. + */ +static int diff_hunks_writer_seed(struct diff_hunks_writer *w, const char *fname) +{ + struct diff_hunks_store *s = load_store_at(w->r->hash_algo, fname); + unsigned int rawsz; + size_t entry_size, keysz; + struct precomputed_hunk *hunks = NULL; + size_t hunks_alloc = 0; + uint32_t i; + + if (!s) + return 0; + if (!hashfile_checksum_valid(w->r->hash_algo, s->data, s->data_len)) { + warning(_("diff-hunks store %s failed its checksum; " + "discarding it"), fname); + free_store(s); + return -1; + } + rawsz = s->hash_algo->rawsz; + entry_size = store_index_entry_size(s->hash_algo); + keysz = store_index_key_size(s->hash_algo); + + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + const unsigned char *old_hash, *new_hash; + struct object_id old_oid, new_oid; + uint32_t xdl_opts, j; + struct precomputed_entry pe; + + decode_store_index_key(ep, rawsz, &old_hash, &new_hash, + &xdl_opts); + oidread(&old_oid, old_hash, s->hash_algo); + oidread(&new_oid, new_hash, s->hash_algo); + if (!precomputed_entry_at(s, index_entry_hdat_offset(ep, keysz), &pe)) + continue; + ALLOC_GROW(hunks, pe.num_hunks, hunks_alloc); + for (j = 0; j < pe.num_hunks; j++) + nth_precomputed_hunk(&pe, j, &hunks[j]); + diff_hunks_writer_add(w, &old_oid, &new_oid, (int)xdl_opts, + hunks, pe.num_hunks); + } + free(hunks); + free_store(s); + return 0; +} + +/* + * Writing is off by default. It is enabled per invocation by the + * GIT_DIFF_HUNKS_WRITE environment variable, or persistently by the + * diffHunks.write config, with the environment variable winning when + * set. Only a warming run (a diff or log the repository owner chooses + * to run with writing on) enables it, so ordinary reads never mutate + * the store. + */ +static int diff_hunks_write_enabled(struct repository *r) +{ + const char *env = getenv("GIT_DIFF_HUNKS_WRITE"); + int val; + + if (env) { + /* + * This is a warming opt-in, so an unparseable value must not + * abort an ordinary read command: treat it as disabled. + */ + val = git_parse_maybe_bool(env); + return val < 0 ? 0 : val; + } + if (!repo_config_get_bool(r, "diffhunks.write", &val)) + return val; + return 0; +} + +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r) +{ + struct diff_hunks_writer *w; + char *fname; + + if (!diff_hunks_write_enabled(r)) + return NULL; + /* + * Seed from the existing store so a flush merges with it rather + * than replacing it: a later warm adds newly computed pairs + * without discarding what earlier warms recorded. + */ + w = diff_hunks_writer_new(r); + fname = diff_hunks_store_path(r); + diff_hunks_writer_seed(w, fname); + free(fname); + w->seed_nr = w->nr; + return w; +} + +static int writer_entry_cmp(const void *va, const void *vb, void *ctx) +{ + const struct writer_entry *a = va, *b = vb; + unsigned int rawsz = *(const unsigned int *)ctx; + return cmp_store_index_key(a->old_oid.hash, a->new_oid.hash, + (uint32_t)a->xdl_opts, + b->old_oid.hash, b->new_oid.hash, + (uint32_t)b->xdl_opts, + rawsz); +} + +struct write_ctx { + struct diff_hunks_writer *w; + unsigned int rawsz; +}; + +static int write_index_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + size_t i; + + for (i = 0; i < ctx->w->nr; i++) { + hashwrite(f, ctx->w->entries[i].old_oid.hash, ctx->rawsz); + hashwrite(f, ctx->w->entries[i].new_oid.hash, ctx->rawsz); + hashwrite_be32(f, ctx->w->entries[i].xdl_opts); + hashwrite_be32(f, ctx->w->entries[i].hdat_offset); + } + return 0; +} + +static int write_data_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + hashwrite(f, ctx->w->hdat.buf, ctx->w->hdat.len); + return 0; +} + +/* Sort, dedup, and write the accumulated entries to the file at fname. */ +static int diff_hunks_writer_flush(struct diff_hunks_writer *w, char *fname) +{ + struct lock_file lk = LOCK_INIT; + struct hashfile *f; + struct chunkfile *cf; + unsigned int rawsz = w->r->hash_algo->rawsz; + struct write_ctx ctx = { w, rawsz }; + size_t entry_size; + + QSORT_S(w->entries, w->nr, writer_entry_cmp, &rawsz); + + /* + * The same blob pair recurs across history (reverts, cherry- + * picks); identical keys carry identical hunks, so keep one of + * each. The index must stay duplicate-free for binary search. + */ + if (w->nr > 1) { + size_t kept = 1, i; + for (i = 1; i < w->nr; i++) + if (writer_entry_cmp(&w->entries[kept - 1], + &w->entries[i], &rawsz)) + w->entries[kept++] = w->entries[i]; + w->nr = kept; + } + + if (safe_create_leading_directories(w->r, fname)) { + error(_("unable to create directory for %s"), fname); + return -1; + } + if (hold_lock_file_for_update(&lk, fname, 0) < 0) { + error_errno(_("unable to lock %s"), fname); + return -1; + } + adjust_shared_perm(w->r, get_lock_file_path(&lk)); + f = hashfd(w->r->hash_algo, get_lock_file_fd(&lk), + get_lock_file_path(&lk)); + + entry_size = store_index_entry_size(w->r->hash_algo); + cf = init_chunkfile(f); + add_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, w->nr * entry_size, + write_index_chunk); + add_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, w->hdat.len, write_data_chunk); + + hashwrite_be32(f, DIFF_HUNKS_SIGNATURE); + hashwrite_u8(f, DIFF_HUNKS_VERSION); + hashwrite_u8(f, oid_version(w->r->hash_algo)); + hashwrite_u8(f, get_num_chunks(cf)); + hashwrite_u8(f, 0); /* reserved */ + + write_chunkfile(cf, &ctx); + free_chunkfile(cf); + + /* + * fsync per the user's configuration (like commit-graph and the + * multi-pack-index), then commit atomically. Readers trust the + * committed file rather than re-checksumming it; diff_hunks_verify() + * checks the checksum separately. + */ + finalize_hashfile(f, NULL, FSYNC_COMPONENT_DIFF_HUNKS, + CSUM_HASH_IN_STREAM); + if (commit_lock_file(&lk)) { + error_errno(_("unable to write %s"), fname); + return -1; + } + return 0; +} + +static void diff_hunks_writer_free(struct diff_hunks_writer *w) +{ + if (!w) + return; + hashmap_clear_and_free(&w->dedup, struct dedup_entry, ent); + free(w->entries); + strbuf_release(&w->hdat); + free(w); +} + +void diff_hunks_writer_finish(struct diff_hunks_writer *w) +{ + if (!w) + return; + /* Skip the flush when the warm recorded nothing beyond its seed. */ + if (w->nr != w->seed_nr) { + char *fname = diff_hunks_store_path(w->r); + diff_hunks_writer_flush(w, fname); + free(fname); + } + diff_hunks_writer_free(w); +} diff --git a/diff-hunks.h b/diff-hunks.h new file mode 100644 index 00000000000000..6bf39df38878fc --- /dev/null +++ b/diff-hunks.h @@ -0,0 +1,113 @@ +#ifndef DIFF_HUNKS_H +#define DIFF_HUNKS_H + +#include "hash.h" +#include "xdiff-interface.h" /* xdl_emit_hunk_consume_func_t */ + +struct object_id; +struct repository; +struct object_database; + +/* + * A persistent store of precomputed diff hunk coordinates, at + * .git/objects/diff-hunks. Entries are keyed by the two blobs diffed + * and the xdl_opts they were diffed under, so a cached result is valid + * in any context that key recurs in, independent of path. The xdl_opts + * key component mirrors the (always non-negative) diff_options field it + * projects from, and is serialized and compared as a 4-byte big-endian + * integer. + * + * The hunks a pair produces are not unique. They vary with the xdiff + * algorithm and ignore flags (xdl_opts, part of the key), and with + * whether the diff was trimmed: a zero-context diff runs + * trim_common_tail, which can pick a different but equally valid set of + * hunks than an untrimmed diff. The store holds one entry per key, so a + * pair is recorded only when its trimmed and untrimmed diffs are + * identical (the recording caller checks); such an entry serves a + * consumer at any context. The rare pair where the two diffs differ is + * never recorded and is always computed. + * + * The store is a cache: ordinary commands read it and fall back to + * computing the diff when it is absent, stale, or corrupt. It is filled + * as a side effect of diff and log runs, but only when writing is + * enabled; writing is off by default, so an ordinary command reads the + * store without recording into it. + */ + +/* + * A hunk's coordinates. The type is long to match the xdiff emit + * callback; the values are a diff's line numbers and counts, always + * within the int32 range the on-disk format stores (see + * diff_hunks_writer_add()). + */ +struct precomputed_hunk { + long old_start; + long old_count; + long new_start; + long new_count; +}; + +/* + * The repository's store, loaded once on first use and cached on the + * object database. Returns NULL when reading is disabled + * (core.diffHunks=false), the store is absent, or its object hash + * differs. The lookup functions below accept a NULL store and treat it + * as empty (every lookup misses), so callers need not check for NULL. + * The object database owns the store; callers must not free it. + */ +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r); + +/* Free the repository's cached store, at object-database teardown. */ +void close_diff_hunks_store(struct object_database *o); + +/* + * Replay the recorded hunks of an (old blob, new blob) pair diffed + * under xdl_opts through hunk_func. The sequence is validated before + * any callback runs: on a hit (return 1) every hunk is emitted, on a + * miss (return 0: absent pair, xdl_opts mismatch, or an entry that + * fails validation) nothing is emitted, so a caller may accumulate + * directly into its result. + */ +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data); + +/* + * A warming run's writer: it accumulates the hunks it computes in memory + * and flushes them to the store in one pass at finish. + */ +struct diff_hunks_writer; + +/* + * Return a writer for a warming run, or NULL when writing is disabled + * (the default). diff_hunks_writer_add() tolerates a NULL writer, so a + * caller may attach the result unconditionally. Pair with + * diff_hunks_writer_finish(). + */ +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r); + +/* + * Record a blob pair's hunks as computed under xdl_opts; a later lookup + * with a matching key is served these hunks. The caller must have + * checked that the pair's trimmed and untrimmed diffs are identical + * (see the top of this file), so the entry answers at any context. + * NULL-safe. + */ +void diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks); + +/* Flush the accumulated entries to the store and free the writer. NULL-safe. */ +void diff_hunks_writer_finish(struct diff_hunks_writer *w); + +/* Remove the store file. Returns 0 (incl. absent) or -1. */ +int diff_hunks_clear(struct repository *r); +/* Validate the store. Returns 0 if valid/absent, -1 if corrupt. */ +int diff_hunks_verify(struct repository *r); + +#endif /* DIFF_HUNKS_H */ diff --git a/diff-process.c b/diff-process.c new file mode 100644 index 00000000000000..84133eb90b8df2 --- /dev/null +++ b/diff-process.c @@ -0,0 +1,665 @@ +/* + * Diff process backend: communicates with a long-running external + * tool via the pkt-line protocol to obtain custom line-matching + * results. The tool controls which lines are marked as changed + * while the display shows the file content (after any textconv + * transformation, if configured). + * + * Protocol: pkt-line over stdin/stdout, following the pattern of + * the long-running filter process protocol (see convert.c). + * + * Handshake: + * git> git-diff-client / version=1 / flush + * tool< git-diff-server / version=1 / flush + * git> capability=hunks / flush + * tool< capability=hunks / flush + * + * Per-file: + * git> command=hunks / pathname= / [old-oid=] / [new-oid=] / flush + * git> / flush + * git> / flush + * tool< hunk + * tool< ... / flush + * tool< status=success / flush + * + * When the tool returns no hunks with status=success, it considers + * the files equivalent. Git will skip the diff for that file. + * + * A tool that also negotiates the "hunks-by-oid" capability may be + * asked without content, when both sides are stored blobs: + * git> command=hunks-by-oid / pathname= / old-oid= / new-oid= / flush + * tool< hunk ... / flush + * tool< status=success / flush + * No content sections follow such a request. The tool may instead + * answer status=need-content, and Git re-asks with a full + * command=hunks request. Because Git holds no content for this + * exchange, the answer is used as sent: the hunks are not re-run + * through xdiff's compaction, and zero hunks assert equivalence + * outright (a tool that cannot rule out a trailing-newline-only + * difference from its cache must answer need-content). + */ + +#include "git-compat-util.h" +#include "diff-process.h" +#include "diff.h" +#include "diff-provider.h" +#include "gettext.h" +#include "hex.h" +#include "repository.h" +#include "sigchain.h" +#include "userdiff.h" +#include "sub-process.h" +#include "pkt-line.h" +#include "strbuf.h" +#include "xdiff/xdiff.h" + +#define CAP_HUNKS (1u << 0) +#define CAP_OID_HUNKS (1u << 1) + +struct diff_subprocess { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_diff_process_fn(struct subprocess_entry *subprocess) +{ + static int versions[] = { 1, 0 }; + static struct subprocess_capability capabilities[] = { + { "hunks", CAP_HUNKS }, + { "hunks-by-oid", CAP_OID_HUNKS }, + { NULL, 0 } + }; + struct diff_subprocess *entry = + container_of(subprocess, struct diff_subprocess, subprocess); + + return subprocess_handshake(subprocess, "git-diff", + versions, NULL, + capabilities, + &entry->supported_capabilities); +} + +static struct diff_subprocess *get_or_launch_process( + struct userdiff_driver *drv) +{ + struct diff_subprocess *entry; + + if (drv->diff_subprocess) + return drv->diff_subprocess; + + entry = xcalloc(1, sizeof(*entry)); + if (subprocess_start_command(&entry->subprocess, drv->process, + start_diff_process_fn)) { + free(entry); + drv->diff_process_failed = 1; + return NULL; + } + + drv->diff_subprocess = entry; + return entry; +} + +static int send_file_content(int fd, const char *buf, long size) +{ + int ret = 0; + + if (size < 0) + return -1; + if (size > 0) + ret = write_packetized_from_buf_no_flush(buf, size, fd); + if (ret) + return ret; + return packet_flush_gently(fd); +} + +/* + * A hunk in the diff process's presentation coordinates: the line + * numbering it reports over the protocol. Kept distinct from struct + * xdl_hunk (xdiff's coordinates) so that only translated hunks ever + * reach the diff algorithm; diff_process_hunk_to_xdl() is the single + * crossing point. + */ +struct diff_process_hunk { + long old_start, old_count; + long new_start, new_count; +}; + +/* + * Parse one non-negative decimal field of a hunk line into *out and + * advance *line past it. Fields must be plain decimal with no leading + * whitespace or sign (isdigit() takes an unsigned char to stay defined + * for high-bit bytes). The first three fields are followed by a single + * space; the last (is_last) is followed by end-of-string or a space. + * Trailing space-separated tokens after the last field are allowed and + * ignored, so a future protocol version can append fields (e.g. a + * "moved" marker) without older tools rejecting the line -- mirroring + * the request-side rule that tools ignore unknown keys. + */ +static int parse_hunk_field(const char **line, long *out, int is_last) +{ + const char *p = *line; + char *end; + + if (!isdigit((unsigned char)*p)) + return -1; + errno = 0; + *out = strtol(p, &end, 10); + if (errno || end == p) + return -1; + if (is_last) { + if (*end != '\0' && *end != ' ') + return -1; + } else { + if (*end != ' ') + return -1; + end++; + } + *line = end; + return 0; +} + +static int parse_hunk_line(const char *line, + struct diff_process_hunk *presented) +{ + /* Format: "hunk " */ + if (!skip_prefix(line, "hunk ", &line)) + return -1; + if (parse_hunk_field(&line, &presented->old_start, 0) || + parse_hunk_field(&line, &presented->old_count, 0) || + parse_hunk_field(&line, &presented->new_start, 0) || + parse_hunk_field(&line, &presented->new_count, 1)) + return -1; + return 0; +} + +/* + * Translate a hunk from the diff process's presentation coordinates + * into xdiff's. + * + * Protocol starts are already 1-based positions (the line a change + * sits before), the same numbering xdiff uses, so the only adjustment + * is for an empty file side: "git diff" addresses it with a start of 0 + * and a count of 0 (e.g. "0 0 1 5" adds five lines to an empty old + * side), and since xdiff uses start-1 as an array index that 0 becomes + * 1 here. This is NOT the full inverse of xdl_emit_hunk_hdr() + * (xdiff/xutils.c): that emitter shifts a count-0 range to start-1 for + * the displayed "@@" header, but the protocol keeps the unshifted + * 1-based position for a mid-file insert or delete. This is the single + * point where presentation coordinates become xdiff coordinates, so + * xdl_populate_hunks_from_external() may assume 1-based starts. + * + * Returns -1 for a start of 0 paired with a nonzero count, which names + * no line in either coordinate system. (parse_hunk_line() already + * guarantees non-negative starts and counts.) + */ +static int diff_process_hunk_to_xdl(const struct diff_process_hunk *presented, + struct xdl_hunk *xdl) +{ + long old_start = presented->old_start; + long new_start = presented->new_start; + + if ((!old_start && presented->old_count) || + (!new_start && presented->new_count)) + return -1; + if (!old_start) + old_start = 1; + if (!new_start) + new_start = 1; + + xdl->old_start = old_start; + xdl->old_count = presented->old_count; + xdl->new_start = new_start; + xdl->new_count = presented->new_count; + return 0; +} + +static enum diff_process_result get_hunks( + struct userdiff_driver *drv, + const char *path, + const char *old_buf, long old_size, + const char *new_buf, long new_size, + const struct object_id *oid_a, + const struct object_id *oid_b, + struct xdl_hunk **hunks_out, + size_t *nr_hunks_out) +{ + struct diff_subprocess *backend; + struct child_process *process; + int fd_in, fd_out; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0; + size_t max_hunks; + int len; + char *line; + + backend = get_or_launch_process(drv); + if (!backend) + return DIFF_PROCESS_ERROR; + + if (!(backend->supported_capabilities & CAP_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&backend->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + /* Send request */ + if (packet_write_fmt_gently(fd_in, "command=hunks\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path)) + goto comm_error; + /* + * old-oid/new-oid let the tool key a cache on the blob pair. A + * side is sent only when its content is the raw blob (the caller + * passes NULL otherwise, e.g. for textconv'd content), so an oid + * that is present always names the bytes the tool receives. + */ + if (oid_a && + packet_write_fmt_gently(fd_in, "old-oid=%s\n", oid_to_hex(oid_a))) + goto comm_error; + if (oid_b && + packet_write_fmt_gently(fd_in, "new-oid=%s\n", oid_to_hex(oid_b))) + goto comm_error; + if (packet_flush_gently(fd_in)) + goto comm_error; + + /* Send old file content */ + if (send_file_content(fd_in, old_buf, old_size)) + goto comm_error; + + /* Send new file content */ + if (send_file_content(fd_in, new_buf, new_size)) + goto comm_error; + + /* + * Hunks are non-overlapping and each useful hunk covers at least + * one line, so a valid response cannot contain more hunks than the + * two files have lines, which is bounded by their byte sizes. Cap + * the accumulation accordingly so a misbehaving tool that floods + * hunk lines cannot drive unbounded memory growth before validation. + */ + max_hunks = (size_t)old_size + (size_t)new_size + 1; + + /* Read hunks until flush packet */ + while ((len = packet_read_line_gently(fd_out, NULL, &line)) >= 0 && + line) { + if (parse_hunk_line(line, &presented) < 0) + goto comm_error; + if (diff_process_hunk_to_xdl(&presented, &hunk) < 0) + goto comm_error; + if (nr_hunks >= max_hunks) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'"), drv->process, path); + goto comm_error; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + if (len < 0) + goto comm_error; + + /* Read status */ + if (subprocess_read_status(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + *hunks_out = hunks; + *nr_hunks_out = nr_hunks; + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_OK; + } + + if (!strcmp(status.buf, "abort")) { + /* + * The tool voluntarily withdrew: stop sending requests + * but do not warn (this is not a failure). + */ + backend->supported_capabilities &= ~CAP_HUNKS; + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_SKIP; + } + + /* status=error or unknown status */ + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; + +comm_error: + /* + * Communication failure (broken pipe, malformed response). + * Tear down the process and mark as failed so we do not + * retry on every subsequent file. + */ + drv->diff_process_failed = 1; + drv->diff_subprocess = NULL; + subprocess_stop_command(&backend->subprocess); + free(backend); + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} + +/* + * Whether exactly one of the two blobs ends in a newline. A change + * that only adds or removes the trailing newline is not expressible as + * line hunks, so a tool comparing lines reports the files as equal. + */ +static int eof_newline_differs(const mmfile_t *a, const mmfile_t *b) +{ + int a_nl = a->size > 0 && a->ptr[a->size - 1] == '\n'; + int b_nl = b->size > 0 && b->ptr[b->size - 1] == '\n'; + return a_nl != b_nl; +} + +/* + * Number of lines in a blob, matching xdiff's record count: one per + * newline, plus one more if the last line has no trailing newline. + */ +static long count_lines(const char *buf, long size) +{ + long lines = 0, i; + + for (i = 0; i < size; i++) + if (buf[i] == '\n') + lines++; + if (size > 0 && buf[size - 1] != '\n') + lines++; + return lines; +} + +/* + * Validate the tool's hunks (already in xdiff coordinates) before they + * bypass the diff algorithm. The content-independent rules (in-order, + * non-overlapping, lockstep-aligned, int32-bounded coordinates) are the + * seam's shared provider rule, diff_provider_hunks_check(); this + * function adds the two checks that need the blobs' line counts (a hunk + * past the end of a file, the run after the last hunk) and the + * per-rule diagnostics naming the tool. This is the git layer's job so + * xdiff stays diagnostic-free; on a bad response we warn and the caller + * falls back to the builtin diff. Returns 0 if valid, -1 (after + * warning) otherwise. + * + * An oid-only answer arrives without content: pass negative line + * counts, and the two content-dependent checks do not apply. + */ +static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, + long old_lines, long new_lines, + const char *process, const char *path) +{ + struct diff_provider_hunks_check c = { 0 }; + size_t i; + + for (i = 0; i < nr; i++) { + const struct xdl_hunk *h = &hunks[i]; + + if (old_lines >= 0 && + (h->old_count > old_lines - h->old_start + 1 || + h->new_count > new_lines - h->new_start + 1)) { + warning(_("diff process '%s' returned a hunk past the " + "end of '%s'; using the builtin diff"), + process, path); + return -1; + } + switch (diff_provider_hunks_check(&c, h->old_start, + h->old_count, h->new_start, + h->new_count)) { + case PROVIDER_HUNKS_OK: + break; + case PROVIDER_HUNKS_RANGE: + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + process, path); + return -1; + case PROVIDER_HUNKS_OVERLAP: + warning(_("diff process '%s' returned overlapping hunks " + "for '%s'; using the builtin diff"), + process, path); + return -1; + case PROVIDER_HUNKS_MISALIGNED: + warning(_("diff process '%s' returned hunks that leave " + "'%s' misaligned; using the builtin diff"), + process, path); + return -1; + } + } + if (old_lines >= 0 && + old_lines - c.prev_old_end != new_lines - c.prev_new_end) { + warning(_("diff process '%s' returned hunks that leave '%s' " + "misaligned; using the builtin diff"), + process, path); + return -1; + } + return 0; +} + +struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp) +{ + struct userdiff_driver *drv; + + if (!diffopt || !path) + return NULL; + if (diffopt->flags.no_diff_process || diffopt->ignore_driver_algorithm) + return NULL; + /* + * Whitespace-ignoring, regex-ignore (-I) and anchored options + * change which lines count as different, but the tool is never + * told about them, so its hunks could not honor them. Rather + * than silently override the user's request, fall back to the + * builtin diff, which does honor these flags. Key this off xpp + * (the parameters this diff actually runs with) rather than + * diffopt, so a caller like blame that keeps its flags outside + * diffopt is covered without a separate guard of its own. + */ + if ((xpp->flags & (XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES)) || + xpp->ignore_regex_nr || xpp->anchors_nr) + return NULL; + + drv = userdiff_find_by_path(diffopt->repo->index, path); + if (!drv || !drv->process) + return NULL; + if (drv->diff_process_failed) + return NULL; + return drv; +} + +enum diff_process_result diff_process_fill_hunks( + struct diff_options *diffopt, + const char *path, + const mmfile_t *file_a, + const mmfile_t *file_b, + const struct object_id *oid_a, + const struct object_id *oid_b, + xpparam_t *xpp) +{ + struct userdiff_driver *drv; + struct xdl_hunk *ext_hunks = NULL; + size_t nr = 0; + enum diff_process_result res; + + drv = diff_process_driver(diffopt, path, xpp); + if (!drv) + return DIFF_PROCESS_SKIP; + + res = get_hunks(drv, path, + file_a->ptr, file_a->size, + file_b->ptr, file_b->size, + oid_a, oid_b, + &ext_hunks, &nr); + if (res == DIFF_PROCESS_OK) { + if (!nr) { + free(ext_hunks); + /* + * Zero hunks means the tool considers the line + * content identical, but it cannot express a + * trailing-newline-only change. When that is the + * actual difference, fall back to the builtin diff + * so the "\ No newline at end of file" marker is + * preserved instead of reporting the files equal. + */ + if (eof_newline_differs(file_a, file_b)) + return DIFF_PROCESS_SKIP; + return DIFF_PROCESS_EQUIVALENT; + } + if (validate_external_hunks(ext_hunks, nr, + count_lines(file_a->ptr, file_a->size), + count_lines(file_b->ptr, file_b->size), + drv->process, path) < 0) { + free(ext_hunks); + return DIFF_PROCESS_SKIP; + } + xpp->external_hunks = ext_hunks; + xpp->external_hunks_nr = nr; + return DIFF_PROCESS_OK; + } + if (res == DIFF_PROCESS_ERROR) { + warning(_("diff process '%s' failed for '%s'," + " falling back to builtin diff"), + drv->process, path); + return DIFF_PROCESS_ERROR; + } + return DIFF_PROCESS_SKIP; +} + +/* + * Without content there is no size-derived bound on a response, so cap + * accumulation at a constant instead. A pair with more changed lines + * than this is served by the content request, whose cap follows the + * file sizes. + */ +#define OID_HUNKS_MAX (1 << 20) + +enum diff_process_result diff_process_query_hunks( + struct diff_options *diffopt, + const char *path, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + struct userdiff_driver *drv; + struct diff_subprocess *backend; + struct child_process *process; + int fd_in, fd_out; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0, i; + int len; + char *line; + enum diff_process_result res; + + if (!old_oid || !new_oid) + return DIFF_PROCESS_SKIP; + drv = diff_process_driver(diffopt, path, xpp); + if (!drv) + return DIFF_PROCESS_SKIP; + + backend = get_or_launch_process(drv); + if (!backend) + return DIFF_PROCESS_ERROR; + if ((backend->supported_capabilities & (CAP_HUNKS | CAP_OID_HUNKS)) + != (CAP_HUNKS | CAP_OID_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&backend->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + if (packet_write_fmt_gently(fd_in, "command=hunks-by-oid\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path) || + packet_write_fmt_gently(fd_in, "old-oid=%s\n", oid_to_hex(old_oid)) || + packet_write_fmt_gently(fd_in, "new-oid=%s\n", oid_to_hex(new_oid)) || + packet_flush_gently(fd_in)) + goto comm_error; + + while ((len = packet_read_line_gently(fd_out, NULL, &line)) >= 0 && + line) { + if (parse_hunk_line(line, &presented) < 0) + goto comm_error; + if (diff_process_hunk_to_xdl(&presented, &hunk) < 0) + goto comm_error; + if (nr_hunks >= OID_HUNKS_MAX) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'"), drv->process, path); + goto comm_error; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + if (len < 0) + goto comm_error; + + if (subprocess_read_status(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + if (validate_external_hunks(hunks, nr_hunks, -1, -1, + drv->process, path) < 0) { + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!nr_hunks) { + res = DIFF_PROCESS_EQUIVALENT; + goto out; + } + /* + * Replay in the coordinates a hunk consumer receives from + * xdiff's emission: 0-based starts. The answer is used as + * the tool sent it; with no content in hand it cannot be + * re-run through xdiff's compaction. + */ + for (i = 0; i < nr_hunks; i++) + hunk_cb(hunks[i].old_start - 1, hunks[i].old_count, + hunks[i].new_start - 1, hunks[i].new_count, + cb_data); + res = DIFF_PROCESS_OK; + goto out; + } + if (!strcmp(status.buf, "need-content")) { + /* The tool wants the content request; the caller sends it. */ + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!strcmp(status.buf, "abort")) { + /* + * The tool withdrew from oid-only answers: stop asking, but + * keep consulting it with content. + */ + backend->supported_capabilities &= ~CAP_OID_HUNKS; + res = DIFF_PROCESS_SKIP; + goto out; + } + warning(_("diff process '%s' failed for '%s'," + " falling back to builtin diff"), + drv->process, path); + res = DIFF_PROCESS_ERROR; +out: + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return res; + +comm_error: + drv->diff_process_failed = 1; + drv->diff_subprocess = NULL; + subprocess_stop_command(&backend->subprocess); + free(backend); + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} diff --git a/diff-process.h b/diff-process.h new file mode 100644 index 00000000000000..2aff886bd48183 --- /dev/null +++ b/diff-process.h @@ -0,0 +1,81 @@ +#ifndef DIFF_PROCESS_H +#define DIFF_PROCESS_H + +#include "xdiff/xdiff.h" + +struct diff_options; +struct object_id; +struct userdiff_driver; + +/* + * The driver whose process a consultation for path would ask, or NULL + * when none applies (no driver, process disabled or failed, or xpp + * carries options the tool is never told about). Needs no content, so + * a caller can pick a producer before loading the blobs. + */ +struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp); + +enum diff_process_result { + DIFF_PROCESS_ERROR = -1, /* failed; caller falls back to builtin */ + DIFF_PROCESS_OK = 0, /* hunks populated in xpp */ + DIFF_PROCESS_SKIP, /* process did not apply: use builtin */ + DIFF_PROCESS_EQUIVALENT, /* tool says files are equivalent */ +}; + +/* + * Consult the diff process configured for 'path' and populate + * xpp->external_hunks with the returned hunks. + * + * Handles driver lookup, flag checks (--no-ext-diff, + * --diff-algorithm), subprocess management, and error reporting. + * + * Returns DIFF_PROCESS_OK when hunks are populated in xpp. + * The caller owns xpp->external_hunks and must free() it. + * + * Returns DIFF_PROCESS_EQUIVALENT when the tool returns no hunks and + * the blobs are not a trailing-newline-only change (files are + * considered identical); caller should skip diff/blame. + * Returns DIFF_PROCESS_SKIP when no process applies; caller + * should use the builtin diff algorithm. + * Returns DIFF_PROCESS_ERROR on tool failure (already warned); + * caller should fall back to the builtin diff algorithm. + * + * oid_a/oid_b, when non-NULL, are sent to the tool as old-oid/new-oid + * so it can key a cache on the blob pair. Pass NULL for a side whose + * content is not the raw blob (e.g. textconv'd) or whose object name is + * unknown, so any oid that is sent always names the bytes the tool + * receives. + */ +enum diff_process_result diff_process_fill_hunks( + struct diff_options *diffopt, + const char *path, + const mmfile_t *file_a, + const mmfile_t *file_b, + const struct object_id *oid_a, + const struct object_id *oid_b, + xpparam_t *xpp); + +/* + * Ask the diff process configured for 'path' to answer from the blob + * pair's object ids alone (the "hunks-by-oid" capability): no content + * is loaded or sent. On DIFF_PROCESS_OK the tool's hunks are emitted + * through hunk_cb in 0-based emission coordinates, validated for order, + * overlap, and lockstep alignment first; because Git holds no content, + * the answer is used as the tool sent it, without xdiff's compaction. + * DIFF_PROCESS_EQUIVALENT means the tool asserts the pair equal. + * DIFF_PROCESS_SKIP covers everything that should fall through to a + * content consult: no driver or capability, a missing object id, a + * status=need-content answer, or an invalid response. + */ +enum diff_process_result diff_process_query_hunks( + struct diff_options *diffopt, + const char *path, + const struct object_id *old_oid, + const struct object_id *new_oid, + const xpparam_t *xpp, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +#endif /* DIFF_PROCESS_H */ diff --git a/diff-provider.c b/diff-provider.c new file mode 100644 index 00000000000000..5186d1caa66f63 --- /dev/null +++ b/diff-provider.c @@ -0,0 +1,110 @@ +#include "git-compat-util.h" +#include "diff-provider.h" +#include "diff-hunks.h" +#include "diff-process.h" + +int diff_provider_active(struct repository *r) +{ + return !!repo_diff_hunks_store(r); +} + +int diff_provider_query_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + if (!old_oid || !new_oid) + return 0; + return diff_hunks_replay(repo_diff_hunks_store(r), old_oid, new_oid, + xdl_opts, hunk_cb, cb_data); +} + +enum diff_provider_hunks_error +diff_provider_hunks_check(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count) +{ + if (old_start < 0 || old_count < 0 || + new_start < 0 || new_count < 0 || + old_start > INT32_MAX || old_count > INT32_MAX || + new_start > INT32_MAX || new_count > INT32_MAX) + return PROVIDER_HUNKS_RANGE; + if (old_start < c->prev_old_end || new_start < c->prev_new_end) + return PROVIDER_HUNKS_OVERLAP; + if (old_start - c->prev_old_end != new_start - c->prev_new_end) + return PROVIDER_HUNKS_MISALIGNED; + /* + * With each field bounded to int32 above, these sums cannot + * overflow the int64 running state even where long is 32-bit. + */ + c->prev_old_end = (int64_t)old_start + old_count; + c->prev_new_end = (int64_t)new_start + new_count; + return PROVIDER_HUNKS_OK; +} + +int diff_provider_emit_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + const char *path, + struct diff_options *diffopt, + const xpparam_t *xpp, + hunk_pair_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + xpparam_t xpp_local = *xpp; + xdemitconf_t xecfg = { .hunk_func = hunk_cb }; + xdemitcb_t ecb = { .priv = cb_data }; + mmfile_t old_file, new_file; + int ret; + + /* Only a tool consulted here may supply external hunks. */ + xpp_local.external_hunks = NULL; + xpp_local.external_hunks_nr = 0; + + /* + * A process-capable driver makes the tool the producer for the + * path, so the store (which holds xdiff's answer, one a semantic + * tool may deliberately contradict) is not consulted. -I + * patterns and anchors shape the diff but are outside the + * settings that key the store, so such a request is computed, + * never served. + */ + if (!diff_process_driver(diffopt, path, xpp) && + !xpp->ignore_regex_nr && !xpp->anchors_nr && + diff_provider_query_hunks(r, old_oid, new_oid, xpp->flags, + hunk_cb, cb_data)) + return 1; + + /* + * A tool that negotiated hunks-by-oid answers the identity phase + * itself; only a fall-through loads content. + */ + switch (diff_process_query_hunks(diffopt, path, old_oid, new_oid, + xpp, hunk_cb, cb_data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return 0; + default: + break; + } + + if (fill(fill_data, &old_file, &new_file) < 0) + return -1; + + switch (diff_process_fill_hunks(diffopt, path, &old_file, &new_file, + old_oid, new_oid, &xpp_local)) { + case DIFF_PROCESS_EQUIVALENT: + /* The tool reports the pair equal: there is no hunk to emit. */ + return 0; + default: + break; /* OK: tool hunks now in xpp_local; SKIP/ERROR: builtin */ + } + + ret = xdi_diff(&old_file, &new_file, &xpp_local, &xecfg, &ecb) < 0 ? + -1 : 0; + free(xpp_local.external_hunks); + return ret; +} diff --git a/diff-provider.h b/diff-provider.h new file mode 100644 index 00000000000000..4ea8855df48210 --- /dev/null +++ b/diff-provider.h @@ -0,0 +1,120 @@ +#ifndef DIFF_PROVIDER_H +#define DIFF_PROVIDER_H + +#include "xdiff-interface.h" + +/* + * The seam between naming a pair of file versions to diff and + * computing their changed line ranges. Consumers that operate on + * hunk coordinates route their diff through here. + * + * A hunk provider answers a consumer's request from the pair's blob + * object ids and the settings that determine the diff, before any + * content is loaded; a request no provider answers falls through to + * the consumer's own computation. Two providers implement this + * interface with different authority. The diff-hunks store + * (diff-hunks.h) is in-process and not authoritative: it may only + * reproduce the builtin result, so it never asserts a pair + * equivalent, and it stands aside wherever a tool outranks it. A + * configured diff..process tool (diff-process.h) is + * authoritative for its paths: its answer may deliberately differ + * from the builtin diff, including asserting a pair equivalent. The + * consult order in diff_provider_emit_hunks() is that authority + * resolution. Whichever provider answers, its coordinates pass + * diff_provider_hunks_check() before any consumer sees them. + */ + +struct diff_options; +struct object_id; +struct repository; + +/* + * Nonzero when a hunk provider is available for the repository, for + * consumers that decide up front whether consulting can pay off. + */ +int diff_provider_active(struct repository *r); + +/* + * Consult hunk providers for the changed ranges of the blob pair + * (old_oid, new_oid) diffed under xdl_opts, without loading content. + * On a hit the ranges are emitted through hunk_cb and 1 is returned; + * nothing is emitted before the provider's answer is validated, so a + * consumer may accumulate directly into its result. 0 is a miss: no + * provider answered, and the consumer computes its diff as it would + * without providers. + */ +int diff_provider_query_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +/* + * Incremental well-formedness check for a provider-supplied hunk + * sequence, shared by every provider. Each coordinate must fit int32 + * (a consumer may truncate to int, and a provider may serialize as + * such); hunks must be in order and must not overlap; and the + * unchanged run between hunks must be the same length on both sides, + * or a consumer that walks the two files in lockstep desynchronizes. + * Every rule constrains differences between coordinates, so the check + * applies to 0-based and 1-based sequences alike. + * + * Feed the hunks in order to a zero-initialized struct; the first + * nonzero return names the violated rule, and the whole sequence must + * then be discarded unemitted. + */ +struct diff_provider_hunks_check { + int64_t prev_old_end, prev_new_end; +}; + +enum diff_provider_hunks_error { + PROVIDER_HUNKS_OK = 0, + PROVIDER_HUNKS_RANGE, /* negative or beyond int32 */ + PROVIDER_HUNKS_OVERLAP, /* out of order or overlapping */ + PROVIDER_HUNKS_MISALIGNED, /* unchanged runs differ in length */ +}; + +enum diff_provider_hunks_error +diff_provider_hunks_check(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count); + +/* + * Load the pair's content. Called at most once per request, only + * when the ranges are computed rather than provided. The buffers + * borrow storage owned by the callback's owner. + */ +typedef int (*hunk_pair_fill_fn)(void *data, mmfile_t *old_file, + mmfile_t *new_file); + +/* + * Emit the exact changed ranges (context 0) for the pair (old_oid, + * new_oid) at path to hunk_cb. + * + * The producer is picked before content is loaded. A path whose + * driver has a diff process makes the tool the producer: the store's + * entries hold xdiff's answer, which a semantic tool may deliberately + * contradict, so the identity phase is skipped, content is loaded, + * and the tool is consulted; its hunks feed xdiff's emission, and a + * tool that reports the pair equivalent emits no hunks at all. + * Otherwise the store is consulted by the object ids and xpp's flags, + * and only a miss loads content and computes. + * + * Pass NULL object ids when the diffed bytes are not those blobs (or + * there are no blobs): the identity phase then misses, and no id is + * sent to a tool. Pass a NULL diffopt or path to consult no tool. + * Returns 1 when the store supplied the ranges, 0 when they were + * emitted any other way, and -1 on failure to load or diff. + */ +int diff_provider_emit_hunks(struct repository *r, + const struct object_id *old_oid, + const struct object_id *new_oid, + const char *path, + struct diff_options *diffopt, + const xpparam_t *xpp, + hunk_pair_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +#endif /* DIFF_PROVIDER_H */ diff --git a/diff.c b/diff.c index 5a584fa1d569e7..5ad295b57866b7 100644 --- a/diff.c +++ b/diff.c @@ -16,6 +16,8 @@ #include "revision.h" #include "quote.h" #include "diff.h" +#include "diff-hunks.h" +#include "diff-provider.h" #include "diffcore.h" #include "delta.h" #include "hex.h" @@ -25,6 +27,7 @@ #include "utf8.h" #include "odb.h" #include "userdiff.h" +#include "diff-process.h" #include "submodule.h" #include "hashmap.h" #include "mem-pool.h" @@ -34,6 +37,7 @@ #include "tmp-objdir.h" #include "graph.h" #include "oid-array.h" +#include "trace2.h" #include "packfile.h" #include "pager.h" #include "parse-options.h" @@ -610,28 +614,74 @@ struct emit_callback { }; /* - * State for the line-range callback wrappers that sit between - * xdi_diff_outf() and fn_out_consume(). xdiff produces a normal, - * unfiltered diff; the wrappers intercept each hunk header and line, - * track post-image position, and forward only lines that fall within - * the requested ranges. Contiguous in-range lines are collected into - * range hunks and flushed with a synthetic @@ header so that - * fn_out_consume() sees well-formed unified-diff fragments. + * Line-range filter: scopes "git log -L" output to the tracked ranges. * - * Removal lines ('-') cannot be classified by post-image position, so - * they are buffered in pending_rm until the next '+' or ' ' line - * reveals whether they precede an in-range line (flush into range hunk) or - * an out-of-range line (discard). + * It sits between xdi_diff_outf() and an output callback (fn_out_consume, + * diffstat_consume, checkdiff_consume). xdiff produces a normal diff; the + * filter forwards only the lines inside the requested ranges, collecting + * contiguous in-range lines into a "range hunk" emitted with a synthetic + * @@ header so the callback sees well-formed unified-diff fragments. + * + * A diff describes the change from a pre-image to a post-image. Each + * line is context (' ', in both), a removal ('-', pre-image only), or + * an addition ('+', post-image only). -L tracks ranges in the + * post-image, so a line is in range by its post-image position. + * + * Two 1-based cursors track the next line in each image, named as in + * struct emit_callback and seeded from the xdiff hunk header: + * + * lno_in_postimage advances on '+' and ' ' (lines in the post-image) + * lno_in_preimage advances on '-' and ' ' (lines in the pre-image) + * + * Ranges are 0-based half-open [start, end), so a line is tested at the + * 0-based index idx_in_postimage = lno_in_postimage - 1. + * + * A '-' is not present in the post-image, so it has no post-image line + * number of its own. Since it does not advance lno_in_postimage, it is + * classified at the idx_in_postimage that the following '+'/' ' will + * occupy. xdiff emits a change's removals before its additions, so that + * index is already known when the '-' arrives. + * + * The synthetic "@@ - + @@" header has two sides, old (the + * pre-image) and new (the post-image), matching the xdiff_emit_hunk_fn + * callback; the hunk.old_begin / hunk.new_begin fields below hold those + * begins, and flush_range_hunk() derives the counts from the buffered + * lines. + * + * Example, tracking post-image line 2 (range [1, 2)) of: + * + * pre-image post-image + * 1 a 1 a + * 2 b 2 X (b -> X) + * 3 c 3 c + * + * classify each line by idx_in_postimage. The pre and post columns + * are each cursor's value while that line is classified, i.e. before + * the line advances them (pre = lno_in_preimage, + * post = lno_in_postimage, idx = idx_in_postimage): + * ' a' pre 1 post 1 idx 0 -> before start, skip + * '-b' pre 2 post 2 idx 1 -> keep (removal) + * '+X' pre 3 post 2 idx 1 -> keep (addition) + * ' c' pre 3 post 3 idx 2 -> past end, flush + * + * -b and +X share idx = 1 because -b did not advance lno_in_postimage; + * both land in the range hunk, flushed when ' c' crosses the range end. */ -struct line_range_callback { +struct line_range_filter { xdiff_emit_line_fn orig_line_fn; + /* + * Optional; consumers that report file line numbers (e.g. + * checkdiff) need the synthetic hunk header to set their + * post-image position before in-range lines are replayed. + */ + xdiff_emit_hunk_fn orig_hunk_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ /* Post/pre-image line counters (1-based, set from hunk headers) */ - long lno_post; - long lno_pre; + long lno_in_postimage; + long lno_in_preimage; /* * Function name from most recent xdiff hunk header; @@ -640,17 +690,17 @@ struct line_range_callback { char func[80]; long funclen; - /* Range hunk being accumulated for the current range */ - struct strbuf rhunk; - long rhunk_old_begin, rhunk_old_count; - long rhunk_new_begin, rhunk_new_count; - int rhunk_active; - int rhunk_has_changes; /* any '+' or '-' lines? */ - - /* Removal lines not yet known to be in-range */ - struct strbuf pending_rm; - int pending_rm_count; - long pending_rm_pre_begin; /* pre-image line of first pending */ + /* + * The range hunk being accumulated. At most one is live at a time: + * it is flushed and reset as the cursor leaves each range (and once + * more at end of diff), then reused for the next range. + */ + struct { + struct strbuf lines; /* buffered in-range diff lines */ + long old_begin; + long new_begin; + int active; + } hunk; int ret; /* latched error from orig_line_fn */ }; @@ -2540,26 +2590,60 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_callback *s) +static void line_range_filter_init(struct line_range_filter *filter, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, + void *cb_data) { - strbuf_reset(&s->pending_rm); - s->pending_rm_count = 0; + memset(filter, 0, sizeof(*filter)); + filter->orig_line_fn = line_fn; + filter->orig_cb_data = cb_data; + filter->ranges = ranges; + strbuf_init(&filter->hunk.lines, 0); } -static void flush_rhunk(struct line_range_callback *s) +/* + * Begin a range hunk at the first in-range line. Its position fixes the + * hunk's begins, taken from the two image cursors before they advance: + * new_begin from the post-image, old_begin from the pre-image. The line + * counts are not tracked here; flush_range_hunk() derives them from the + * buffered lines. + */ +static void begin_range_hunk(struct line_range_filter *filter) +{ + filter->hunk.active = 1; + filter->hunk.new_begin = filter->lno_in_postimage; + filter->hunk.old_begin = filter->lno_in_preimage; + strbuf_reset(&filter->hunk.lines); +} + +static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; + long old_count = 0, new_count = 0; + int has_changes = 0; - if (!s->rhunk_active || s->ret) + if (!filter->hunk.active || filter->ret) return; - /* Drain any pending removal lines into the range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + /* + * Derive the hunk's geometry from the buffered lines: a ' ' + * counts on both sides, a '-' on the old side, a '+' on the new. + * A '-' or '+' marks a real change; the "\ No newline at end of + * file" marker (line[0] == '\\') counts on neither side. + */ + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + if (*p == ' ' || *p == '-') + old_count++; + if (*p == ' ' || *p == '+') + new_count++; + if (*p == '-' || *p == '+') + has_changes = 1; + p = eol ? eol + 1 : end; } /* @@ -2568,22 +2652,28 @@ static void flush_rhunk(struct line_range_callback *s) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!s->rhunk_has_changes) { - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + if (!has_changes) { + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); return; } - strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - s->rhunk_old_begin, s->rhunk_old_count, - s->rhunk_new_begin, s->rhunk_new_count); - if (s->funclen > 0) { - strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, s->func, s->funclen); - } - strbuf_addch(&hdr, '\n'); + xdiff_emit_hunk_header(&hdr, filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); + + /* + * Inform a line-numbering consumer of the post-image position + * before replaying lines, mirroring the hunk callback xdiff + * would have issued for a non-scoped diff. + */ + if (filter->orig_hunk_fn) + filter->orig_hunk_fn(filter->orig_cb_data, + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); - s->ret = s->orig_line_fn(s->orig_cb_data, hdr.buf, hdr.len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); /* @@ -2591,135 +2681,159 @@ static void flush_rhunk(struct line_range_callback *s) * The cast discards const because xdiff_emit_line_fn takes * char *, though fn_out_consume does not modify the buffer. */ - p = s->rhunk.buf; - end = p + s->rhunk.len; - while (!s->ret && p < end) { + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (!filter->ret && p < end) { const char *eol = memchr(p, '\n', end - p); unsigned long line_len = eol ? (unsigned long)(eol - p + 1) : (unsigned long)(end - p); - s->ret = s->orig_line_fn(s->orig_cb_data, (char *)p, line_len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, (char *)p, line_len); p += line_len; } - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); } static void line_range_hunk_fn(void *data, - long old_begin, long old_nr UNUSED, - long new_begin, long new_nr UNUSED, + long old_begin, long old_nr, + long new_begin, long new_nr, const char *func, long funclen) { - struct line_range_callback *s = data; + struct line_range_filter *filter = data; /* - * When count > 0, begin is 1-based. When count == 0, begin is - * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of - * that type will arrive, so the value is unused. - * - * Any pending removal lines from the previous xdiff hunk are - * intentionally left in pending_rm: the line callback will - * flush or discard them when the next content line reveals - * whether the removals precede in-range content. + * Seed the per-image line cursors from the hunk header's begins. For + * a side with no lines (count 0), xdiff's callback has already moved + * its begin to the line before the change, so add one back to recover + * the true 1-based start. xdiff_emit_hunk_header() reapplies that -1 + * when the clipped hunk is emitted. */ - s->lno_post = new_begin; - s->lno_pre = old_begin; + filter->lno_in_postimage = new_nr ? new_begin : new_begin + 1; + filter->lno_in_preimage = old_nr ? old_begin : old_begin + 1; if (funclen > 0) { - if (funclen > (long)sizeof(s->func)) - funclen = sizeof(s->func); - memcpy(s->func, func, funclen); + if (funclen > (long)sizeof(filter->func)) + funclen = sizeof(filter->func); + memcpy(filter->func, func, funclen); } - s->funclen = funclen; + filter->funclen = funclen; } static int line_range_line_fn(void *priv, char *line, unsigned long len) { - struct line_range_callback *s = priv; - const struct range *cur; - long lno_0, cur_pre; + struct line_range_filter *filter = priv; + long idx_in_postimage; + int in_range; - if (s->ret) - return s->ret; - - if (line[0] == '-') { - if (!s->pending_rm_count) - s->pending_rm_pre_begin = s->lno_pre; - s->lno_pre++; - strbuf_add(&s->pending_rm, line, len); - s->pending_rm_count++; - return s->ret; - } + if (filter->ret) + return filter->ret; if (line[0] == '\\') { - if (s->pending_rm_count) - strbuf_add(&s->pending_rm, line, len); - else if (s->rhunk_active) - strbuf_add(&s->rhunk, line, len); - /* otherwise outside tracked range; drop silently */ - return s->ret; + if (filter->hunk.active) + strbuf_add(&filter->hunk.lines, line, len); + return filter->ret; } - if (line[0] != '+' && line[0] != ' ') + if (line[0] != '+' && line[0] != ' ' && line[0] != '-') BUG("unexpected diff line type '%c'", line[0]); - lno_0 = s->lno_post - 1; - cur_pre = s->lno_pre; /* save before advancing for context lines */ - s->lno_post++; - if (line[0] == ' ') - s->lno_pre++; + /* + * idx_in_postimage is this line's 0-based post-image index (see the model on + * struct line_range_filter). The cursors are advanced only after + * the line is classified, so a '-' is tested at the same idx_in_postimage as + * the '+'/' ' that follows it. + */ + idx_in_postimage = filter->lno_in_postimage - 1; - /* Advance past ranges we've passed */ - while (s->cur_range < s->ranges->nr && - lno_0 >= s->ranges->ranges[s->cur_range].end) { - if (s->rhunk_active) - flush_rhunk(s); - discard_pending_rm(s); - s->cur_range++; + /* Retire ranges we have passed, flushing the one we leave. */ + while (filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { + if (filter->hunk.active) + flush_range_hunk(filter); + filter->cur_range++; } - /* Past all ranges */ - if (s->cur_range >= s->ranges->nr) { - discard_pending_rm(s); - return s->ret; - } + in_range = filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].start && + idx_in_postimage < filter->ranges->ranges[filter->cur_range].end; - cur = &s->ranges->ranges[s->cur_range]; + if (in_range) { + if (!filter->hunk.active) + begin_range_hunk(filter); - /* Before current range */ - if (lno_0 < cur->start) { - discard_pending_rm(s); - return s->ret; + strbuf_add(&filter->hunk.lines, line, len); } - /* In range so start a new range hunk if needed */ - if (!s->rhunk_active) { - s->rhunk_active = 1; - s->rhunk_has_changes = 0; - s->rhunk_new_begin = lno_0 + 1; - s->rhunk_old_begin = s->pending_rm_count - ? s->pending_rm_pre_begin : cur_pre; - s->rhunk_old_count = 0; - s->rhunk_new_count = 0; - strbuf_reset(&s->rhunk); + /* + * Advance each image's cursor: a line present in that image (see + * the model) consumes one of its line numbers. + */ + if (line[0] != '-') + filter->lno_in_postimage++; + if (line[0] != '+') + filter->lno_in_preimage++; + + return filter->ret; +} + +/* + * Run an xdiff pass through an initialized line-range filter, flush the + * final range hunk, and release the filter. Inflates ctxlen to the largest + * range span first, so that every change within a single range lands in one + * xdiff hunk and the inter-change context is emitted; the filter then clips + * back to range boundaries. The optimal ctxlen depends on where changes fall + * within the range, which is only known after xdiff runs, so the max span is + * the upper bound that guarantees correctness in a single pass. Every + * consumer (patch, diffstat, check) relies on one xdiff hunk per range, so + * this lives here rather than at each call site. Also clears + * XDL_EMIT_NO_HUNK_HDR: the filter seeds its per-image position from the hunk + * headers, so a consumer that otherwise suppresses them (diffstat) still gets + * them here. Returns non-zero if xdiff or any forwarded callback failed. + */ +static int line_range_filter_diff(struct line_range_filter *filter, + mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + const struct range_set *ranges = filter->ranges; + long max_span = 0; + unsigned int i; + int ret; + + for (i = 0; i < ranges->nr; i++) { + long span = ranges->ranges[i].end - ranges->ranges[i].start; + if (span > max_span) + max_span = span; } + if (max_span > xecfg->ctxlen) + xecfg->ctxlen = max_span; - /* Flush pending removals into range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + /* the filter seeds its per-image position from hunk headers */ + xecfg->flags &= ~XDL_EMIT_NO_HUNK_HDR; + + ret = xdi_diff_outf(mf1, mf2, line_range_hunk_fn, + line_range_line_fn, filter, xpp, xecfg); + if (!ret) { + flush_range_hunk(filter); + ret = filter->ret; } + strbuf_release(&filter->hunk.lines); + return ret; +} - strbuf_add(&s->rhunk, line, len); - s->rhunk_new_count++; - if (line[0] == '+') - s->rhunk_has_changes = 1; - else - s->rhunk_old_count++; +/* + * Expose the in-file line-range filter to callers outside diff.c (e.g. + * pickaxe -G); see xdiff-interface.h for the contract. + */ +int diff_emit_line_ranges(mmfile_t *one, mmfile_t *two, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + struct line_range_filter filter; - return s->ret; + line_range_filter_init(&filter, ranges, line_fn, cb_data); + return line_range_filter_diff(&filter, one, two, xpp, xecfg); } static void pprint_rename(struct strbuf *name, const char *a, const char *b) @@ -2819,6 +2933,75 @@ static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat, return x; } +struct diffstat_hunk_cb_data { + struct precomputed_hunk **h; + size_t *nr, *alloc; +}; + +/* + * Hunk callback that appends each hunk's coordinates to a growable + * array, so one xdiff pass can both sum a diffstat and record hunks for + * the store. + */ +static int diffstat_hunk_cb(long start_a, long count_a, + long start_b, long count_b, + void *cb_data) +{ + struct diffstat_hunk_cb_data *d = cb_data; + + ALLOC_GROW(*d->h, *d->nr + 1, *d->alloc); + (*d->h)[*d->nr].old_start = start_a; + (*d->h)[*d->nr].old_count = count_a; + (*d->h)[*d->nr].new_start = start_b; + (*d->h)[*d->nr].new_count = count_b; + (*d->nr)++; + return 0; +} + +/* + * Collect the hunks of the two files at zero context. diff_fn chooses + * whether trimming runs: xdi_diff applies trim_common_tail, yielding the + * zero-context hunks blame reads; xdl_diff does not, yielding the + * untrimmed hunks. Both run at zero context, so the untrimmed hunks are + * not grouped the way a nonzero context would group them; diffstat only + * sums their counts, which grouping does not change. Sets *ph (caller + * frees) and *ph_nr. + */ +typedef int (*xdiff_fn)(mmfile_t *, mmfile_t *, xpparam_t const *, + xdemitconf_t const *, xdemitcb_t *); +static int collect_hunks(xdiff_fn diff_fn, mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, struct precomputed_hunk **ph, + size_t *ph_nr) +{ + size_t ph_alloc = 0; + xdemitcb_t ecb = { 0 }; + xdemitconf_t xecfg = { 0 }; + struct diffstat_hunk_cb_data cd = { ph, ph_nr, &ph_alloc }; + + *ph = NULL; + *ph_nr = 0; + xecfg.hunk_func = diffstat_hunk_cb; + ecb.priv = &cd; + return diff_fn(mf1, mf2, xpp, &xecfg, &ecb); +} + +void diff_hunks_attach(struct diff_options *o) +{ + if (!(o->output_format & + (DIFF_FORMAT_DIFFSTAT | DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_NUMSTAT))) + return; + o->hunks_writer = diff_hunks_writer_maybe_new(o->repo); +} + +void diff_hunks_detach(struct diff_options *o) +{ + if (o->hunks_read_hits) + trace2_data_intmax("diff-hunks", o->repo, "read-hits", + o->hunks_read_hits); + diff_hunks_writer_finish(o->hunks_writer); + o->hunks_writer = NULL; +} + static int diffstat_consume(void *priv, char *line, unsigned long len) { struct diffstat_t *diffstat = priv; @@ -4054,6 +4237,25 @@ static void builtin_diff(const char *name_a, xpp.ignore_regex_nr = o->ignore_regex_nr; xpp.anchors = o->anchors; xpp.anchors_nr = o->anchors_nr; + + /* + * Send the blob oids only for a side whose content is the + * raw blob: textconv rewrites the bytes, and a working-tree + * side has no stored oid, so pass NULL there rather than an + * oid that would not name what the tool receives. + */ + if (diff_process_fill_hunks(o, name_a, &mf1, &mf2, + (textconv_one || !one->oid_valid) ? NULL : &one->oid, + (textconv_two || !two->oid_valid) ? NULL : &two->oid, + &xpp) + == DIFF_PROCESS_EQUIVALENT) { + if (textconv_one) + free(mf1.ptr); + if (textconv_two) + free(mf2.ptr); + goto free_ab_and_return; + } + xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; xecfg.flags = XDL_EMIT_FUNCNAMES; @@ -4086,54 +4288,19 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_callback lr_state; - unsigned int i; - long max_span = 0; - - memset(&lr_state, 0, sizeof(lr_state)); - lr_state.orig_line_fn = fn_out_consume; - lr_state.orig_cb_data = &ecbdata; - lr_state.ranges = line_ranges; - strbuf_init(&lr_state.rhunk, 0); - strbuf_init(&lr_state.pending_rm, 0); - - /* - * Inflate ctxlen so that all changes within - * any single range are merged into one xdiff - * hunk and the inter-change context is emitted. - * The callback clips back to range boundaries. - * - * The optimal ctxlen depends on where changes - * fall within the range, which is only known - * after xdiff runs; the max range span is the - * upper bound that guarantees correctness in a - * single pass. - */ - for (i = 0; i < line_ranges->nr; i++) { - long span = line_ranges->ranges[i].end - - line_ranges->ranges[i].start; - if (span > max_span) - max_span = span; - } - if (max_span > xecfg.ctxlen) - xecfg.ctxlen = max_span; + struct line_range_filter lr_filter; - if (xdi_diff_outf(&mf1, &mf2, - line_range_hunk_fn, - line_range_line_fn, - &lr_state, &xpp, &xecfg)) - die("unable to generate diff for %s", - one->path); + line_range_filter_init(&lr_filter, line_ranges, + fn_out_consume, &ecbdata); - flush_rhunk(&lr_state); - if (lr_state.ret) + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) die("unable to generate diff for %s", one->path); - strbuf_release(&lr_state.rhunk); - strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); + free(xpp.external_hunks); if (o->word_diff) free_diff_words_data(&ecbdata); if (textconv_one) @@ -4178,6 +4345,180 @@ static const char *get_compact_summary(const struct diff_filepair *p, int is_ren return NULL; } +/* Hunk callback for the provider seam: sum counts into a diffstat entry. */ +static int diffstat_sum_hunk_cb(long start_a UNUSED, long count_a, + long start_b UNUSED, long count_b, + void *cb_data) +{ + struct diffstat_file *data = cb_data; + + data->added += count_b; + data->deleted += count_a; + return 0; +} + +/* + * Fill data->added/deleted for a modified pair from the hunk provider seam: on + * a hit, sum the provided counts; on a warming run, compute and record them. + * Returns 1 when it produced the counts, 0 when neither a provider nor the + * writer is usable for this pair and the caller must compute the diffstat + * itself. + * + * The store is keyed by (old blob, new blob, xdl_opts), so it may only serve + * or receive pairs whose result is determined by that key alone. Inputs that + * perturb the hunks outside the key (-B, -I, --anchored) must be off. + * --ignore-blank-lines coalesces hunks differently between the emit and + * hunk-callback paths, so it is excluded to keep output identical to a + * store-less run. (--inter-hunk-context is not excluded: it only groups hunks, + * and diffstat sums their counts, which grouping does not change.) Both sides + * must be valid regular files with known blob IDs. blame applies the same rule + * to its own perturbing inputs. The -I and --anchored exclusions here, plus + * xdl_opts in the key, cover every field of xpparam_t; the assert in the + * function body checks that at compile time. + */ +static int diffstat_from_hunks(struct diff_options *o, + const char *name_a, + struct diff_filespec *one, + struct diff_filespec *two, + struct diffstat_file *data) +{ + struct precomputed_hunk *ph_trim, *ph_full, *counts; + size_t n_trim, n_full, n_counts, k; + int stable; + mmfile_t mf1, mf2; + xpparam_t xpp; + xpparam_t probe = { .flags = o->xdl_opts, + .ignore_regex_nr = o->ignore_regex_nr, + .anchors_nr = o->anchors_nr }; + + /* + * A process-capable driver makes the tool the producer for the + * path: the stat must reflect the tool's hunks, so neither a + * store read (it holds xdiff's answer) nor this function's own + * xdiff-and-record may stand in. A tool that negotiated + * hunks-by-oid can answer right here, before any blob is read; + * otherwise step aside and let the caller consult it with + * content. + */ + if (diff_process_driver(o, name_a, &probe)) { + switch (diff_process_query_hunks(o, name_a, + one->oid_valid ? &one->oid : NULL, + two->oid_valid ? &two->oid : NULL, + &probe, diffstat_sum_hunk_cb, + data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return 1; + default: + return 0; + } + } + + /* + * xpparam_t is the diff algorithm's input. Its flags are the key's + * xdl_opts; ignore_regex (-I) and anchors (--anchored) are instead + * excluded by the guard below. external_hunks carries a + * caller-supplied answer rather than a setting: the store paths + * build their xpparam_t locally and never populate it, so a + * recorded diff is always xdiff's own. + * + * Adding an xpparam_t field fires this assert (its size no longer + * matches the reference struct). To clear it: (1) add the field to + * the reference struct below; then (2) decide how it affects the + * key -- make it part of the key, or exclude diffs that use it in + * the guard below. The assert only tracks size: a same-size reorder + * or a changed field meaning slips past, so re-read the fields when + * it fires. + */ + (void)BUILD_ASSERT_OR_ZERO(sizeof(xpparam_t) == sizeof(struct { + unsigned long flags; + regex_t **ignore_regex; + size_t ignore_regex_nr; + char **anchors; + size_t anchors_nr; + struct xdl_hunk *external_hunks; + size_t external_hunks_nr; + })); + + if (!((diff_provider_active(o->repo) || o->hunks_writer) && + o->break_opt == -1 && + !o->ignore_regex_nr && + !o->anchors_nr && + !(o->xdl_opts & XDF_IGNORE_BLANK_LINES) && + DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two) && + S_ISREG(one->mode) && S_ISREG(two->mode) && + one->oid_valid && two->oid_valid)) + return 0; + + /* + * On a provider hit, sum hunk counts directly without decompressing + * blobs or running xdiff. The consult keys on the current + * diff_options, so a hit carries the hunks a provider-less run would + * have produced; the answer is validated before the first count + * reaches data. + */ + if (diff_provider_query_hunks(o->repo, &one->oid, &two->oid, + o->xdl_opts, + diffstat_sum_hunk_cb, data)) { + o->hunks_read_hits++; + return 1; + } + + /* A miss on a read-only run: let the caller compute the diffstat. */ + if (!o->hunks_writer) + return 0; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + memset(&xpp, 0, sizeof(xpp)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + + /* + * Compute the zero-context trimmed diff (what blame reads) and the + * untrimmed diff (whose counts a nonzero-context stat matches). + * xdi_diff runs first: it enforces the size limit, so the xdl_diff + * call is already bounded. + */ + if (collect_hunks(xdi_diff, &mf1, &mf2, &xpp, &ph_trim, &n_trim) || + collect_hunks(xdl_diff, &mf1, &mf2, &xpp, &ph_full, &n_full)) + die("unable to generate diffstat for %s", one->path); + + /* + * Match a store-less run: at zero context xdi_diff trims, so sum the + * trimmed diff; otherwise sum the untrimmed one. + */ + counts = o->context ? ph_full : ph_trim; + n_counts = o->context ? n_full : n_trim; + for (k = 0; k < n_counts; k++) { + data->added += counts[k].new_count; + data->deleted += counts[k].old_count; + } + + /* + * Record only a trim-stable pair, one whose trimmed and untrimmed + * diffs are identical, so the single entry answers any consumer at + * any context (see diff-hunks.h). A pair where the two differ is + * never recorded and every consumer computes it. + */ + stable = n_trim == n_full; + for (k = 0; stable && k < n_trim; k++) + stable = ph_trim[k].old_start == ph_full[k].old_start && + ph_trim[k].old_count == ph_full[k].old_count && + ph_trim[k].new_start == ph_full[k].new_start && + ph_trim[k].new_count == ph_full[k].new_count; + if (stable) + diff_hunks_writer_add(o->hunks_writer, &one->oid, &two->oid, + o->xdl_opts, ph_trim, n_trim); + free(ph_trim); + free(ph_full); + return 1; +} + static void builtin_diffstat(const char *name_a, const char *name_b, struct diff_filespec *one, struct diff_filespec *two, @@ -4229,27 +4570,77 @@ static void builtin_diffstat(const char *name_a, const char *name_b, } else if (may_differ) { - /* Crazy xdl interfaces.. */ - xpparam_t xpp; - xdemitconf_t xecfg; - - if (fill_mmfile(o->repo, &mf1, one) < 0 || - fill_mmfile(o->repo, &mf2, two) < 0) - die("unable to read files to diff"); - - memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xpp.flags = o->xdl_opts; - xpp.ignore_regex = o->ignore_regex; - xpp.ignore_regex_nr = o->ignore_regex_nr; - xpp.anchors = o->anchors; - xpp.anchors_nr = o->anchors_nr; - xecfg.ctxlen = o->context; - xecfg.interhunkctxlen = o->interhunkcontext; - xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (xdi_diff_outf(&mf1, &mf2, NULL, - diffstat_consume, diffstat, &xpp, &xecfg)) - die("unable to generate diffstat for %s", one->path); + /* + * Serve or record via the diff-hunks store. A "log -L" + * range-scoped stat is not the whole-pair diff the store + * keys, so it neither reads nor records. Otherwise diff + * normally. + */ + if (p->line_ranges || + !diffstat_from_hunks(o, name_a, one, two, data)) { + /* Crazy xdl interfaces.. */ + xpparam_t xpp; + xdemitconf_t xecfg; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + memset(&xpp, 0, sizeof(xpp)); + memset(&xecfg, 0, sizeof(xecfg)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + xecfg.ctxlen = o->context; + xecfg.interhunkctxlen = o->interhunkcontext; + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; + /* + * Consult the diff process so --stat reflects the + * tool's view of which lines changed rather than the + * builtin line diff. --stat never applies textconv, + * so the tool is fed the same raw mmfiles the stat + * itself diffs (unlike builtin_diff, which consults + * the process on textconv'd content). + * When the tool reports the files as equivalent we + * skip xdiff entirely, leaving added and deleted at + * zero so the file is pruned below, just as + * builtin_diff() emits no patch for an equivalent + * file. + * + * Under -L, feed the tool's hunks through the same + * line-range filter the builtin stat uses, so a + * process-provided diff is scoped to the tracked + * range. + */ + if (diff_process_fill_hunks(o, name_a, &mf1, &mf2, + one->oid_valid ? &one->oid : NULL, + two->oid_valid ? &two->oid : NULL, + &xpp) + != DIFF_PROCESS_EQUIVALENT) { + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, + p->line_ranges, + diffstat_consume, + diffstat); + + if (line_range_filter_diff(&lr_filter, + &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, + diffstat_consume, + diffstat, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } + free(xpp.external_hunks); + } if (DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two)) { struct diffstat_file *file = @@ -4277,11 +4668,29 @@ static void builtin_diffstat(const char *name_a, const char *name_b, diff_free_filespec_data(two); } +/* + * Is the 0-based line index within any of the tracked ranges? + * (range_set ranges are 0-based, half-open [start, end).) This is a + * one-shot query for a single line and scans; the streaming filter + * (line_range_line_fn) uses a forward cursor instead. + */ +static int idx_in_ranges(const struct range_set *ranges, long idx) +{ + unsigned int i; + + for (i = 0; i < ranges->nr; i++) + if (idx >= ranges->ranges[i].start && + idx < ranges->ranges[i].end) + return 1; + return 0; +} + static void builtin_checkdiff(const char *name_a, const char *name_b, const char *attr_path, struct diff_filespec *one, struct diff_filespec *two, - struct diff_options *o) + struct diff_options *o, + const struct range_set *line_ranges) { mmfile_t mf1, mf2; struct checkdiff_t data; @@ -4321,7 +4730,19 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 1; /* at least one context line */ xpp.flags = 0; - if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, + + if (line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, line_ranges, + checkdiff_consume, &data); + lr_filter.orig_hunk_fn = checkdiff_consume_hunk; + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate checkdiff for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, checkdiff_consume, &data, &xpp, &xecfg)) die("unable to generate checkdiff for %s", one->path); @@ -4334,6 +4755,17 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, check_blank_at_eof(&mf1, &mf2, &ecbdata); blank_at_eof = ecbdata.blank_at_eof_in_postimage; + /* + * check_blank_at_eof() scans the whole file; with -L, + * keep the report only when its line is in a tracked + * range. The error's location is the first trailing + * blank line (blank_at_eof, 1-based; ranges 0-based), so + * we scope by that line. + */ + if (blank_at_eof && line_ranges && + !idx_in_ranges(line_ranges, blank_at_eof - 1)) + blank_at_eof = 0; + if (blank_at_eof) { static char *err; if (!err) @@ -5126,7 +5558,8 @@ static void run_checkdiff(struct diff_filepair *p, struct diff_options *o) diff_fill_oid_info(p->one, o->repo->index); diff_fill_oid_info(p->two, o->repo->index); - builtin_checkdiff(name, other, attr_path, p->one, p->two, o); + builtin_checkdiff(name, other, attr_path, p->one, p->two, o, + p->line_ranges); } void repo_diff_setup(struct repository *r, struct diff_options *options) @@ -5923,6 +6356,17 @@ static int diff_opt_submodule(const struct option *opt, return 0; } +static int diff_opt_ext_diff(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_external = !unset; + options->flags.no_diff_process = unset; + return 0; +} + static int diff_opt_textconv(const struct option *opt, const char *arg, int unset) { @@ -6253,8 +6697,9 @@ struct option *add_diff_options(const struct option *opts, N_("exit with 1 if there were differences, 0 otherwise")), OPT_BOOL(0, "quiet", &options->flags.quick, N_("disable all output of the program")), - OPT_BOOL(0, "ext-diff", &options->flags.allow_external, - N_("allow an external diff helper to be executed")), + OPT_CALLBACK_F(0, "ext-diff", options, NULL, + N_("allow an external diff helper to be executed"), + PARSE_OPT_NOARG, diff_opt_ext_diff), OPT_CALLBACK_F(0, "textconv", options, NULL, N_("run external text conversion filters when comparing binary files"), PARSE_OPT_NOARG, diff_opt_textconv), diff --git a/diff.h b/diff.h index bb5cddaf3499e9..7bc3184d77fbb3 100644 --- a/diff.h +++ b/diff.h @@ -173,6 +173,12 @@ struct diff_flags { */ unsigned allow_external; + /** + * Disables diff..process. Set by --no-ext-diff and by + * format-patch. + */ + unsigned no_diff_process; + /** * For communication between the calling program and the options parser; * tell the calling program to signal the presence of difference using @@ -224,6 +230,17 @@ static inline void diff_flags_or(struct diff_flags *a, #define DIFF_WITH_ALG(opts, flag) (((opts)->xdl_opts & ~XDF_DIFF_ALGORITHM_MASK) | XDF_##flag) +/* + * The xdl_opts bits git turns on by default that a from-scratch xdl_opts + * (git blame's own option parsing) does not set, and so must OR in to match + * a store warmed at the default diff settings; a diff_options-based consumer + * (diffstat) already has them in o->xdl_opts. Today this is only the indent + * heuristic. It does NOT cover a non-default diff.algorithm: a repo that + * configures one records under that algorithm, and a consumer keying without + * it misses (a lost hit, not wrong output). + */ +#define DIFF_HUNKS_DEFAULT_XDL_OPTS XDF_INDENT_HEURISTIC + enum diff_words_type { DIFF_WORDS_NONE = 0, DIFF_WORDS_PORCELAIN, @@ -420,6 +437,18 @@ struct diff_options { */ int max_depth; int max_depth_valid; + + /* + * Precomputed diff hunks (see diff-hunks.h). diffstat consults the + * repository's store, repo_diff_hunks_store(), before running xdiff, + * keyed by each file pair's blob object IDs. When hunks_writer is set + * (a warming run), diffstat also records the hunks it computes; the + * writer is attached only for the stat output formats. hunks_read_hits + * counts store hits on the read path, emitted via trace2 so tests can + * confirm the reader is consulted. + */ + struct diff_hunks_writer *hunks_writer; + int hunks_read_hits; }; unsigned diff_filter_bit(char status); @@ -668,6 +697,17 @@ void diffcore_fix_diff_index(void); int diff_queue_is_empty(struct diff_options *o); void diff_flush(struct diff_options*); void diff_free(struct diff_options*); + +/* + * Attach a diff-hunks writer to a diff producing a stat format, so a + * warming run records the hunks it computes; a no-op when writing is off + * or for other formats. (Reads consult the repository's store directly; + * see repo_diff_hunks_store() in diff-hunks.h.) Pair with + * diff_hunks_detach() once the diff is done. + */ +void diff_hunks_attach(struct diff_options *o); +void diff_hunks_detach(struct diff_options *o); + void diff_warn_rename_limit(const char *varname, int needed, int degraded_cc); /* diff-raw status letters */ diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index a52d569911c48e..047b2bf7ac65d5 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -16,7 +16,8 @@ typedef int (*pickaxe_fn)(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws); + regex_t *regexp, kwset_t kws, + const struct range_set *ranges); struct diffgrep_cb { regex_t *regexp; @@ -42,7 +43,8 @@ static int diffgrep_consume(void *priv, char *line, unsigned long len) static int diff_grep(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws UNUSED) + regex_t *regexp, kwset_t kws UNUSED, + const struct range_set *ranges) { struct diffgrep_cb ecbdata; xpparam_t xpp; @@ -50,8 +52,11 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, int ret; /* - * We have both sides; need to run textual diff and see if - * the pattern appears on added/deleted lines. + * We have both sides; need to run textual diff and see if the + * pattern appears on added/deleted lines. Under -L (ranges set), + * forward only the tracked range's lines so the match is scoped. + * -G needs only a hit/no-hit answer, so the line-number bookkeeping + * the filter does for -L patch and check output is irrelevant here. */ memset(&xpp, 0, sizeof(xpp)); memset(&xecfg, 0, sizeof(xecfg)); @@ -65,8 +70,12 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, * An xdiff error might be our "data->hit" from above. See the * comment for xdiff_emit_line_fn in xdiff-interface.h */ - ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, - &ecbdata, &xpp, &xecfg); + if (ranges) + ret = diff_emit_line_ranges(one, two, ranges, diffgrep_consume, + &ecbdata, &xpp, &xecfg); + else + ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, + &ecbdata, &xpp, &xecfg); if (ecbdata.hit) return 1; if (ret) @@ -119,8 +128,13 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o UNUSED, - regex_t *regexp, kwset_t kws) + regex_t *regexp, kwset_t kws, + const struct range_set *ranges UNUSED) { + /* + * -S counts needle occurrences in each whole blob. Scoping this to + * a -L range is left to a follow-up; for now -S ignores the range. + */ unsigned int c1 = one ? contains(one, regexp, kws, 0) : 0; unsigned int c2 = two ? contains(two, regexp, kws, c1 + 1) : 0; return c1 != c2; @@ -132,6 +146,7 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, struct userdiff_driver *textconv_one = NULL; struct userdiff_driver *textconv_two = NULL; mmfile_t mf1, mf2; + const struct range_set *ranges; int ret; /* ignore unmerged */ @@ -169,7 +184,13 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, mf1.size = fill_textconv(o->repo, textconv_one, p->one, &mf1.ptr); mf2.size = fill_textconv(o->repo, textconv_two, p->two, &mf2.ptr); - ret = fn(&mf1, &mf2, o, regexp, kws); + /* + * -L scopes the search to the tracked range, but the range is in + * original-file line coordinates that do not map onto textconv + * output, so search the whole file when textconv is in play. + */ + ranges = (textconv_one || textconv_two) ? NULL : p->line_ranges; + ret = fn(&mf1, &mf2, o, regexp, kws, ranges); if (textconv_one) free(mf1.ptr); diff --git a/environment.c b/environment.c index fc3ed8bb1c7a66..a28b22ae0b2c9d 100644 --- a/environment.c +++ b/environment.c @@ -233,6 +233,7 @@ static const struct fsync_component_name { { "pack", FSYNC_COMPONENT_PACK }, { "pack-metadata", FSYNC_COMPONENT_PACK_METADATA }, { "commit-graph", FSYNC_COMPONENT_COMMIT_GRAPH }, + { "diff-hunks", FSYNC_COMPONENT_DIFF_HUNKS }, { "index", FSYNC_COMPONENT_INDEX }, { "objects", FSYNC_COMPONENTS_OBJECTS }, { "reference", FSYNC_COMPONENT_REFERENCE }, diff --git a/git.c b/git.c index 36f08891ef5476..09c1ae73ac9ab7 100644 --- a/git.c +++ b/git.c @@ -566,6 +566,7 @@ static struct cmd_struct commands[] = { { "diagnose", cmd_diagnose, RUN_SETUP_GENTLY }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, + { "diff-hunks", cmd_diff_hunks, RUN_SETUP }, { "diff-index", cmd_diff_index, RUN_SETUP | NO_PARSEOPT }, { "diff-pairs", cmd_diff_pairs, RUN_SETUP | NO_PARSEOPT }, { "diff-tree", cmd_diff_tree, RUN_SETUP | NO_PARSEOPT }, diff --git a/line-log.c b/line-log.c index 5fc75ae275e03a..75aed95f6f7b3e 100644 --- a/line-log.c +++ b/line-log.c @@ -12,6 +12,7 @@ #include "repository.h" #include "revision.h" #include "xdiff-interface.h" +#include "diff-provider.h" #include "strbuf.h" #include "line-log.h" #include "setup.h" @@ -330,22 +331,50 @@ static int collect_diff_cb(long start_a, long count_a, return 0; } -static int collect_diff(mmfile_t *parent, mmfile_t *target, struct diff_ranges *out) +struct collect_diff_fill { + mmfile_t *parent, *target; +}; + +/* The caller already holds the content; hand it to the provider seam. */ +static int collect_diff_fill(void *data, mmfile_t *old_file, + mmfile_t *new_file) +{ + struct collect_diff_fill *f = data; + + *old_file = *f->parent; + *new_file = *f->target; + return 0; +} + +static int collect_diff(struct diff_options *diffopt, const char *path, + mmfile_t *parent, mmfile_t *target, + struct diff_ranges *out) { struct collect_diff_cbdata cbdata = {NULL}; + struct collect_diff_fill fill = { parent, target }; xpparam_t xpp; - xdemitconf_t xecfg; - xdemitcb_t ecb; memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xecfg.ctxlen = xecfg.interhunkctxlen = 0; - cbdata.diff = out; - xecfg.hunk_func = collect_diff_cb; - memset(&ecb, 0, sizeof(ecb)); - ecb.priv = &cbdata; - return xdi_diff(parent, target, &xpp, &xecfg, &ecb); + + /* + * Consult the diff process (via the provider seam) so range + * tracking agrees with the diff that will be shown. When the + * tool reports the files as equivalent no ranges are collected, + * so the tracked range maps across unchanged and the commit + * drops out of the log, rather than being selected here but + * rendered with an empty diff by the process-aware + * builtin_diff(). Blob oids are not threaded to this path yet, + * so pass NULL: the store is not consulted and no + * old-oid/new-oid is sent (a later change can supply the pair, + * where they would let a tool cache across the range-tracking + * and display passes over the same commit). + */ + if (diff_provider_emit_hunks(diffopt->repo, NULL, NULL, path, + diffopt, &xpp, collect_diff_fill, &fill, + collect_diff_cb, &cbdata) < 0) + return -1; + return 0; } /* @@ -927,7 +956,13 @@ static int process_diff_filepair(struct rev_info *rev, } diff_ranges_init(&diff); - if (collect_diff(&file_parent, &file_target, &diff)) + /* + * Select the driver by the old (parent) path, as builtin_diff() does + * with name_a, so a renamed file resolves to the same driver for + * range tracking as for the diff that is shown. + */ + if (collect_diff(&rev->diffopt, pair->one->path, + &file_parent, &file_target, &diff)) die("unable to generate diff for %s", pair->one->path); /* NEEDSWORK should apply some heuristics to prevent mismatches */ diff --git a/meson.build b/meson.build index 064fe2e2f1f4e5..11f7a73ad2e89e 100644 --- a/meson.build +++ b/meson.build @@ -328,6 +328,8 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-process.c', + 'diff-provider.c', 'diff.c', 'diffcore-break.c', 'diffcore-delta.c', @@ -335,6 +337,7 @@ libgit_sources = [ 'diffcore-pickaxe.c', 'diffcore-rename.c', 'diffcore-rotate.c', + 'diff-hunks.c', 'dir-iterator.c', 'dir.c', 'editor.c', @@ -612,6 +615,7 @@ builtin_sources = [ 'builtin/describe.c', 'builtin/diagnose.c', 'builtin/diff-files.c', + 'builtin/diff-hunks.c', 'builtin/diff-index.c', 'builtin/diff-pairs.c', 'builtin/diff-tree.c', diff --git a/odb.c b/odb.c index 965ef68e4eca22..0c69e1cea04a91 100644 --- a/odb.c +++ b/odb.c @@ -2,6 +2,7 @@ #include "abspath.h" #include "commit-graph.h" #include "config.h" +#include "diff-hunks.h" #include "dir.h" #include "environment.h" #include "gettext.h" @@ -1033,6 +1034,7 @@ void odb_close(struct object_database *o) for (source = o->sources; source; source = source->next) odb_source_close(source); close_commit_graph(o); + close_diff_hunks_store(o); } static void odb_free_sources(struct object_database *o) diff --git a/odb.h b/odb.h index 73553ed5a7b1ea..450f087e4e2e26 100644 --- a/odb.h +++ b/odb.h @@ -8,6 +8,7 @@ #include "thread-utils.h" struct cached_object_entry; +struct diff_hunks_store; struct odb_source_inmemory; struct packed_git; struct repository; @@ -75,6 +76,9 @@ struct object_database { struct commit_graph *commit_graph; unsigned commit_graph_attempted : 1; /* if loading has been attempted */ + struct diff_hunks_store *diff_hunks_store; + unsigned diff_hunks_store_attempted : 1; /* if loading has been attempted */ + /* * This is meant to hold a *small* number of objects that you would * want odb_read_object() to be able to return, but yet you do not want diff --git a/repo-settings.c b/repo-settings.c index 208e09ff17fcee..e1282cdaa1fa56 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -77,6 +77,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "pack.usepathwalk", &r->settings.pack_use_path_walk, 0); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); + repo_cfg_bool(r, "core.diffhunks", &r->settings.core_diff_hunks, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); diff --git a/repo-settings.h b/repo-settings.h index cad9c3f0cc15f3..2b1987c02fa6a7 100644 --- a/repo-settings.h +++ b/repo-settings.h @@ -29,6 +29,7 @@ struct repo_settings { int core_commit_graph; int commit_graph_generation_version; int commit_graph_changed_paths_version; + int core_diff_hunks; int gc_write_commit_graph; int fetch_write_commit_graph; int command_requires_full_index; diff --git a/revision.c b/revision.c index 6a8101e8b7ef5f..7abb287451ba14 100644 --- a/revision.c +++ b/revision.c @@ -3193,8 +3193,10 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s (revs->diffopt.output_format & ~(DIFF_FORMAT_PATCH | DIFF_FORMAT_NO_OUTPUT | DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | - DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY)))) - die(_("-L does not yet support the requested diff format")); + DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | + DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | + DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_CHECKDIFF)))) + die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) revs->expand_tabs_in_log = revs->expand_tabs_in_log_default; diff --git a/sub-process.c b/sub-process.c index 83bf0a0e82e56d..5468939338e6c0 100644 --- a/sub-process.c +++ b/sub-process.c @@ -49,7 +49,7 @@ int subprocess_read_status(int fd, struct strbuf *status) return (len < 0) ? len : 0; } -void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +void subprocess_stop_command(struct subprocess_entry *entry) { if (!entry) return; @@ -57,7 +57,14 @@ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) entry->process.clean_on_exit = 0; kill(entry->process.pid, SIGTERM); finish_command(&entry->process); +} +void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +{ + if (!entry) + return; + + subprocess_stop_command(entry); hashmap_remove(hashmap, &entry->ent, NULL); } @@ -72,7 +79,7 @@ static void subprocess_exit_handler(struct child_process *process) finish_command(process); } -int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn) { int err; @@ -96,15 +103,26 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return err; } - hashmap_entry_init(&entry->ent, strhash(cmd)); - err = startfn(entry); if (err) { error("initialization for subprocess '%s' failed", cmd); - subprocess_stop(hashmap, entry); + subprocess_stop_command(entry); return err; } + return 0; +} + +int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn) +{ + int err; + + err = subprocess_start_command(entry, cmd, startfn); + if (err) + return err; + + hashmap_entry_init(&entry->ent, strhash(cmd)); hashmap_add(hashmap, &entry->ent); return 0; } diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..45f1b8e5e3212f 100644 --- a/sub-process.h +++ b/sub-process.h @@ -52,10 +52,17 @@ int cmd2process_cmp(const void *unused_cmp_data, */ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); -/* Start a subprocess and add it to the subprocess hashmap. */ +/* Start a subprocess and run the startfn (typically handshake). */ +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn); + +/* Start a subprocess, run startfn, and add it to the subprocess hashmap. */ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +/* Kill a subprocess. */ +void subprocess_stop_command(struct subprocess_entry *entry); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..6abcda4afb89c0 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -12,6 +12,7 @@ test_tool_sources = [ 'test-date.c', 'test-delete-gpgsig.c', 'test-delta.c', + 'test-diff-process-backend.c', 'test-dir-iterator.c', 'test-drop-caches.c', 'test-dump-cache-tree.c', diff --git a/t/helper/test-diff-process-backend.c b/t/helper/test-diff-process-backend.c new file mode 100644 index 00000000000000..2c64d2ca4ec633 --- /dev/null +++ b/t/helper/test-diff-process-backend.c @@ -0,0 +1,417 @@ +/* + * Test backend for the long-running diff process protocol + * (see diff-process.c and Documentation/gitattributes.adoc). + * + * Usage: test-tool diff-process-backend --mode= [--log=] + * + * Implements the server side of the pkt-line handshake and a per-file + * response loop. The --mode= switch selects the response shape + * (success, error, abort, crash, malformed hunks). + * + * Per-file request from Git: + * + * packet: git> command=hunks + * packet: git> pathname= + * packet: git> [old-oid=] (omitted for textconv/worktree) + * packet: git> [new-oid=] + * packet: git> 0000 + * packet: git> OLD_CONTENT + * packet: git> 0000 + * packet: git> NEW_CONTENT + * packet: git> 0000 + * + * Response varies by --mode (default: whole-file): + * + * whole-file packet: git< hunk <1|0> <1|0> + * (start is 0 for an empty side, matching git diff) + * fixed-hunk packet: git< hunk 5 2 5 2 + * no-hunks (no hunk packets) + * bad-hunk packet: git< hunk 999 1 999 1 + * bad-parse packet: git< garbage not a hunk + * bad-sync packet: git< hunk 1 2 1 1 + * bad-gap packet: git< hunk 1 1 3 1 + * bad-start packet: git< hunk 0 1 1 1 + * multi-hunk packet: git< hunk 5 2 5 2 + * packet: git< hunk 9 2 9 2 + * insert packet: git< hunk 3 0 3 2 (mid-file count-0 insertion) + * flood packet: git< hunk 1 1 1 1 (x100000) + * overlap packet: git< hunk 1 5 1 5 + * packet: git< hunk 3 2 3 2 + * no-cap (omits capability=hunks during handshake) + * error (status=error instead of status=success) + * abort (status=abort instead of status=success) + * crash exit(1) before sending any response + * oid-fixed (advertises hunks-by-oid; answers any request, + * oid-only or content, with the fixed-hunk response) + * oid-need-content + * (advertises hunks-by-oid; answers an oid-only request + * with status=need-content, a content request with the + * fixed-hunk response) + * + * All success modes (not error/abort/crash) end with: + * + * packet: git< 0000 + * packet: git< status=success + * packet: git< 0000 + * + * Each request is logged to --log as: + * + * command= pathname= old-oid= new-oid= old= new= + */ + +#include "test-tool.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "strbuf.h" + +static FILE *logfile; + +enum mode { + MODE_WHOLE_FILE, + MODE_FIXED_HUNK, + MODE_NO_HUNKS, + MODE_BAD_HUNK, + MODE_BAD_PARSE, + MODE_BAD_SYNC, + MODE_BAD_GAP, + MODE_BAD_START, + MODE_MULTI_HUNK, + MODE_INSERT, + MODE_FLOOD, + MODE_OVERLAP, + MODE_NO_CAP, + MODE_ERROR, + MODE_ABORT, + MODE_CRASH, + MODE_OID_FIXED, + MODE_OID_NEED_CONTENT, +}; + +static enum mode parse_mode(const char *s) +{ + if (!strcmp(s, "whole-file")) + return MODE_WHOLE_FILE; + if (!strcmp(s, "fixed-hunk")) + return MODE_FIXED_HUNK; + if (!strcmp(s, "no-hunks")) + return MODE_NO_HUNKS; + if (!strcmp(s, "bad-hunk")) + return MODE_BAD_HUNK; + if (!strcmp(s, "bad-parse")) + return MODE_BAD_PARSE; + if (!strcmp(s, "bad-sync")) + return MODE_BAD_SYNC; + if (!strcmp(s, "bad-gap")) + return MODE_BAD_GAP; + if (!strcmp(s, "bad-start")) + return MODE_BAD_START; + if (!strcmp(s, "multi-hunk")) + return MODE_MULTI_HUNK; + if (!strcmp(s, "insert")) + return MODE_INSERT; + if (!strcmp(s, "flood")) + return MODE_FLOOD; + if (!strcmp(s, "overlap")) + return MODE_OVERLAP; + if (!strcmp(s, "no-cap")) + return MODE_NO_CAP; + if (!strcmp(s, "error")) + return MODE_ERROR; + if (!strcmp(s, "abort")) + return MODE_ABORT; + if (!strcmp(s, "crash")) + return MODE_CRASH; + if (!strcmp(s, "oid-fixed")) + return MODE_OID_FIXED; + if (!strcmp(s, "oid-need-content")) + return MODE_OID_NEED_CONTENT; + die("unknown --mode=%s", s); +} + +/* + * Read "key=value" packets up to a flush, capturing "command" and + * "pathname". Returns 1 if a request was read, 0 on EOF. + * + * The first packet uses the gentle variant so that a clean shutdown + * by Git (EOF) does not produce a spurious "the remote end hung up + * unexpectedly" on stderr. Subsequent packets use the non-gentle + * variant: once inside a request, truncation is a protocol violation + * and dying loudly is the correct response. + */ +static int read_request_header(char **command, char **pathname, + char **old_oid, char **new_oid) +{ + int first = 1; + char *line; + + *command = *pathname = *old_oid = *new_oid = NULL; + for (;;) { + const char *value; + + if (first) { + if (packet_read_line_gently(0, NULL, &line) < 0) + return 0; + first = 0; + } else { + line = packet_read_line(0, NULL); + } + if (!line) + break; + if (skip_prefix(line, "command=", &value)) + *command = xstrdup(value); + else if (skip_prefix(line, "pathname=", &value)) + *pathname = xstrdup(value); + else if (skip_prefix(line, "old-oid=", &value)) + *old_oid = xstrdup(value); + else if (skip_prefix(line, "new-oid=", &value)) + *new_oid = xstrdup(value); + } + return 1; +} + +static size_t count_lines(const struct strbuf *buf) +{ + size_t lines = 0; + + for (size_t i = 0; i < buf->len; i++) + if (buf->buf[i] == '\n') + lines++; + + return lines + (buf->len > 0 && buf->buf[buf->len - 1] != '\n'); +} + +static void send_status(const char *status) +{ + packet_flush(1); + packet_write_fmt(1, "%s\n", status); + packet_flush(1); +} + +static void respond(enum mode mode, + const struct strbuf *old_buf, + const struct strbuf *new_buf) +{ + switch (mode) { + case MODE_ERROR: + send_status("status=error"); + return; + case MODE_ABORT: + send_status("status=abort"); + return; + case MODE_CRASH: + exit(1); + case MODE_FIXED_HUNK: + case MODE_OID_FIXED: + case MODE_OID_NEED_CONTENT: + /* the oid modes reach here only for a content request */ + packet_write_fmt(1, "hunk 5 2 5 2\n"); + break; + case MODE_BAD_HUNK: + packet_write_fmt(1, "hunk 999 1 999 1\n"); + break; + case MODE_BAD_PARSE: + packet_write_fmt(1, "garbage not a hunk\n"); + break; + case MODE_BAD_SYNC: + packet_write_fmt(1, "hunk 1 2 1 1\n"); + break; + case MODE_BAD_GAP: + /* + * Globally balanced (1 changed line on each side, so the + * total unchanged counts match) but the gap before the + * change differs between sides: old line 1 vs new line 3. + * Exercises the per-gap lockstep-alignment check. + */ + packet_write_fmt(1, "hunk 1 1 3 1\n"); + break; + case MODE_BAD_START: + /* + * A start of 0 is valid only for an empty (count 0) range; + * pairing it with a nonzero count names no line in either + * the protocol's or xdiff's coordinates, so the translation + * rejects it and git falls back to the builtin diff. + */ + packet_write_fmt(1, "hunk 0 1 1 1\n"); + break; + case MODE_MULTI_HUNK: + /* + * Two valid, non-overlapping, gap-aligned hunks. Exercises + * the accepting branch of the per-gap lockstep check with a + * non-zero previous-hunk end (the realistic two-region case). + */ + packet_write_fmt(1, "hunk 5 2 5 2\n"); + packet_write_fmt(1, "hunk 9 2 9 2\n"); + break; + case MODE_INSERT: + /* + * A mid-file pure insertion (count 0 on the old side) in the + * protocol's 1-based-position form: 2 lines inserted before + * old line 3. Exercises the count-0 path, which uses the + * unshifted position (not git diff's "-3,0" display start). + */ + packet_write_fmt(1, "hunk 3 0 3 2\n"); + break; + case MODE_FLOOD: { + /* + * Emit far more hunks than any small file has lines, so Git + * trips its accumulation cap and falls back before reading + * them all. + */ + int i; + for (i = 0; i < 100000; i++) + packet_write_fmt(1, "hunk 1 1 1 1\n"); + break; + } + case MODE_OVERLAP: + packet_write_fmt(1, "hunk 1 5 1 5\n"); + packet_write_fmt(1, "hunk 3 2 3 2\n"); + break; + case MODE_NO_HUNKS: + break; + case MODE_NO_CAP: + case MODE_WHOLE_FILE: { + size_t old_lines = count_lines(old_buf); + size_t new_lines = count_lines(new_buf); + /* + * Match git diff output: start=0 when count=0 + * (empty file side), 1 otherwise. + */ + packet_write_fmt(1, "hunk %"PRIuMAX" %"PRIuMAX + " %"PRIuMAX" %"PRIuMAX"\n", + (uintmax_t)(old_lines ? 1 : 0), + (uintmax_t)old_lines, + (uintmax_t)(new_lines ? 1 : 0), + (uintmax_t)new_lines); + break; + } + } + send_status("status=success"); +} + +static void command_loop(enum mode mode) +{ + for (;;) { + char *command = NULL, *pathname = NULL; + char *old_oid = NULL, *new_oid = NULL; + struct strbuf obuf = STRBUF_INIT; + struct strbuf nbuf = STRBUF_INIT; + int by_oid; + + if (!read_request_header(&command, &pathname, + &old_oid, &new_oid)) + break; /* EOF: Git closed its end */ + + /* An oid-only request has no content sections to read. */ + by_oid = command && !strcmp(command, "hunks-by-oid"); + if (!by_oid) { + read_packetized_to_strbuf(0, &obuf, 0); + read_packetized_to_strbuf(0, &nbuf, 0); + } + + if (logfile) { + fprintf(logfile, + "command=%s pathname=%s old-oid=%s new-oid=%s" + " old=%.*s new=%.*s\n", + command ? command : "(none)", + pathname ? pathname : "(none)", + old_oid ? old_oid : "(none)", + new_oid ? new_oid : "(none)", + (int)(strchrnul(obuf.buf, '\n') - obuf.buf), + obuf.buf, + (int)(strchrnul(nbuf.buf, '\n') - nbuf.buf), + nbuf.buf); + fflush(logfile); + } + + if (by_oid) { + if (mode == MODE_OID_FIXED) { + packet_write_fmt(1, "hunk 5 2 5 2\n"); + send_status("status=success"); + } else { + /* + * oid-need-content, and the safe default for + * a mode that never advertised the + * capability: ask for the content request. + */ + send_status("status=need-content"); + } + } else { + respond(mode, &obuf, &nbuf); + } + + free(command); + free(pathname); + free(old_oid); + free(new_oid); + strbuf_release(&obuf); + strbuf_release(&nbuf); + } +} + +static void handshake(enum mode mode) +{ + char *line; + + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "git-diff-client")) + die("bad welcome: '%s'", line ? line : "(eof)"); + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "version=1")) + die("bad version: '%s'", line ? line : "(eof)"); + if (packet_read_line(0, NULL)) + die("expected flush after version"); + + packet_write_fmt(1, "git-diff-server\n"); + packet_write_fmt(1, "version=1\n"); + packet_flush(1); + + /* Drain capabilities advertised by Git */ + while ((line = packet_read_line(0, NULL))) + ; /* drain */ + + /* Respond with our capabilities (or none for no-cap mode) */ + if (mode != MODE_NO_CAP) + packet_write_fmt(1, "capability=hunks\n"); + if (mode == MODE_OID_FIXED || mode == MODE_OID_NEED_CONTENT) + packet_write_fmt(1, "capability=hunks-by-oid\n"); + packet_flush(1); +} + +static const char *const usage_str[] = { + "test-tool diff-process-backend --mode= [--log=]", + NULL +}; + +int cmd__diff_process_backend(int argc, const char **argv) +{ + const char *mode_str = NULL, *log_path = NULL; + enum mode mode = MODE_WHOLE_FILE; + struct option options[] = { + OPT_STRING(0, "mode", &mode_str, "mode", + "response shape (default whole-file);" + " see the file header for the full list of modes"), + OPT_STRING(0, "log", &log_path, "path", + "append per-request summary to this file"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, usage_str, 0); + if (argc) + usage_with_options(usage_str, options); + + if (mode_str) + mode = parse_mode(mode_str); + + if (log_path) { + logfile = fopen(log_path, "a"); + if (!logfile) + die_errno("failed to open log '%s'", log_path); + } + + handshake(mode); + command_loop(mode); + + if (logfile && fclose(logfile)) + die_errno("error closing log"); + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..3c3f95269c6279 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -22,6 +22,7 @@ static struct test_cmd cmds[] = { { "date", cmd__date }, { "delete-gpgsig", cmd__delete_gpgsig }, { "delta", cmd__delta }, + { "diff-process-backend", cmd__diff_process_backend }, { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, { "dump-cache-tree", cmd__dump_cache_tree }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..a5bb7555162c8e 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -15,6 +15,7 @@ int cmd__csprng(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); int cmd__delete_gpgsig(int argc, const char **argv); +int cmd__diff_process_backend(int argc, const char **argv); int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); diff --git a/t/meson.build b/t/meson.build index c5832fee053561..5f7a90047ef8c5 100644 --- a/t/meson.build +++ b/t/meson.build @@ -512,6 +512,7 @@ integration_tests = [ 't4072-diff-max-depth.sh', 't4073-diff-stat-name-width.sh', 't4074-diff-shifted-matched-group.sh', + 't4080-diff-process.sh', 't4100-apply-stat.sh', 't4101-apply-nonl.sh', 't4102-apply-rename.sh', @@ -576,6 +577,7 @@ integration_tests = [ 't4215-log-skewed-merges.sh', 't4216-log-bloom.sh', 't4217-log-limit.sh', + 't4218-diff-hunks.sh', 't4252-am-options.sh', 't4253-am-keep-cr-dos.sh', 't4254-am-corrupt.sh', diff --git a/t/perf/p4218-diff-hunks.sh b/t/perf/p4218-diff-hunks.sh new file mode 100755 index 00000000000000..a96bbd9f6c0eba --- /dev/null +++ b/t/perf/p4218-diff-hunks.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +test_description='diff-hunks store performance' +. ./perf-lib.sh + +test_perf_default_repo + +# Pick a file to blame pseudo-randomly. The sort key is the blob +# hash, so it is stable. +test_expect_success 'select a file' ' + git ls-tree HEAD | grep ^100644 | + sort -k 3 | head -n 1 | cut -f 2 >filelist +' + +file=$(cat filelist) +export file + +# Warm the store the way an owner would: a stat walk with writing on. +test_perf 'warm the store' ' + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_expect_success 'warm the store' ' + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_perf 'log --stat -1000 (store)' ' + git log --stat -1000 >/dev/null +' + +test_perf 'log --stat -1000 (no store)' ' + git -c core.diffhunks=false log --stat -1000 >/dev/null +' + +test_perf 'blame $file (store)' ' + git blame "$file" >/dev/null +' + +test_perf 'blame $file (no store)' ' + git -c core.diffhunks=false blame "$file" >/dev/null +' + +test_expect_success 'clean up store' ' + git diff-hunks clear +' + +test_done diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh new file mode 100755 index 00000000000000..6fba150d109a5d --- /dev/null +++ b/t/t4080-diff-process.sh @@ -0,0 +1,1076 @@ +#!/bin/sh + +test_description='diff process via long-running process' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +# See t/helper/test-diff-process-backend.c for the backend implementation +# and available --mode= options. + +BACKEND="test-tool diff-process-backend" + +test_expect_success 'setup' ' + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + + # boundary.c: 10 lines, changes at 5-6 and 9-10. + # Used by: hunk boundaries, error fallback, crash, bad hunks, overlap. + cat >boundary.c <<-\EOF && + line1 + line2 + line3 + line4 + OLD5 + OLD6 + line7 + line8 + OLD9 + OLD10 + EOF + git add boundary.c && + + # worddiff.c: single-line function, value changes 1 -> 999. + # Used by: word-diff, --diff-algorithm, --no-ext-diff, --stat. + cat >worddiff.c <<-\EOF && + int value(void) { return 1; } + EOF + git add worddiff.c && + + # newfile.c: single-line function, value changes 42 -> 99. + # Used by: modified file, --exit-code, multiple drivers. + cat >newfile.c <<-\EOF && + int new_func(void) { return 42; } + EOF + git add newfile.c && + + # logtest.c: single-line function for log/format-patch tests. + # Needs two commits so log -1 has a diff. + cat >logtest.c <<-\EOF && + int logfunc(void) { return 1; } + EOF + git add logtest.c && + + # one.c/two.c: two-file pair for error/abort/startup-failure tests. + cat >one.c <<-\EOF && + int first(void) { return 1; } + EOF + cat >two.c <<-\EOF && + int second(void) { return 2; } + EOF + git add one.c two.c && + + git commit -m "initial" && + + # Second commit for logtest.c (so log -1 has something to show). + cat >logtest.c <<-\EOF && + int logfunc(void) { return 2; } + EOF + git add logtest.c && + git commit -m "change logtest.c" && + + # Working tree modifications (not committed). + cat >boundary.c <<-\EOF && + line1 + line2 + line3 + line4 + NEW5 + NEW6 + line7 + line8 + NEW9 + NEW10 + EOF + + cat >worddiff.c <<-\EOF && + int value(void) { return 999; } + EOF + + cat >newfile.c <<-\EOF && + int new_func(void) { return 99; } + EOF + + cat >one.c <<-\EOF && + int first(void) { return 10; } + EOF + + cat >two.c <<-\EOF + int second(void) { return 20; } + EOF +' + +# +# Core behavior: the tool controls which lines are marked as changed. +# + +test_expect_success 'diff process hunk boundaries affect output' ' + # The file has changes at lines 5-6 and 9-10, but fixed-hunk + # only reports lines 5-6 as changed. Lines 9-10 should not + # appear as changed in the output. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + diff boundary.c >actual && + test_grep "^-OLD5" actual && + test_grep "^-OLD6" actual && + test_grep "^+NEW5" actual && + test_grep "^+NEW6" actual && + test_grep ! "^-OLD9" actual && + test_grep ! "^-OLD10" actual && + test_grep ! "^+NEW9" actual && + test_grep ! "^+NEW10" actual +' + +test_expect_success 'diff process accepts valid multi-hunk output' ' + # multi-hunk reports both changed regions (5-6 and 9-10) as two + # gap-aligned hunks. This exercises the accepting branch of the + # per-gap lockstep check (non-zero previous-hunk end) and must + # produce a correct two-region diff with the lines between the + # hunks kept as context. + git -c diff.cdiff.process="$BACKEND --mode=multi-hunk" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + test_grep "^ line7" actual && + test_grep "^ line8" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process accepts a mid-file count-0 insertion' ' + # insert mode reports "hunk 3 0 3 2": a pure insertion (count 0 on + # the old side) in the protocol 1-based-position form. Exercises + # the count-0 hunk path that the other valid-hunk modes (full + # replacements, equal-count modifies) never hit. Empty stderr is + # the discriminator: a mishandled count-0 start would be rejected + # by the lockstep check and warn. + cat >insert.c <<-\EOF && + a + b + c + d + e + EOF + git add insert.c && + git commit -m "add insert.c" && + cat >insert.c <<-\EOF && + a + b + X + Y + c + d + e + EOF + git -c diff.cdiff.process="$BACKEND --mode=insert" \ + diff insert.c >actual 2>stderr && + test_grep "^+X" actual && + test_grep "^+Y" actual && + test_grep "^ c" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with modified file' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- newfile.c >actual 2>stderr && + test_grep "return 99" actual && + test_grep "pathname=newfile.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with added file (empty old side)' ' + cat >added.c <<-\EOF && + int added(void) { return 1; } + EOF + git add added.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --cached -- added.c >actual 2>stderr && + test_grep "added" actual && + test_grep "pathname=added.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with deleted file (empty new side)' ' + git add added.c && + git commit -m "commit added.c" && + git rm added.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --cached -- added.c >actual 2>stderr && + test_grep "deleted file" actual && + test_grep "pathname=added.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process skipped for binary files' ' + printf "\\0binary" >binary.c && + git add binary.c && + git commit -m "add binary" && + printf "\\0changed" >binary.c && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- binary.c >actual && + test_grep "Binary files" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not consulted for unmatched driver' ' + echo "not tracked by cdiff" >unmatched.txt && + git add unmatched.txt && + git commit -m "add unmatched.txt" && + + echo "modified" >unmatched.txt && + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- unmatched.txt >actual && + test_grep "modified" actual && + test_path_is_missing backend.log +' + +test_expect_success 'multiple drivers use separate processes' ' + echo "*.h diff=hdiff" >>.gitattributes && + git add .gitattributes && + + cat >multi.h <<-\EOF && + int header(void) { return 1; } + EOF + git add multi.h && + git commit -m "add multi.h" && + + cat >multi.h <<-\EOF && + int header(void) { return 2; } + EOF + + test_when_finished "rm -f backend-c.log backend-h.log" && + git -c diff.cdiff.process="$BACKEND --log=backend-c.log" \ + -c diff.hdiff.process="$BACKEND --log=backend-h.log" \ + diff -- newfile.c multi.h >actual 2>stderr && + test_grep "pathname=newfile.c" backend-c.log && + test_grep "pathname=multi.h" backend-h.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works alongside textconv' ' + write_script uppercase-filter <<-\EOF && + tr "a-z" "A-Z" <"$1" + EOF + + cat >textconv.c <<-\EOF && + hello world + EOF + git add textconv.c && + git commit -m "add textconv.c" && + + cat >textconv.c <<-\EOF && + goodbye world + EOF + + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.textconv="./uppercase-filter" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff -- textconv.c >actual 2>stderr && + # The diff process receives textconv-transformed (uppercase) content. + test_grep "pathname=textconv.c" backend.log && + test_grep "old=HELLO WORLD" backend.log && + test_grep "new=GOODBYE WORLD" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process --stat is fed raw, not textconv, content' ' + # Reuses textconv.c from the previous test (committed "hello + # world", modified to "goodbye world"). Unlike patch output, + # --stat does not apply textconv, so the tool sees raw lowercase + # content here even with a textconv configured. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.textconv="./uppercase-filter" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --stat -- textconv.c >actual 2>stderr && + test_grep "pathname=textconv.c" backend.log && + test_grep "old=hello world" backend.log && + test_grep "new=goodbye world" backend.log && + test_must_be_empty stderr +' + +# +# Downstream features: word diff, log, equivalent files, exit code. +# + +test_expect_success 'diff process with --word-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --word-diff worddiff.c >actual 2>stderr && + test_grep "\[-1;-\]" actual && + test_grep "{+999;+}" actual && + test_grep "pathname=worddiff.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process works with git log -p' ' + # With no-hunks mode, the tool says the files are equivalent, + # so log -p should show the commit but no diff content. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + log -1 -p -- logtest.c >actual 2>stderr && + test_grep "change logtest.c" actual && + test_grep ! "return 2" actual && + test_grep "command=hunks pathname=logtest.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process no hunks suppresses diff output' ' + cat >nohunks.c <<-\EOF && + int zero(void) { return 0; } + EOF + git add nohunks.c && + git commit -m "add nohunks.c" && + + cat >nohunks.c <<-\EOF && + int zero(void) { return 999; } + EOF + + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff nohunks.c >actual && + test_must_be_empty actual +' + +test_expect_success 'diff process no hunks with --exit-code returns success' ' + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --exit-code nohunks.c +' + +test_expect_success 'diff process equivalent commit: --exit-code and --quiet agree' ' + # A committed blob pair (not a worktree file) whose oids differ but + # the tool reports equivalent. --exit-code and --quiet must agree + # with the shown diff (empty) and report success, not fall back to + # the byte-level "oids differ" answer. + cat >ecq.c <<-\EOF && + alpha + EOF + git add ecq.c && + git commit -m "ecq v1" && + cat >ecq.c <<-\EOF && + beta + EOF + git add ecq.c && + git commit -m "ecq v2" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --exit-code HEAD^ HEAD -- ecq.c && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --quiet HEAD^ HEAD -- ecq.c +' + +test_expect_success 'diff process falls back for trailing-newline-only change' ' + test_when_finished "rm -f backend.log" && + printf "a\nb\nc\n" >eofnl.c && + git add eofnl.c && + git commit -m "add eofnl.c" && + printf "a\nb\nc" >eofnl.c && + # Same lines, only the final newline removed. The tool reports + # no hunks (it sees identical lines), but that change is not + # expressible as hunks, so git falls back to the builtin diff + # rather than treating the files as equivalent. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff eofnl.c >actual 2>stderr && + test_grep "No newline at end of file" actual && + test_grep "pathname=eofnl.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process falls back for added file (empty old side)' ' + test_when_finished "rm -f backend.log" && + printf "x\ny\nz\n" >addnl.c && + git add addnl.c && + # The empty old side has no trailing newline while the new side + # does, so the newline fallback shows the addition rather than + # letting no-hunks suppress the whole new file. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --cached addnl.c >actual 2>stderr && + test_grep "^+x" actual && + test_grep "pathname=addnl.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process with --exit-code and hunks returns failure' ' + test_expect_code 1 git -c diff.cdiff.process="$BACKEND" \ + diff --exit-code newfile.c +' + +test_expect_success 'diff process feeds --numstat counts' ' + # fixed-hunk reports only lines 5-6 as changed, so the stat + # counts come from the tool (2/2), not the builtin diff (4/4). + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + diff --numstat boundary.c >actual 2>stderr && + printf "2\t2\tboundary.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks pathname=boundary.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process --numstat sums multi-hunk counts' ' + # multi-hunk reports both 2-line regions (5-6 and 9-10), so the + # counts add up across both hunks: 4 inserted, 4 deleted. This + # exercises the two-region hunk path through builtin_diffstat. + git -c diff.cdiff.process="$BACKEND --mode=multi-hunk" \ + diff --numstat boundary.c >actual && + printf "4\t4\tboundary.c\n" >expect && + test_cmp expect actual +' + +test_expect_success 'diff process equivalent files produce no --stat line' ' + # A file the tool calls equivalent contributes no stat line, + # matching the empty patch that git diff produces for it. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --stat worddiff.c >actual 2>stderr && + test_must_be_empty actual && + test_grep "command=hunks pathname=worddiff.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process feeds --shortstat counts' ' + # fixed-hunk reports lines 5-6 only, so the summary counts come + # from the tool (2 insertions, 2 deletions), not builtin (4/4). + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + diff --shortstat boundary.c >actual && + test_grep "2 insertions" actual && + test_grep "2 deletions" actual +' + +test_expect_success 'diff process scopes --stat to the tracked range under log -L' ' + test_when_finished "rm -f backend.log" && + cat >rangestat.c <<-\EOF && + line1 + line2 + line3 + line4 + OLD5 + OLD6 + line7 + line8 + OLD9 + OLD10 + EOF + git add rangestat.c && + git commit -m "add rangestat.c" && + + cat >rangestat.c <<-\EOF && + line1 + line2 + line3 + line4 + NEW5 + NEW6 + line7 + line8 + NEW9 + NEW10 + EOF + git add rangestat.c && + git commit -m "change rangestat.c" && + + # The file changes at lines 5-6 and 9-10, but fixed-hunk reports + # only 5-6. The builtin line diff counts both regions (4/4); the + # tool hunks flow through the same line-range filter the stat uses, + # so the range-scoped stat reflects the tool view instead (2/2). + git log --no-ext-diff -L1,10:rangestat.c --oneline --stat >builtin && + test_grep "4 insertions(+), 4 deletions(-)" builtin && + + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + log -L1,10:rangestat.c --oneline --stat >actual && + test_grep "2 insertions(+), 2 deletions(-)" actual && + test_grep ! "4 insertions" actual && + test_grep "command=hunks pathname=rangestat.c" backend.log +' + +test_expect_success 'diff process equivalent file makes --stat --exit-code succeed' ' + # The tool reports worddiff.c equivalent, so --exit-code reports + # no change (0); the builtin diff would report a change (1). + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + diff --stat --exit-code worddiff.c && + test_expect_code 1 git diff --no-ext-diff --stat --exit-code worddiff.c +' + +test_expect_success 'diff process --numstat with mixed equivalent and changed files' ' + test_when_finished "rm -f c.log h.log" && + # Self-contained fixtures: *.c uses whole-file (changed); *.mh + # uses no-hunks (equivalent). + echo "*.mh diff=hdiff" >>.gitattributes && + git add .gitattributes && + printf "int a(void) { return 1; }\n" >mixed.c && + printf "int b(void) { return 1; }\n" >mixed.mh && + git add mixed.c mixed.mh && + git commit -m "add mixed fixtures" && + printf "int a(void) { return 2; }\n" >mixed.c && + printf "int b(void) { return 2; }\n" >mixed.mh && + git -c diff.cdiff.process="$BACKEND --mode=whole-file --log=c.log" \ + -c diff.hdiff.process="$BACKEND --mode=no-hunks --log=h.log" \ + diff --numstat mixed.c mixed.mh >actual 2>stderr && + test_grep "mixed.c" actual && + test_grep ! "mixed.mh" actual && + test_grep "pathname=mixed.c" c.log && + test_grep "pathname=mixed.mh" h.log && + test_must_be_empty stderr +' + +test_expect_success POSIXPERM 'diff process keeps mode-only change in --stat' ' + test_when_finished "rm -f backend.log" && + cat >modeonly.c <<-\EOF && + int m(void) { return 1; } + EOF + git add modeonly.c && + git commit -m "add modeonly.c" && + cat >modeonly.c <<-\EOF && + int m(void) { return 2; } + EOF + git add modeonly.c && + test_chmod +x modeonly.c && + git commit -m "edit and chmod modeonly.c" && + # Content and mode both changed, but no-hunks reports the content + # equivalent. The tool is consulted (counts are zero, not the + # builtin 1/1), yet the mode change keeps the file from being + # pruned. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --stat HEAD^ HEAD >actual 2>stderr && + test_grep "modeonly.c" actual && + test_grep "command=hunks pathname=modeonly.c" backend.log && + test_grep ! "1 insertion" actual && + test_must_be_empty stderr +' + +test_expect_success 'diff process not consulted for default --dirstat' ' + # The default (change-based) --dirstat algorithm counts via its + # own path and never contacts the tool (here --dirstat=0 just + # sets a 0% threshold), so the change is still reported even + # though no-hunks would call it equivalent. --dirstat=lines + # instead uses the process-aware stat path. + test_when_finished "rm -f backend.log" && + mkdir -p dsub && + printf "a\nb\nc\n" >dsub/d.c && + git add dsub/d.c && + git commit -m "add dsub/d.c" && + printf "a\nB\nc\n" >dsub/d.c && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + diff --dirstat=0 dsub/d.c >actual && + test_grep "dsub" actual && + test_path_is_missing backend.log +' + +# +# Bypass mechanisms: flags and commands that skip the diff process. +# + +test_expect_success 'diff process bypassed by --diff-algorithm' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --diff-algorithm=patience worddiff.c >actual && + test_grep "return 999" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process bypassed by --no-ext-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff --no-ext-diff worddiff.c >actual && + test_grep "return 999" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not used by format-patch' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + format-patch -1 --stdout -- logtest.c >actual && + test_grep "return 2" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process bypassed under whitespace-ignoring flags' ' + test_when_finished "rm -f backend.log" && + printf "a\nb\nc\n" >wsbypass.c && + git add wsbypass.c && + git commit -m "add wsbypass.c" && + printf "a\n b \nc\n" >wsbypass.c && + # The tool is never told about these options and could not honor + # them, so git bypasses the process for each (covering the whole + # XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES mask, not just -w). + for opt in -w -b --ignore-space-at-eol --ignore-blank-lines + do + rm -f backend.log && + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff $opt wsbypass.c >actual 2>stderr && + test_path_is_missing backend.log && + test_must_be_empty stderr || + return 1 + done && + # -w additionally suppresses the whitespace-only change via the + # builtin diff that now runs. + git -c diff.cdiff.process="$BACKEND" diff -w wsbypass.c >actual && + test_must_be_empty actual +' + +# +# Error handling and fallback. +# + +test_expect_success 'diff process fallback on tool error status' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=error --log=backend.log" \ + diff boundary.c >actual 2>stderr && + # Fallback produces the full builtin diff (both change regions). + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + # Tool was contacted (it replied with error, not crash). + test_grep "command=hunks pathname=boundary.c" backend.log && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process error keeps tool available for next file' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=error --log=backend.log" \ + diff -- one.c two.c >actual 2>stderr && + # Unlike abort, error keeps the tool available: both files + # are sent to the tool (and both fall back). + test_grep "pathname=one.c" backend.log && + test_grep "pathname=two.c" backend.log && + test_grep "return 10" actual && + test_grep "return 20" actual && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process abort disables for session' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=abort --log=backend.log" \ + diff -- one.c two.c >actual 2>stderr && + # Both files should still produce diff output via fallback. + test_grep "return 10" actual && + test_grep "return 20" actual && + # The tool aborts on the first file and git clears its + # capability. The second file never contacts the tool. + test_grep "pathname=one.c" backend.log && + test_grep ! "pathname=two.c" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process fallback on tool crash' ' + git -c diff.cdiff.process="$BACKEND --mode=crash" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + # Crash is a communication failure, so a warning is emitted. + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process startup failure only warns once' ' + git -c diff.cdiff.process="/nonexistent/tool" \ + diff -- one.c two.c >actual 2>stderr && + # Both files produce diff output via fallback. + test_grep "return 10" actual && + test_grep "return 20" actual && + # Sentinel prevents repeated warnings: only one, not one per file. + test_grep "diff process.*failed" stderr >warnings && + test_line_count = 1 warnings +' + + +test_expect_success 'diff process fallback on bad hunks' ' + git -c diff.cdiff.process="$BACKEND --mode=bad-hunk" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "^-OLD9" actual && + test_grep "^+NEW9" actual && + test_grep "hunk past the end" stderr +' + +test_expect_success 'diff process fallback on mismatched unchanged totals' ' + cat >synctest.c <<-\EOF && + line1 + line2 + line3 + EOF + git add synctest.c && + git commit -m "add synctest.c" && + + cat >synctest.c <<-\EOF && + line1 + changed + line3 + EOF + + # bad-sync reports hunk 1 2 1 1: marks 2 old lines and 1 new + # line as changed, leaving 1 unchanged old vs 2 unchanged new. + # The synchronization invariant fails and git falls back. + git -c diff.cdiff.process="$BACKEND --mode=bad-sync" \ + diff synctest.c >actual 2>stderr && + test_grep "changed" actual && + test_grep "misaligned" stderr +' + +test_expect_success 'diff process fallback on misaligned hunk gap' ' + # bad-gap reports hunk 1 1 3 1 on boundary.c: one changed line + # on each side, so the total unchanged counts match, but the + # unchanged run before the change differs (old line 1 vs new + # line 3). A global count check would accept this and emit a + # corrupt diff; the per-gap lockstep check rejects it and git + # falls back to the builtin algorithm. + git -c diff.cdiff.process="$BACKEND --mode=bad-gap" \ + diff boundary.c >actual 2>stderr && + # The builtin fallback shows both changed regions as additions + # (a corrupt-accepted hunk would show NEW5 only as context). + test_grep "^+NEW5" actual && + test_grep "^+NEW9" actual && + test_grep "misaligned" stderr +' + +test_expect_success 'diff process fallback on overlapping hunks' ' + # boundary.c has 10 lines, so both hunks are in bounds + # but they overlap at lines 3-4, triggering the ordering check. + git -c diff.cdiff.process="$BACKEND --mode=overlap" \ + diff boundary.c >actual 2>stderr && + test_grep "NEW5" actual && + test_grep "overlapping hunks" stderr +' + +test_expect_success 'diff process fallback on malformed hunk line' ' + git -c diff.cdiff.process="$BACKEND --mode=bad-parse" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual +' + +test_expect_success 'diff process fallback on start 0 with nonzero count' ' + # bad-start reports hunk 0 1 1 1. A start of 0 is valid only for + # an empty (count 0) range, so the presentation-to-xdiff + # translation rejects it and git falls back to the builtin diff + # instead of handing xdiff an out-of-range start. + git -c diff.cdiff.process="$BACKEND --mode=bad-start" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "^+NEW5" actual && + test_grep "diff process.*failed" stderr +' + +test_expect_success 'diff process caps a flood of hunks and falls back' ' + # flood emits far more hunks than the file has lines. Git must + # stop accumulating and fall back to the builtin diff rather than + # grow memory without bound. + git -c diff.cdiff.process="$BACKEND --mode=flood" \ + diff boundary.c >actual 2>stderr && + test_grep "^-OLD5" actual && + test_grep "too many hunks" stderr +' + +test_expect_success 'diff process skipped when tool omits capability' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-cap --log=backend.log" \ + diff boundary.c >actual 2>stderr && + # Builtin diff runs: all changes appear, including lines 9-10 + # that a tool-provided hunk would have narrowed away. + test_grep "^-OLD5" actual && + test_grep "^-OLD9" actual && + # The process launched (creating the log) but was + # never sent a per-file request, so no hunks command is logged. + test_path_is_file backend.log && + test_grep ! "command=hunks" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process receives old-oid and new-oid for a blob pair' ' + test_when_finished "rm -f backend.log" && + cat >oidpair.c <<-\EOF && + int f(void) { return 1; } + EOF + git add oidpair.c && + git commit -m "oidpair v1" && + old=$(git rev-parse HEAD:oidpair.c) && + + cat >oidpair.c <<-\EOF && + int f(void) { return 2; } + EOF + git add oidpair.c && + git commit -m "oidpair v2" && + new=$(git rev-parse HEAD:oidpair.c) && + + # Both sides are stored blobs, so their object names are sent. + git -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff HEAD^ HEAD -- oidpair.c >actual 2>stderr && + test_grep "old-oid=$old new-oid=$new" backend.log && + test_must_be_empty stderr +' + +test_expect_success 'diff process omits old-oid and new-oid for textconv content' ' + test_when_finished "rm -f backend.log" && + write_script oidcat <<-\EOF && + cat "$1" + EOF + cat >oidtc.c <<-\EOF && + alpha + EOF + git add oidtc.c && + git commit -m "oidtc v1" && + cat >oidtc.c <<-\EOF && + beta + EOF + git add oidtc.c && + git commit -m "oidtc v2" && + + # textconv rewrites the bytes, so the raw-blob object name that + # would otherwise identify each side is omitted. + git -c diff.cdiff.textconv="./oidcat" \ + -c diff.cdiff.process="$BACKEND --log=backend.log" \ + diff HEAD^ HEAD -- oidtc.c >actual 2>stderr && + test_grep "pathname=oidtc.c" backend.log && + test_grep "old-oid=(none) new-oid=(none)" backend.log && + test_must_be_empty stderr +' + +# +# Blame integration. +# + +test_expect_success 'blame uses tool-provided hunks' ' + cat >blame-hunk.c <<-\EOF && + line1 + line2 + line3 + line4 + original5 + original6 + line7 + line8 + line9 + line10 + EOF + git add blame-hunk.c && + git commit -m "add blame-hunk.c" && + ORIG=$(git rev-parse --short HEAD) && + + cat >blame-hunk.c <<-\EOF && + line1 + line2 + line3 + line4 + changed5 + changed6 + line7 + line8 + changed9 + changed10 + EOF + git add blame-hunk.c && + git commit -m "change blame-hunk.c" && + CHANGE=$(git rev-parse --short HEAD) && + + # With fixed-hunk mode the tool reports only lines 5-6 as changed, + # so blame should attribute lines 9-10 to the original commit + # even though the builtin diff would show them as changed. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + sed -n "5p" actual >line5 && + sed -n "6p" actual >line6 && + test_grep "$CHANGE" line5 && + test_grep "$CHANGE" line6 +' + +test_expect_success 'a warmed hunk store does not override tool hunks in blame' ' + ORIG=$(git rev-parse --short HEAD~1) && + GIT_DIFF_HUNKS_WRITE=1 git log -2 --stat -- blame-hunk.c >/dev/null && + + # Control: without a process, blame is served from the store. + git blame --show-stats blame-hunk.c >stats && + test_grep "num precomputed hits: 1" stats && + + # The store holds the builtin hunks (lines 5-6 and 9-10 changed), + # but a process-capable driver makes the tool the producer, so + # blame must reflect the tool hunks (only lines 5-6), not a store + # hit: lines 9-10 stay attributed to the original commit. + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + git diff-hunks clear +' + +test_expect_success 'a warmed hunk store does not override tool hunks in --stat' ' + test_when_finished "git diff-hunks clear && rm -f trace.json" && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --stat -- blame-hunk.c >/dev/null && + + # Control: without a process, the counts are served from the store. + GIT_TRACE2_EVENT="$PWD/trace.json" git log -1 --numstat -- blame-hunk.c >/dev/null && + test_grep read-hits trace.json && + + # The tool reports only lines 5-6 as changed, so the counts must + # be the tool hunks (2/2), not the stored builtin hunks (4/4). + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual +' + +test_expect_success 'an oid-capable tool answers blame without content' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse --short HEAD~1) && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame blame-hunk.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep ! "command=hunks pathname=" backend.log +' + +test_expect_success 'an oid-capable tool answers --stat without content' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep ! "command=hunks pathname=" backend.log +' + +test_expect_success 'need-content falls back to a content request' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content --log=backend.log" \ + log -1 --format= --numstat -- blame-hunk.c >actual && + printf "2\t2\tblame-hunk.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=blame-hunk.c" backend.log && + test_grep "command=hunks pathname=blame-hunk.c" backend.log +' + +test_expect_success 'a worktree side sends content to an oid-capable tool' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat boundary.c >actual && + printf "2\t2\tboundary.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks pathname=boundary.c" backend.log && + test_grep ! "command=hunks-by-oid" backend.log +' + +test_expect_success 'blame skips commits with no hunks from diff process' ' + cat >blame.c <<-\EOF && + int main(void) { + return 0; + } + EOF + git add blame.c && + git commit -m "add blame.c" && + ORIG_COMMIT=$(git rev-parse --short HEAD) && + + cat >blame.c <<-\EOF && + int main(void) + { + return 0; + } + EOF + git add blame.c && + git commit -m "reformat blame.c" && + BLAME_COMMIT=$(git rev-parse --short HEAD) && + + # Without no-hunks mode, blame attributes the change. + git blame blame.c >without && + test_grep "$BLAME_COMMIT" without && + + # With no-hunks mode, the process considers the files equivalent + # and blame skips the reformat commit, attributing to the original. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks" \ + blame blame.c >with && + test_grep ! "$BLAME_COMMIT" with && + test_grep "$ORIG_COMMIT" with +' + +test_expect_success 'blame --no-ext-diff bypasses diff process' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + blame --no-ext-diff blame.c >actual && + # Without the process, blame attributes the reformat commit normally. + test_grep "$BLAME_COMMIT" actual && + test_path_is_missing backend.log +' + +test_expect_success 'blame --no-ext-diff uses builtin hunks' ' + # fixed-hunk mode would narrow blame to lines 5-6, but + # --no-ext-diff should bypass it and use the builtin diff. + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=fixed-hunk --log=backend.log" \ + blame --no-ext-diff blame-hunk.c >actual && + # Builtin diff attributes lines 9-10 to the change commit. + sed -n "9p" actual >line9 && + test_grep "$CHANGE" line9 && + test_path_is_missing backend.log +' + +test_expect_success 'blame -w bypasses diff process' ' + test_when_finished "rm -f backend.log" && + printf "alpha\nbeta\ngamma\n" >blamew.c && + git add blamew.c && + git commit -m "add blamew.c" && + orig=$(git rev-parse --short HEAD) && + printf "alpha\n beta \ngamma\n" >blamew.c && + git commit -am "reindent beta" && + reindent=$(git rev-parse --short HEAD) && + # blame -w must ignore the whitespace-only change and attribute + # beta to the original commit, not the reindent commit. The tool + # is never told about -w, so blame must bypass it (not let tool + # hunks override -w). + git -c diff.cdiff.process="$BACKEND --mode=whole-file --log=backend.log" \ + blame -w blamew.c >actual && + sed -n "2p" actual >line2 && + test_grep "$orig" line2 && + test_grep ! "$reindent" line2 && + test_path_is_missing backend.log +' + +# +# Line-log (git log -L) range tracking. +# + +test_expect_success 'diff process drops equivalent commit from log -L' ' + test_when_finished "rm -f backend.log" && + cat >linelog.c <<-\EOF && + int tracked(void) { return 1; } + EOF + git add linelog.c && + git commit -m "add linelog.c" && + + cat >linelog.c <<-\EOF && + int tracked(void) { return 2; } + EOF + git commit -am "change tracked line" && + + # Builtin line tracking selects the change commit. + git log --no-ext-diff -L1,1:linelog.c --format="%s" >builtin && + test_grep "change tracked line" builtin && + + # With the tool reporting the change as equivalent, tracking + # drops the commit (the range maps across unchanged) instead of + # selecting it and rendering an empty diff. + git -c diff.cdiff.process="$BACKEND --mode=no-hunks --log=backend.log" \ + log -L1,1:linelog.c --format="%s" >actual && + test_grep ! "change tracked line" actual && + # The creating commit still appears, so the change commit was + # selectively dropped rather than the whole log going empty. + test_grep "add linelog.c" actual && + test_grep "command=hunks pathname=linelog.c" backend.log +' + +test_done diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index ca4eb7bbc713ef..bac74d44794778 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -176,24 +176,15 @@ test_expect_success '--name-status shows status and path' ' test_grep ! "^@@" actual ' -test_expect_success '--stat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --stat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--numstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --numstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--shortstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --shortstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--dirstat is not yet supported with -L' ' +test_expect_success '--dirstat is not supported with -L' ' + # --dirstat is not supported with -L: its default mode measures + # whole-file change, not the tracked lines, and the + # --dirstat=lines variant is deferred too, so both forms are + # rejected like any other unsupported format. test_must_fail git log -L1,24:b.c --dirstat 2>err && - test_grep "does not yet support" err + test_grep "does not support" err && + test_must_fail git log -L1,24:b.c --dirstat=lines 2>err && + test_grep "does not support" err ' test_expect_success 'setup for checking fancy rename following' ' @@ -731,13 +722,138 @@ test_expect_success '-L with -S filters to string-count changes' ' test_expect_success '-L with -G filters to diff-text matches' ' git checkout parent-oids && git log -L:func2:file.c -G "F2 [+] 2" --format= >actual && - # -G greps the whole-file diff text, not just the tracked range; - # combined with -L, this selects commits that both touch func2 - # and have "F2 + 2" in their diff. + # -G greps the diff text, and under -L only the lines in the + # tracked range (unlike -S above, which searches the whole file); + # this selects commits whose change to func2 contains "F2 + 2". test $(grep -c "^diff --git" actual) = 1 && grep "F2 + 2" actual ' +test_expect_success 'setup for trailing deletion test' ' + git checkout --orphan trailing-del && + git reset --hard && + cat >file.c <<-\EOF && + void tracked() + { + return 1; + } + // trailing comment + EOF + git add file.c && + test_tick && + git commit -m "add file with trailing comment" && + # Modify tracked() AND delete the trailing comment in + # one commit, so the commit touches the tracked range + # and is not filtered out by the revision walker. + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete trailing comment" +' + +test_expect_success '-L does not include deletions past end of tracked range' ' + git log -L:tracked:file.c --format= -1 -p >actual && + # The trailing comment deletion is outside the tracked + # range and should not appear in the patch output. + test_grep "return 2" actual && + test_grep ! "trailing comment" actual +' + +test_expect_success '-L includes leading deletions resolved by in-range line' ' + git checkout --orphan leading-del && + git reset --hard && + cat >file.c <<-\EOF && + // leading comment + void tracked() + { + return 1; + } + EOF + git add file.c && + test_tick && + git commit -m "add file with leading comment" && + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete leading comment" && + git log -L:tracked:file.c --format= -1 -p >actual && + # The leading comment deletion is resolved by the next + # non-removal line (void tracked), which is in range: a + # removal is classified by the position of the following + # line, so it joins the range that line falls in. + test_grep "return 2" actual && + test_grep "leading comment" actual +' + +test_expect_success 'setup for line-range filter edge cases' ' + git checkout --orphan filter-edge && + git reset --hard && + cat >file.c <<-\EOF && + void before() + { + return 0; + } + + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + return a + b + c; + } + + void after() + { + return 9; + } + EOF + git add file.c && + test_tick && + git commit -m "initial" +' + +test_expect_success '-L change at exact first line of range' ' + git checkout filter-edge && + # Change the function signature (first line of range) + sed "s/void tracked/int tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change first line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change first line" actual && + test_grep "+int tracked" actual && + test_grep "\\-void tracked" actual +' + +test_expect_success '-L change at exact last line of range' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Change the closing brace line (last line of range) + sed "s/^}$/} \/\/ end tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change last line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change last line" actual && + test_grep "end tracked" actual +' + +test_expect_success '-L pure deletion in range (no additions)' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Delete a line inside tracked() without adding anything + sed "/int c/d" file.c >tmp && + mv tmp file.c && + git commit -a -m "pure deletion" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "pure deletion" actual && + test_grep "\\-.*int c" actual +' + test_expect_success '-L with --diff-filter=M excludes root commit' ' git checkout parent-oids && git log -L:func2:file.c --diff-filter=M --format=%s --no-patch >actual && @@ -762,9 +878,9 @@ test_expect_success '-L with -S suppresses non-matching commits' ' test_cmp expect actual ' -test_expect_success '--full-diff is not yet supported with -L' ' +test_expect_success '--full-diff is not supported with -L' ' test_must_fail git log -L1,24:b.c --full-diff 2>err && - test_grep "does not yet support" err + test_grep "does not support" err ' test_expect_success '-L --oneline has no extra blank line before diff' ' @@ -775,10 +891,289 @@ test_expect_success '-L --oneline has no extra blank line before diff' ' test_grep "^diff --git" line2 ' +test_expect_success 'setup for stat range-scoping tests' ' + git checkout --orphan stat-scoping && + git reset --hard && + cat >file.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add file.c && + test_tick && + git commit -m "Add func1() and func2()" && + + # Modify both functions in a single commit so that + # whole-file stats differ from the counts for the tracked range. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" file.c >tmp && + mv tmp file.c && + git commit -a -m "Modify both functions" +' + +test_expect_success '--numstat counts only lines in tracked range' ' + # "Modify both functions" changes one line in func1 and one in + # func2. Whole-file numstat would show 2 added, 2 deleted. + # numstat for func2 within the tracked range should show only 1 and 1. + git log -L:func2:file.c --numstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "^1 1 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat counts only additions for root commit' ' + # Root commit creates both func1 (4 lines) and func2 (4 lines). + # Whole-file numstat would show 9 lines added. numstat for func2 + # within the tracked range should show only 4. + git log -L:func2:file.c --numstat --format=%s >actual && + test_grep "Add func1() and func2()" actual && + test_grep "^4 0 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--stat counts only lines in tracked range' ' + git log -L:func2:file.c --stat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "file.c |" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--shortstat counts only lines in tracked range' ' + # --shortstat prints only the summary line: no per-file "file.c |" + # line. Counts cover only the tracked range, as for --numstat above. + git log -L:func2:file.c --shortstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "file.c |" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat across renames and multiple commits' ' + # parallel-change carries the tracked function f across an a.c -> b.c + # rename and a merge of two parallel histories. With -M, --numstat + # follows the rename and reports added/removed counts for f within + # the tracked range (not whole-file) per commit; the file column flips from + # b.c to a.c at the rename as the walk goes back in time. Commits + # that do not change the range of f emit no row (the merge and the + # pure file-move produce nothing), so there are fewer rows than + # commits. + git checkout parallel-change && + git log -M -L ":f:b.c" --format= --numstat >actual && + cat >expect <<-\EOF && + 1 1 b.c + 1 1 a.c + 1 1 a.c + 1 1 a.c + 1 0 a.c + 13 0 a.c + EOF + test_cmp expect actual +' + +test_expect_success '-L multiple ranges with --numstat excludes untracked change' ' + git checkout --orphan multi-range && + git reset --hard && + cat >m.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + + int func3() + { + return F3; + } + EOF + git add m.c && + test_tick && + git commit -m "add m.c" && + # Change all three functions but track only func1 and func2. + # Whole-file numstat would be 3 3; a 2 2 result proves the + # untracked func3 change is excluded and the two ranges just sum. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" -e "s/F3/F3 + 3/" m.c >tmp && + mv tmp m.c && + git commit -a -m "Modify all three functions" && + git log -L:func1:m.c -L:func2:m.c --numstat --format=%s -1 >actual && + test_grep "Modify all three functions" actual && + test_grep "^2 2 m.c$" actual && + test_grep ! "^3 3 m.c$" actual +' + test_expect_success '--summary shows new file on root commit' ' git checkout parent-oids && git log -L:func2:file.c --summary --format= >actual && test_grep "create mode 100644 file.c" actual ' +test_expect_success 'setup for --check test' ' + git checkout --orphan check-test && + git reset --hard && + cat >check.c <<-\EOF && + void tracked() + { + return; + } + + void other() + { + return; + } + EOF + git add check.c && + test_tick && + git commit -m "add check.c" && + # Introduce trailing whitespace errors in both functions + sed "s/return;/return; /" check.c >check.c.tmp && + mv check.c.tmp check.c && + git commit -a -m "introduce trailing whitespace" +' + +test_expect_success '--check scoped to tracked range with correct file line' ' + # tracked() trailing whitespace is at check.c:3; report it with the + # real file line number, not a count from the start of the range + # hunk. other() at check.c:8 is outside the range and is excluded. + test_must_fail git log -L:tracked:check.c --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep ! "check.c:8:" actual +' + +test_expect_success '--check reports each of several tracked ranges' ' + # Track both functions as separate ranges. Each range is flushed + # as its own hunk, so the second error must report its real file + # line (check.c:8), not continue the numbering from the first + # range (check.c:3). + test_must_fail git log -L:tracked:check.c -L:other:check.c \ + --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep "check.c:8: trailing whitespace" actual +' + +test_expect_success '--check line numbers stay correct across a gap in one range' ' + git checkout --orphan check-gap && + git reset --hard && + cat >gap.c <<-\EOF && + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + int d = 4; + int e = 5; + int g = 7; + return; + } + EOF + git add gap.c && + test_tick && + git commit -m "add gap.c" && + # Two trailing-whitespace errors within one tracked range, + # separated by clean lines. ctxlen is inflated to the range span, + # so they land in a single xdiff hunk with the gap as context; + # both must report their real file line number, with the context + # lines between them counted. + sed -e "s/int a = 1;/int a = 1; /" -e "s/int g = 7;/int g = 7; /" gap.c >tmp && + mv tmp gap.c && + git commit -a -m "ws errors with a gap" && + test_must_fail git log -L:tracked:gap.c --check --format= >actual && + test_grep "gap.c:3: trailing whitespace" actual && + test_grep "gap.c:8: trailing whitespace" actual +' + +test_expect_success '--check does not report blank-at-eof outside the range' ' + git checkout --orphan check-eof && + git reset --hard && + printf "void tracked()\n{\n return;\n}\n\nint tail = 1;\n" >eof.c && + git add eof.c && + test_tick && + git commit -m "add eof.c" && + # One commit introduces a trailing-whitespace error inside tracked() + # (line 3) and a blank line at end of file (line 7, outside the + # range). The blank-at-eof check scans the whole file, so it must be + # scoped: report the in-range error, not the out-of-range EOF blank. + printf "void tracked()\n{\n return; \n}\n\nint tail = 1;\n\n" >eof.c && + git commit -a -m "ws in range, blank at eof out of range" && + test_must_fail git log -L:tracked:eof.c --check --format= >actual && + test_grep "eof.c:3: trailing whitespace" actual && + test_grep ! "blank line at EOF" actual +' + +test_expect_success '-L -G is scoped to the tracked range' ' + git checkout --orphan grep-scope && + git reset --hard && + cat >gp.c <<-\EOF && + int func1() + { + return ALPHA; + } + + int func2() + { + return BETA; + } + EOF + git add gp.c && + test_tick && + git commit -m "add gp.c" && + sed -e "s/ALPHA/ALPHA2/" -e "s/BETA/BETA2/" gp.c >tmp && + mv tmp gp.c && + git commit -a -m "touch both functions" && + # The commit changes ALPHA (func1) and BETA (func2). Tracking func2, + # -G BETA matches its in-range change; -G ALPHA must not, since ALPHA + # changes only outside the tracked range. + git log -L:func2:gp.c -G BETA --format=%s >actual && + test_grep "touch both functions" actual && + git log -L:func2:gp.c -G ALPHA --format=%s >actual && + test_grep ! "touch both functions" actual +' + +test_expect_success '-L -G searches the whole file under textconv' ' + git checkout --orphan grep-textconv && + git reset --hard && + cat >tc.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add tc.c && + test_tick && + git commit -m "add tc.c" && + # One commit changes func1 and func2; MAGIC lands only in the + # func2 change, outside func1. + sed -e "s/F1/F1 + 1/" -e "s/return F2/return MAGIC/" tc.c >tmp && + mv tmp tc.c && + git commit -a -m "change both funcs" && + echo "tc.c diff=tc" >.gitattributes && + + # Without a textconv driver, -G is scoped to func1, so MAGIC (only + # in the func2 change) does not select the commit. + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_must_be_empty actual && + + # A textconv driver makes the range (original-file line numbers) + # meaningless against the driver output, so -G falls back to the + # whole file and MAGIC now selects the commit. + git config diff.tc.textconv cat && + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_grep "change both funcs" actual +' + test_done diff --git a/t/t4211/sha1/expect.no-assertion-error b/t/t4211/sha1/expect.no-assertion-error index 54c568f273a6d2..95faf51a7b46d9 100644 --- a/t/t4211/sha1/expect.no-assertion-error +++ b/t/t4211/sha1/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index bf79c2f..27c829c 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha1/expect.vanishes-early b/t/t4211/sha1/expect.vanishes-early index a413ad36598ddf..e4b1a201d5d254 100644 --- a/t/t4211/sha1/expect.vanishes-early +++ b/t/t4211/sha1/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index 0b9cae5..5de3ea4 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 100b61a6f2f720f812620a9d10afb3a960ccb73c @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index 5e709a1..0b9cae5 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..444e415 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4211/sha256/expect.no-assertion-error b/t/t4211/sha256/expect.no-assertion-error index c25f2ce19c05d9..815d27f7f17b7e 100644 --- a/t/t4211/sha256/expect.no-assertion-error +++ b/t/t4211/sha256/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index 69cb69c..a0d566e 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha256/expect.vanishes-early b/t/t4211/sha256/expect.vanishes-early index bc33b963dc8570..263fc9eaace442 100644 --- a/t/t4211/sha256/expect.vanishes-early +++ b/t/t4211/sha256/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index e4fa1d8..62c1fc2 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 29f32ac3141c48b22803e5c4127b719917b67d0f8ca8c5248bebfa2a19f7da10 @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index d325124..e4fa1d8 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..9f550c3 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4218-diff-hunks.sh b/t/t4218-diff-hunks.sh new file mode 100755 index 00000000000000..5b8f376a22030d --- /dev/null +++ b/t/t4218-diff-hunks.sh @@ -0,0 +1,585 @@ +#!/bin/sh + +test_description='precomputed diff hunks store (git diff-hunks) + +The store maps an (old blob, new blob, diff settings) key to the hunks of +diffing the pair. It is a cache: reading is always on, while writing is +off by default and enabled per run by GIT_DIFF_HUNKS_WRITE (or the +diffHunks.write config), so a diff or log warms the store only when the +owner opts in. These tests check that a warmed store never changes +output, that lookups honor the diff settings, and that a corrupt store is +read as absent while verify reports the corruption.' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +STORE=.git/objects/diff-hunks + +# Warm the store the way a repository owner would: a stat walk with +# writing enabled. A --stat walk records one entry per trim-stable blob +# pair, serving blame and the summary formats alike. Extra arguments +# (e.g. -c options) are passed to git before "log". +warm () { + GIT_DIFF_HUNKS_WRITE=1 git "$@" log --all --stat >/dev/null +} + +# Run a command with the store disabled, for ground truth. +no_store () { + git -c core.diffhunks=false "$@" +} + +test_expect_success 'setup' ' + test_commit initial file.txt "line 1" && + test_commit second file.txt "line 1 +line 2" && + test_commit third file.txt "line 1 +line 2 +line 3" && + test_commit fourth file.txt "changed line 1 +line 2 +line 3 +line 4" +' + +test_expect_success 'ordinary commands do not create the store' ' + git log --stat >/dev/null && + git blame file.txt >/dev/null && + git diff --stat second third >/dev/null && + test_path_is_missing $STORE +' + +test_expect_success 'writing is gated by env and config, env wins' ' + test_when_finished "git diff-hunks clear" && + # The diffHunks.write config enables writing. + git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_file $STORE && + git diff-hunks clear && + # GIT_DIFF_HUNKS_WRITE overrides the config: 0 disables it. + GIT_DIFF_HUNKS_WRITE=0 git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_missing $STORE && + # and enables it without any config. + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null && + test_path_is_file $STORE +' + +test_expect_success 'a warm builds a store that verifies' ' + warm && + test_path_is_file $STORE && + git diff-hunks verify +' + +test_expect_success 'a second warming run refreshes the store in place' ' + warm && + test_commit fifth file.txt "brand new line" && + warm && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'core.diffhunks=false disables lookups' ' + warm && + git -c core.diffhunks=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out +' + +# Writing seeds from the current store and merges into it, so a later +# warming run keeps the entries an earlier one recorded rather than +# rebuilding. Warm one pair, then a different pair, and confirm the first +# is still served. +test_expect_success 'a later warming run preserves earlier entries' ' + git init incr && + ( + cd incr && + test_commit a1 f.txt "1" && + test_commit a2 f.txt "1 +2" && + test_commit a3 f.txt "1 +2 +3" && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a1 a2 >/dev/null && + git diff-hunks verify && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a2 a3 >/dev/null && + git diff-hunks verify && + + # Blaming as of a2 diffs the a1..a2 pair. If seeding had + # dropped it when the a2..a3 pair was warmed, this would + # report zero precomputed hits. + git blame --show-stats a2 -- f.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9]" out && + + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual + ) +' + +test_expect_success 'log --stat matches with and without the store' ' + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'log --numstat and --shortstat match' ' + no_store log --numstat >expect_num && + no_store log --shortstat >expect_short && + warm && + git log --numstat >actual_num && + git log --shortstat >actual_short && + test_cmp expect_num actual_num && + test_cmp expect_short actual_short +' + +# A built store must reproduce diffstat output at every context length: +# the built context is served, and an unbuilt one falls back to xdiff. +# Zero context is where trim_common_tail runs, so it is keyed apart from +# the nonzero contexts. +test_expect_success 'diffstat matches at several context lengths' ' + no_store log --stat >expect_def && + no_store log -U0 --stat >expect_u0 && + no_store log -U7 --stat >expect_u7 && + warm && + git log --stat >got_def && + git log -U0 --stat >got_u0 && + git log -U7 --stat >got_u7 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + test_cmp expect_u7 got_u7 +' + +test_expect_success 'store built at a nonzero context serves that context' ' + no_store -c diff.context=5 log --stat >expect && + warm -c diff.context=5 && + git -c diff.context=5 log --stat >actual && + test_cmp expect actual +' + +# This blob pair (a real git test file being modernized) decomposes to +# fewer changed lines at zero context, where trim_common_tail runs, than +# at nonzero context: "diff -U0" reports 9/6, "diff -U3" reports 10/7. +# Keying the contexts apart is what keeps each reader correct. The split +# is an emergent property of the whole ~300-line file and does not reduce +# to a few lines, so the pair is shipped as a fixture under t4218/. +test_expect_success 'a trim-divergent file is correct at each context' ' + cp "$TEST_DIRECTORY/t4218/trim-divergent-old" div.sh && + git add div.sh && + git commit -m divergent-old && + cp "$TEST_DIRECTORY/t4218/trim-divergent-new" div.sh && + git add div.sh && + git commit -m divergent-new && + no_store log -1 --format= --stat -- div.sh >expect_def && + no_store log -1 --format= -U0 --stat -- div.sh >expect_u0 && + warm && + git log -1 --format= --stat -- div.sh >got_def && + git log -1 --format= -U0 --stat -- div.sh >got_u0 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + # The fixture must actually diverge, or the test would pass without + # exercising the split; fail loudly if a diff change ever levels it. + ! test_cmp expect_def expect_u0 +' + +# A warming run displays the diffstat it computes. At zero context xdi_diff +# trims, so the displayed counts must be the trimmed ones (what a store-less +# run shows), not the untrimmed counts recorded for a nonzero-context reader. +test_expect_success 'warming --stat at zero context matches a store-less run' ' + git init -q warm-u0 && + ( + cd warm-u0 && + cp "$TEST_DIRECTORY/t4218/trim-divergent-old" div.sh && + git add div.sh && git commit -q -m old && + cp "$TEST_DIRECTORY/t4218/trim-divergent-new" div.sh && + git add div.sh && git commit -q -m new && + git -c core.diffhunks=false log -1 --format= -U0 --stat -- div.sh >expect && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --format= -U0 --stat -- div.sh >got && + test_cmp expect got + ) +' + +test_expect_success 'diff --stat matches with and without the store, both directions' ' + no_store diff --stat second fourth >expect_fwd && + no_store diff --stat fourth second >expect_rev && + warm && + git diff --stat second fourth >got_fwd && + git diff --stat fourth second >got_rev && + test_cmp expect_fwd got_fwd && + test_cmp expect_rev got_rev +' + +test_expect_success 'show and diff-tree --stat use the store' ' + test_when_finished "git diff-hunks clear" && + # The store attaches: a gated show/diff-tree --stat records into the + # store (without the attach there is no writer, so nothing is written). + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git show --stat fourth >/dev/null && + test_path_is_file "$STORE" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff-tree --stat fourth >/dev/null && + test_path_is_file "$STORE" && + # Reading never changes their output. + git diff-hunks clear && + no_store show --stat fourth >expect_show && + no_store diff-tree --stat fourth >expect_dt && + warm && + git show --stat fourth >got_show && + git diff-tree --stat fourth >got_dt && + test_cmp expect_show got_show && + test_cmp expect_dt got_dt +' + +test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' + no_store log -R --stat >expect && + warm && + git log -R --stat >actual && + test_cmp expect actual +' + +# One warm serves both diffstat and blame: the blob pairs a blame +# walks are the same parent-child pairs the diffstat warm recorded. +test_expect_success 'a single warming run serves both blame and diffstat' ' + warm && + git blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9][0-9]*" out +' + +# The diffstat read path produces identical output on a hit or a miss, so +# it emits a trace2 "read-hits" count to prove it consulted the store. +test_expect_success 'diffstat consults the store (trace shows read hits)' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_on.json" git log --stat >/dev/null && + test_grep read-hits trace_on.json && + test_env GIT_TRACE2_EVENT="$PWD/trace_off.json" no_store log --stat >/dev/null && + test_grep ! read-hits trace_off.json +' + +test_expect_success 'blame matches with and without the store' ' + no_store blame file.txt >expect && + warm && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame --porcelain and --incremental match' ' + no_store blame --porcelain file.txt >expect_p && + no_store blame --incremental file.txt >expect_i && + warm && + git blame --porcelain file.txt >got_p && + git blame --incremental file.txt >got_i && + test_cmp expect_p got_p && + test_cmp expect_i got_i +' + +# Diff settings that change hunks but are not part of the store key must +# bypass it in both directions, so output stays byte-identical to a +# store-less run. +test_expect_success 'setup ignore fixture' ' + git init ignore-repo && + ( + cd ignore-repo && + test_write_lines code keep "# c" >f && + git add f && + git commit -m c1 && + test_write_lines codeCH keep "# cX" >f && + git add f && + git commit -m c2 && + warm + ) +' + +test_expect_success '-I bypasses the store' ' + ( + cd ignore-repo && + no_store diff -I"^#" --numstat HEAD~ HEAD >expect && + git diff -I"^#" --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '-B bypasses the store' ' + git init break-repo && + ( + cd break-repo && + test_write_lines a b c d e f g h >f && + git add f && + git commit -m orig && + test_write_lines 1 2 3 4 5 6 7 8 >f && + git add f && + git commit -m rewrite && + warm && + no_store diff -B --stat HEAD~ HEAD >expect && + git diff -B --stat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '--anchored bypasses the store' ' + ( + cd ignore-repo && + no_store diff --stat --anchored=keep HEAD~ HEAD >expect && + git diff --stat --anchored=keep HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success '--ignore-blank-lines output is unaffected by the store' ' + git init ibl-repo && + ( + cd ibl-repo && + printf "a\n\nx\ny\nb\n" >f && + git add f && + git commit -m v1 && + printf "a\nx\ny\nB\n" >f && + git add f && + git commit -m v2 && + warm && + no_store diff --stat --ignore-blank-lines HEAD~ HEAD >expect && + git diff --stat --ignore-blank-lines HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'a whitespace-ignoring diff is not served default entries' ' + git init ws-repo && + ( + cd ws-repo && + test_write_lines alpha beta gamma >f && + git add f && + git commit -m c1 && + test_write_lines " alpha" beta gamma delta >f && + git add f && + git commit -m c2 && + warm && + no_store diff -w --numstat HEAD~ HEAD >expect && + git diff -w --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame -w stays correct and does not hit default entries' ' + ( + cd ws-repo && + no_store blame -w f >expect && + git blame -w --show-stats f >out 2>&1 && + test_grep "num precomputed hits: 0" out && + git blame -w f >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame with indentHeuristic off stays correct and misses' ' + warm && + git -c diff.indentHeuristic=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store -c diff.indentHeuristic=false blame file.txt >expect && + git -c diff.indentHeuristic=false blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a driver algorithm override keeps output correct' ' + git init driver-algo && + ( + cd driver-algo && + echo "file.foo diff=foo" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + test_write_lines 1 2 3 4 5 >file.foo && + git add file.foo && + git commit -m one && + test_write_lines 1 2 X 4 5 6 >file.foo && + git add file.foo && + git commit -m two && + warm -c diff.foo.algorithm=histogram && + no_store -c diff.foo.algorithm=histogram log --stat >expect && + git -c diff.foo.algorithm=histogram log --stat >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame --reverse never consults the store' ' + warm && + git blame --reverse HEAD~3..HEAD file.txt >actual 2>/dev/null && + no_store blame --reverse HEAD~3..HEAD file.txt >expect 2>/dev/null && + test_cmp expect actual +' + +test_expect_success 'blame with a textconv driver bypasses the store' ' + echo "tc.txt diff=tc" >>.gitattributes && + git add .gitattributes && + git commit -m tc-attr && + git config diff.tc.textconv "sed -e s/1/one/" && + test_commit tc1 tc.txt "line 1" && + test_commit tc2 tc.txt "line 1 +line 2" && + warm && + git blame --show-stats tc.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store blame tc.txt >expect && + git blame tc.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame -M and -C stay correct with the store' ' + warm && + no_store blame -M file.txt >expect_m && + no_store blame -C file.txt >expect_c && + git blame -M file.txt >got_m && + git blame -C file.txt >got_c && + test_cmp expect_m got_m && + test_cmp expect_c got_c +' + +# Copy-detecting (and reverse) blame still diff blob pairs through +# pass_blame_to_parent, so they must use the real blame xdl_opts. A +# whitespace-only change is invisible under -w; if -w were silently dropped +# (as it was for -C and --reverse) the -w and non-w results would coincide. +test_expect_success 'blame -C honors -w' ' + git init -q blame-cw && + ( + cd blame-cw && + printf "one\ntwo\nthree\n" >f && + git add f && git commit -q -m base && + printf "one\n two \nthree\n" >f && + git add f && git commit -q -m reindent && + git blame -C -w f >with_w && + git blame -C f >without_w && + ! test_cmp with_w without_w + ) +' + +# Robustness across the shapes an object walk encounters. +test_expect_success 'binary and mode-only changes do not break the writer' ' + printf "\\000\\001\\002" >bin.dat && + git add bin.dat && + git commit -m binary-1 && + printf "\\000\\001\\003\\004" >bin.dat && + git add bin.dat && + git commit -m binary-2 && + echo "mode content" >mode.txt && + git add mode.txt && + git commit -m mode-1 && + test_chmod +x mode.txt && + git commit -m mode-2 && + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'blame across a rename matches' ' + echo "original content" >rename-src.txt && + git add rename-src.txt && + git commit -m "add rename-src" && + echo "more" >>rename-src.txt && + git add rename-src.txt && + git commit -m "modify rename-src" && + git mv rename-src.txt rename-dst.txt && + git commit -m "rename" && + echo "post" >>rename-dst.txt && + git add rename-dst.txt && + git commit -m "modify after rename" && + no_store blame rename-dst.txt >expect && + warm && + git blame rename-dst.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame handles merge commits' ' + git checkout -b merge-side main~2 && + test_commit merge-change merge-file.txt "side content" && + git checkout main && + git merge --no-edit merge-side && + no_store blame merge-file.txt >expect && + warm && + git blame merge-file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'distinct --contents against one revision do not collide' ' + warm && + test_write_lines "line 1" "appended line" >c1 && + test_write_lines "rewritten line" >c2 && + # Ground truth without the store. + no_store blame -s --contents=c2 file.txt initial >expect && + # With the store, an intervening c1 run must not poison the c2 lookup. + git blame -s --contents=c1 file.txt initial >/dev/null && + git blame -s --contents=c2 file.txt initial >actual && + test_cmp expect actual +' + +# Integrity: a structurally broken header is read as absent (the reader +# falls back to xdiff and stays correct); a checksum mismatch is caught +# by verify, which is when integrity is checked. +test_expect_success 'a truncated store is read as absent' ' + warm && + test_copy_bytes 20 <$STORE >truncated && + mv truncated $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a corrupt signature is read as absent' ' + warm && + printf "XXXX" >corrupt && + tail -c +5 <$STORE >>corrupt && + mv corrupt $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# Byte 6 of the header is the chunk count; a value larger than the file +# can hold must be rejected before the chunk table is walked. +test_expect_success 'an over-claimed chunk count is read as absent' ' + warm && + printf "\377" | dd of=$STORE bs=1 seek=6 count=1 conv=notrunc 2>/dev/null && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'verify succeeds on a valid store and on an absent one' ' + warm && + git diff-hunks verify && + git diff-hunks clear && + test_path_is_missing $STORE && + git diff-hunks verify +' + +test_expect_success 'verify detects a checksum mismatch' ' + test_when_finished "git diff-hunks clear" && + warm && + fsize=$(test_file_size $STORE) && + mid=$((fsize / 2)) && + printf "\\377" | dd of=$STORE bs=1 seek=$mid count=1 conv=notrunc 2>/dev/null && + test_must_fail git diff-hunks verify +' + +test_expect_success 'a warm discards a corrupt store rather than seeding from it' ' + test_when_finished "git diff-hunks clear" && + warm && + # Corrupt the checksum: the next warm must not carry the corrupt + # entries forward into a fresh checksum-valid file; it discards + # them (with a warning) and rewrites a store that verifies. + fsize=$(test_file_size $STORE) && + printf "\\377" | dd of=$STORE bs=1 seek=$((fsize / 2)) count=1 conv=notrunc 2>/dev/null && + warm 2>err && + test_grep "failed its checksum" err && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'diff-hunks clear removes the store file' ' + warm && + test_path_is_file $STORE && + git diff-hunks clear && + test_path_is_missing $STORE +' + +test_done diff --git a/t/t4218/trim-divergent-new b/t/t4218/trim-divergent-new new file mode 100644 index 00000000000000..f2de40b5ed8f14 --- /dev/null +++ b/t/t4218/trim-divergent-new @@ -0,0 +1,319 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(($_list_size - $_bisection_size * 2)) + if test "$_bisect_err" -lt 0 + then + _bisect_err=$((0 - $_bisect_err)) + fi + _bisect_err=$(($_bisect_err / 2)) ; # floor + + test_expect_success "bisection diff $_bisect_option $_head $* <= $_max_diff" ' + test $_bisect_err -le $_max_diff + ' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/t/t4218/trim-divergent-old b/t/t4218/trim-divergent-old new file mode 100644 index 00000000000000..daa009c9a1b4b6 --- /dev/null +++ b/t/t4218/trim-divergent-old @@ -0,0 +1,316 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(expr $_list_size - $_bisection_size \* 2) + test "$_bisect_err" -lt 0 && _bisect_err=$(expr 0 - $_bisect_err) + _bisect_err=$(expr $_bisect_err / 2) ; # floor + + test_expect_success \ + "bisection diff $_bisect_option $_head $* <= $_max_diff" \ + 'test $_bisect_err -le $_max_diff' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/userdiff.c b/userdiff.c index b5412e6bc3ecd3..7547874aa2569d 100644 --- a/userdiff.c +++ b/userdiff.c @@ -509,6 +509,13 @@ int userdiff_config(const char *k, const char *v) drv->algorithm = drv->algorithm_owned; return ret; } + if (!strcmp(type, "process")) { + int ret; + FREE_AND_NULL(drv->process_owned); + ret = git_config_string(&drv->process_owned, k, v); + drv->process = drv->process_owned; + return ret; + } return 0; } diff --git a/userdiff.h b/userdiff.h index 827361b0bc9569..a98eabe3770cc6 100644 --- a/userdiff.h +++ b/userdiff.h @@ -3,6 +3,7 @@ #include "notes-cache.h" +struct diff_subprocess; struct index_state; struct repository; @@ -31,6 +32,10 @@ struct userdiff_driver { char *textconv_owned; struct notes_cache *textconv_cache; int textconv_want_cache; + const char *process; + char *process_owned; + struct diff_subprocess *diff_subprocess; + unsigned diff_process_failed : 1; }; enum userdiff_driver_type { USERDIFF_DRIVER_TYPE_BUILTIN = 1<<0, diff --git a/write-or-die.h b/write-or-die.h index ff0408bd849fd8..35ed324307a21b 100644 --- a/write-or-die.h +++ b/write-or-die.h @@ -22,13 +22,15 @@ enum fsync_component { FSYNC_COMPONENT_INDEX = 1 << 4, FSYNC_COMPONENT_REFERENCE = 1 << 5, FSYNC_COMPONENT_OBJECT_MAP = 1 << 6, + FSYNC_COMPONENT_DIFF_HUNKS = 1 << 7, }; #define FSYNC_COMPONENTS_OBJECTS (FSYNC_COMPONENT_LOOSE_OBJECT | \ FSYNC_COMPONENT_PACK) #define FSYNC_COMPONENTS_DERIVED_METADATA (FSYNC_COMPONENT_PACK_METADATA | \ - FSYNC_COMPONENT_COMMIT_GRAPH) + FSYNC_COMPONENT_COMMIT_GRAPH | \ + FSYNC_COMPONENT_DIFF_HUNKS) #define FSYNC_COMPONENTS_DEFAULT ((FSYNC_COMPONENTS_OBJECTS | \ FSYNC_COMPONENTS_DERIVED_METADATA) & \ @@ -46,7 +48,8 @@ enum fsync_component { FSYNC_COMPONENT_COMMIT_GRAPH | \ FSYNC_COMPONENT_INDEX | \ FSYNC_COMPONENT_REFERENCE | \ - FSYNC_COMPONENT_OBJECT_MAP) + FSYNC_COMPONENT_OBJECT_MAP | \ + FSYNC_COMPONENT_DIFF_HUNKS) #ifndef FSYNC_COMPONENTS_PLATFORM_DEFAULT #define FSYNC_COMPONENTS_PLATFORM_DEFAULT FSYNC_COMPONENTS_DEFAULT diff --git a/xdiff-interface.c b/xdiff-interface.c index 5ee2b96d0a756f..afe4c12e9e9bcf 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -91,6 +91,25 @@ static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) return 0; } +static int strbuf_out_line(void *priv, mmbuffer_t *mb, int nbuf) +{ + struct strbuf *out = priv; + int i; + for (i = 0; i < nbuf; i++) + strbuf_add(out, mb[i].ptr, mb[i].size); + return 0; +} + +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen) +{ + xdemitcb_t ecb = { .priv = out, .out_line = strbuf_out_line }; + xdl_emit_hunk_hdr(old_begin, old_count, new_begin, new_count, + func, funclen, &ecb); +} + /* * Trim down common substring at the end of the buffers, * but end on a complete line. @@ -124,7 +143,12 @@ int xdi_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdemitconf_t co if (mf1->size > MAX_XDIFF_SIZE || mf2->size > MAX_XDIFF_SIZE) return -1; - if (!xecfg->ctxlen && !(xecfg->flags & XDL_EMIT_FUNCCONTEXT)) + /* + * External hunks reference line numbers in the original content; + * trimming the tail would change line counts and invalidate them. + */ + if (!xpp->external_hunks && + !xecfg->ctxlen && !(xecfg->flags & XDL_EMIT_FUNCCONTEXT)) trim_common_tail(&a, &b); return xdl_diff(&a, &b, xpp, xecfg, xecb); diff --git a/xdiff-interface.h b/xdiff-interface.h index ce54e1c0e002f8..71e5dffefb8ded 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -46,6 +46,19 @@ int xdi_diff_outf(mmfile_t *mf1, mmfile_t *mf2, xdiff_emit_line_fn line_fn, void *consume_callback_data, xpparam_t const *xpp, xdemitconf_t const *xecfg); + +struct range_set; +/* + * Like xdi_diff_outf(), but forwards only the lines within the given + * (post-image) line ranges to line_fn, as "git log -L" scopes its output. + * Returns line_fn's latched return value (so a consumer can signal a hit + * with a non-zero return), or non-zero on xdiff failure. Defined in + * diff.c (it reuses the line-range filter there). + */ +int diff_emit_line_ranges(mmfile_t *mf1, mmfile_t *mf2, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg); int read_mmfile(mmfile_t *ptr, const char *filename); void read_mmblob(mmfile_t *ptr, struct object_database *odb, const struct object_id *oid); @@ -76,4 +89,19 @@ int xdiff_compare_lines(const char *l1, long s1, */ unsigned long xdiff_hash_string(const char *s, size_t len, long flags); +struct strbuf; + +/* + * Append a unified-diff hunk header to `out`, e.g. + * "@@ - + @@ func\n". The header comes from wrapping xdiff's + * own hunk-header emitter, so it matches what a normal diff would + * produce for these begins and counts. For a side with no lines + * (count 0) the begin is the line before the change, and a count of 1 + * is omitted. + */ +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen); + #endif diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index dc370712e92860..4736bcdb07f16a 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -78,6 +78,18 @@ typedef struct s_mmbuffer { long size; } mmbuffer_t; +/* + * Hunk descriptor for externally computed diffs, in xdiff's own + * coordinates: line numbers are 1-based and a hunk's start is the + * first line it covers. A caller translates any external "empty side" + * idiom (such as git diff's start-0/count-0) to a 1-based start before + * handing hunks over. + */ +struct xdl_hunk { + long old_start, old_count; + long new_start, new_count; +}; + typedef struct s_xpparam { unsigned long flags; @@ -88,6 +100,10 @@ typedef struct s_xpparam { /* See Documentation/diff-options.adoc. */ char **anchors; size_t anchors_nr; + + /* Externally computed hunks: bypass the diff algorithm. Owned by caller. */ + struct xdl_hunk *external_hunks; + size_t external_hunks_nr; } xpparam_t; typedef struct s_xdemitcb { diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index c5a892f91e00c0..73a456f5dd6d28 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -1085,16 +1085,96 @@ static void xdl_mark_ignorable_regex(xdchange_t *xscr, const xdfenv_t *xe, } } +/* + * Populate the changed[] arrays from externally supplied hunks, + * bypassing the diff algorithm. The caller normalizes and validates + * the hunks first (order, overlap, and lockstep alignment), so this + * only marks lines changed after asserting the memory-safety + * preconditions it depends on: non-negative counts and 1-based starts + * (checked with BUG()), and an in-bounds range (a silent -1 so the + * caller can fall back to the builtin diff rather than index changed[] + * out of range). Keeping this diagnostic-free leaves user-facing + * messages to the git layer. + * + * Returns 0 on success, -1 if a hunk is out of range. + */ +static int xdl_populate_hunks_from_external(xdfenv_t *xe, + struct xdl_hunk *hunks, + size_t nr_hunks) +{ + size_t i; + long j; + + /* + * xdl_prepare_env() may dirty changed[] via xdl_cleanup_records(). + * Clear them so only the external hunks are marked. + */ + xdl_clear_changed(&xe->xdf1); + xdl_clear_changed(&xe->xdf2); + + for (i = 0; i < nr_hunks; i++) { + struct xdl_hunk *h = &hunks[i]; + + /* + * Non-negative counts and 1-based starts are caller + * preconditions (it normalizes hunks into xdiff coordinates + * before this point), so a violation is a bug, not a bad + * tool response. + */ + if (h->old_count < 0 || h->new_count < 0) + BUG("external hunk %"PRIuMAX": " + "negative count (old=%ld, new=%ld)", + (uintmax_t)(i + 1), + h->old_count, h->new_count); + if (h->old_start < 1 || h->new_start < 1) + BUG("external hunk %"PRIuMAX": " + "start not 1-based (old=%ld, new=%ld)", + (uintmax_t)(i + 1), + h->old_start, h->new_start); + + /* + * The caller validates ordering, overlap and lockstep + * alignment (and diagnoses a bad response). This is only a + * silent in-bounds guard so the marking loop cannot index + * changed[] out of range: start + count - 1 <= nrec, + * rewritten to avoid overflow. A count of 0 (pure + * insert/delete) allows start == nrec + 1, the position + * after the last line. On a miss, return -1 and let the + * caller fall back to the builtin diff. + */ + if (h->old_count > (long)xe->xdf1.nrec - h->old_start + 1 || + h->new_count > (long)xe->xdf2.nrec - h->new_start + 1) + return -1; + + for (j = 0; j < h->old_count; j++) + xe->xdf1.changed[h->old_start - 1 + j] = true; + for (j = 0; j < h->new_count; j++) + xe->xdf2.changed[h->new_start - 1 + j] = true; + } + + return 0; +} + int xdl_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdemitconf_t const *xecfg, xdemitcb_t *ecb) { xdchange_t *xscr; xdfenv_t xe; emit_func_t ef = xecfg->hunk_func ? xdl_call_hunk_func : xdl_emit_diff; - if (xdl_do_diff(mf1, mf2, xpp, &xe) < 0) { + if (xpp->external_hunks) { + if (xdl_prepare_env(mf1, mf2, xpp, &xe) < 0) + return -1; + if (xdl_populate_hunks_from_external(&xe, + xpp->external_hunks, + xpp->external_hunks_nr) == 0) + goto diff_done; + xdl_free_env(&xe); + } + if (xdl_do_diff(mf1, mf2, xpp, &xe) < 0) return -1; - } + +diff_done: if (xdl_change_compact(&xe.xdf1, &xe.xdf2, xpp->flags) < 0 || xdl_change_compact(&xe.xdf2, &xe.xdf1, xpp->flags) < 0 || xdl_build_script(&xe, &xscr) < 0) { diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 11bada2608a7a4..f4ab93533286da 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -471,3 +471,13 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, return 0; } + +/* + * Reset the changed[] array so that no lines are marked as changed. + * Also clears the sentinel slots at changed[-1] and changed[nrec] + * that xdl_change_compact() relies on during backward scans. + */ +void xdl_clear_changed(xdfile_t *xdf) +{ + memset(xdf->changed - 1, 0, (xdf->nrec + 2) * sizeof(bool)); +} diff --git a/xdiff/xprepare.h b/xdiff/xprepare.h index 947d9fc1bb8cf9..0413baf07bcc90 100644 --- a/xdiff/xprepare.h +++ b/xdiff/xprepare.h @@ -28,6 +28,7 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdfenv_t *xe); void xdl_free_env(xdfenv_t *xe); +void xdl_clear_changed(xdfile_t *xdf);