Add pks list-references to list the references between packs

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+#### `pks list-references` lists the references between packs
+
+`pks list-references` prints, as JSON, every reference from one pack to a
+constant that another pack defines, with any violations `pks check` finds in
+it, recorded or not. Each violation says whether a `package_todo.yml` records
+it and whether the pack enforcing it is strict. A reference that the packs'
+rules allow, such as one covered by a declared dependency, is listed with no
+violations. So this shows how other packs use a pack, which `package_todo.yml`
+files alone can't. Unlike `check`, it exits 0 when references have violations.
+
+`--defining-pack` and `--referencing-pack` limit the list to references to or
+from the packs they name, and each can be given more than once. Given both, a
+reference must match both.
+
+pks keeps no source positions in ERB, so a reference in an ERB file has a null
+`line` and `column`, and the file's references to one constant are listed once.
+`schema/list-references-output.json` describes the output.
+
 ## 0.6.0
 
 ### Features

Cargo.toml

@@ -52,7 +52,7 @@ csv = "1.3.0"                                                          # csv de/
 itertools = "0.13.0"                                                   # tools for iterating over iterable things
 jwalk = "0.8.1"                                                        # for walking the file tree
 path-clean = "1.0.1"                                                   # Pathname#cleaname in Ruby
-rayon = "1.7.0"                                                        # for parallel iteration
+rayon = "1.10.0"                                                       # for parallel iteration
 regex = "1.7.3"
 serde = { version = "~1", features = ["derive"] }                      # de(serialization)
 yaml_serde = "0.10.6"                                                  # de(serialization)

README.md

@@ -59,6 +59,7 @@ Commands:
   list-packs                      List packs based on configuration in packwerk.yml (for debugging purposes)
   list-included-files             List analyzed files based on configuration in packwerk.yml (for debugging purposes)
   list-definitions                List the constants that packs sees and where it sees them (for debugging purposes)
+  list-references                 List references between packs as JSON, with any violations they make
   help                            Print this message or the help of the given subcommand(s)
 
 Options:

schema/list-references-output.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "pks list-references JSON output",
+  "type": "object",
+  "required": ["references"],
+  "additionalProperties": false,
+  "properties": {
+    "references": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/Reference" },
+      "description": "Sorted by file, line, column, constant_name and defining_pack_name. A constant that several packs define gives a reference to each of them at the same place."
+    }
+  },
+  "$defs": {
+    "ViolationType": {
+      "type": "string",
+      "enum": ["dependency", "privacy", "visibility", "layer", "folder_privacy"]
+    },
+    "Reference": {
+      "type": "object",
+      "required": [
+        "file",
+        "line",
+        "column",
+        "constant_name",
+        "referencing_pack_name",
+        "defining_pack_name",
+        "violations"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "file": {
+          "type": "string",
+          "description": "Relative to the project root"
+        },
+        "line": {
+          "type": ["integer", "null"],
+          "minimum": 1,
+          "description": "Counted from 1. null in ERB, where pks keeps no source positions, so an ERB file's references to one constant are listed once. `pks check -o json` gives 0 there."
+        },
+        "column": {
+          "type": ["integer", "null"],
+          "minimum": 0,
+          "description": "Counted in bytes from 0. null when line is null."
+        },
+        "constant_name": { "type": "string" },
+        "referencing_pack_name": { "type": "string" },
+        "defining_pack_name": { "type": "string" },
+        "violations": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/Violation" },
+          "description": "The violations `pks check --ignore-recorded-violations` finds in this reference, in the order dependency, privacy, visibility, layer, folder_privacy. Empty when the reference breaks no rule that is enforced, as when a dependency is declared or enforce_dependencies is false."
+        }
+      },
+      "if": { "properties": { "line": { "type": "null" } } },
+      "then": { "properties": { "column": { "type": "null" } } },
+      "else": { "properties": { "column": { "type": "integer" } } }
+    },
+    "Violation": {
+      "type": "object",
+      "required": ["violation_type", "strict", "recorded"],
+      "additionalProperties": false,
+      "properties": {
+        "violation_type": { "$ref": "#/$defs/ViolationType" },
+        "strict": {
+          "type": "boolean",
+          "description": "Whether the pack that enforces the rule is strict about it: the referencing pack for dependency and layer, and the defining pack for privacy, visibility and folder_privacy"
+        },
+        "recorded": {
+          "type": "boolean",
+          "description": "Whether a package_todo.yml records this violation. `pks check` reports only the violations that are not recorded."
+        }
+      }
+    }
+  }
+}

src/packs.rs

@@ -18,6 +18,7 @@ pub(crate) mod monkey_patch_detection;
 pub(crate) mod pack;
 pub(crate) mod parsing;
 pub(crate) mod raw_configuration;
+pub(crate) mod reference_lister;
 pub(crate) mod remover;
 pub(crate) mod template;
 pub(crate) mod text;
@@ -84,6 +85,14 @@ pub fn remove(
     remover::remove(configuration, &name, force)
 }
 
+pub fn list_references(
+    configuration: &Configuration,
+    defining_packs: &[String],
+    referencing_packs: &[String],
+) -> anyhow::Result<()> {
+    reference_lister::list(configuration, defining_packs, referencing_packs)
+}
+
 /// Determine whether to use colors based on the color choice
 fn color_mode_for(color: ColorChoice) -> text::ColorMode {
     match color {

src/packs/checker.rs

@@ -569,7 +569,7 @@ fn get_all_violations(
     violations
 }
 
-fn get_checkers(
+pub(crate) fn get_checkers(
     configuration: &Configuration,
 ) -> Vec<Box<dyn CheckerInterface + Send + Sync>> {
     vec![

src/packs/cli.rs

@@ -185,6 +185,19 @@ enum Command {
         about = "List the constants that packs sees and where it sees them (for debugging purposes)"
     )]
     ListDefinitions(ListDefinitionsArgs),
+
+    #[clap(
+        about = "List references between packs as JSON, with any violations they make"
+    )]
+    ListReferences {
+        /// List only references to this pack's constants (can be repeated)
+        #[arg(long, value_name = "PACK")]
+        defining_pack: Vec<String>,
+
+        /// List only references from this pack's files (can be repeated)
+        #[arg(long, value_name = "PACK")]
+        referencing_pack: Vec<String>,
+    },
 }
 
 #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
@@ -337,6 +350,14 @@ pub fn run() -> anyhow::Result<()> {
             let ambiguous = args.ambiguous;
             packs::list_definitions(&configuration, ambiguous)
         }
+        Command::ListReferences {
+            defining_pack,
+            referencing_pack,
+        } => packs::list_references(
+            &configuration,
+            &defining_pack,
+            &referencing_pack,
+        ),
         Command::ExposeMonkeyPatches(args) => packs::expose_monkey_patches(
             &configuration,
             &args.rubydir,

src/packs/reference_lister.rs

@@ -0,0 +1,149 @@
+//! `pks list-references`: every reference from one pack to a constant that
+//! another pack defines, with the violations `check` finds in it, as JSON.
+//!
+//! See `schema/list-references-output.json` for the JSON Schema specification.
+
+use std::{
+    collections::HashSet,
+    io::{BufWriter, Write},
+};
+
+use rayon::prelude::{ParallelIterator, ParallelSlice, ParallelSliceMut};
+use serde::Serialize;
+
+use super::{
+    checker::{get_checkers, reference::Reference},
+    checker_configuration::CheckerType,
+    reference_extractor::get_all_references,
+    Configuration,
+};
+
+#[derive(Serialize)]
+struct JsonOutput<'a> {
+    references: Vec<JsonReference<'a>>,
+}
+
+#[derive(Serialize)]
+struct JsonReference<'a> {
+    file: &'a str,
+    line: Option<usize>,
+    column: Option<usize>,
+    constant_name: &'a str,
+    referencing_pack_name: &'a str,
+    defining_pack_name: &'a str,
+    violations: Vec<JsonViolation>,
+}
+
+#[derive(Serialize)]
+struct JsonViolation {
+    violation_type: CheckerType,
+    strict: bool,
+    recorded: bool,
+}
+
+pub(crate) fn list(
+    configuration: &Configuration,
+    defining_packs: &[String],
+    referencing_packs: &[String],
+) -> anyhow::Result<()> {
+    let defining_packs = pack_names(configuration, defining_packs)?;
+    let referencing_packs = pack_names(configuration, referencing_packs)?;
+
+    let all_references =
+        get_all_references(configuration, &configuration.included_files)?;
+    let mut references: Vec<&Reference> = all_references
+        .iter()
+        .filter(|reference| {
+            let Some(defining) = reference.defining_pack_name.as_deref() else {
+                return false; // A constant no pack defines, such as a gem's
+            };
+            let referencing = reference.referencing_pack_name.as_str();
+            defining != referencing
+                && selected(&defining_packs, defining)
+                && selected(&referencing_packs, referencing)
+        })
+        .collect();
+    // The order the schema promises. It also puts the copies of a reference
+    // next to each other, and as they give the same output in any order, an
+    // unstable sort will do.
+    references.par_sort_unstable_by(|a, b| identity(a).cmp(&identity(b)));
+
+    let checkers = get_checkers(configuration);
+    let recorded_violations = &configuration.pack_set.all_violations;
+    let references = references
+        // A constant that a pack defines in several files yields a copy of the
+        // reference for each file, and their violations can differ, as when
+        // only one of the files is public. Like `check`, list each violation
+        // that any copy makes.
+        .par_chunk_by(|a, b| identity(a) == identity(b))
+        .map(|copies| {
+            let mut violations = Vec::new();
+            for checker in &checkers {
+                for reference in copies {
+                    if let Some(violation) =
+                        checker.check(reference, configuration)?
+                    {
+                        violations.push(JsonViolation {
+                            strict: violation.strict,
+                            recorded: recorded_violations
+                                .contains(&violation.identifier),
+                            violation_type: violation.identifier.violation_type,
+                        });
+                        break; // One of each type, as `check` lists them
+                    }
+                }
+            }
+
+            let reference = copies[0];
+            // pks keeps no source positions in ERB, and reports line 0 there
+            let location = (reference.source_location.line > 0)
+                .then_some(&reference.source_location);
+            Ok(JsonReference {
+                file: &reference.relative_referencing_file,
+                line: location.map(|location| location.line),
+                column: location.map(|location| location.column),
+                constant_name: &reference.constant_name,
+                referencing_pack_name: &reference.referencing_pack_name,
+                defining_pack_name: reference
+                    .defining_pack_name
+                    .as_deref()
+                    .expect("references that no pack defines are filtered out"),
+                violations,
+            })
+        })
+        .collect::<anyhow::Result<Vec<_>>>()?;
+
+    let mut stdout = BufWriter::new(std::io::stdout().lock());
+    serde_json::to_writer(&mut stdout, &JsonOutput { references })?;
+    stdout.flush()?;
+    Ok(())
+}
+
+/// The packs given on the command line, which must exist.
+fn pack_names<'a>(
+    configuration: &'a Configuration,
+    names: &[String],
+) -> anyhow::Result<HashSet<&'a str>> {
+    names
+        .iter()
+        .map(|name| Ok(configuration.pack_set.for_pack(name)?.name.as_str()))
+        .collect()
+}
+
+/// No packs given means every pack.
+fn selected(packs: &HashSet<&str>, pack: &str) -> bool {
+    packs.is_empty() || packs.contains(pack)
+}
+
+/// What sets a reference apart from the others. The copies of a reference made
+/// for each file that defines its constant share it. So do all of an ERB file's
+/// references to one constant, since they are all at line 0.
+fn identity(reference: &Reference) -> (&str, usize, usize, &str, Option<&str>) {
+    (
+        &reference.relative_referencing_file,
+        reference.source_location.line,
+        reference.source_location.column,
+        &reference.constant_name,
+        reference.defining_pack_name.as_deref(),
+    )
+}