Merge pull request #585 from pattern-lab/bugfix/incremental-builds

Bugfix/incremental builds

Author: bmuenzenmeyer

core/lib/changes_hunter.js

@@ -40,7 +40,8 @@ ChangesHunter.prototype = {
         pattern.compileState = CompileState.CLEAN;
       }
     } catch (e) {
-      // Output does not exist yet, needs recompile
+      // Output does not exist yet, force recompile
+      pattern.compileState = CompileState.NEEDS_REBUILD;
     }
 
     let node = patternlab.graph.node(pattern);

core/lib/pattern_assembler.js

@@ -15,8 +15,8 @@ var path = require('path'),
   ch = require('./changes_hunter'),
   JSON5 = require('json5');
 
-var markdown_parser = new mp();
-var changes_hunter = new ch();
+const markdown_parser = new mp();
+const changes_hunter = new ch();
 
 var pattern_assembler = function () {
   // HELPER FUNCTIONS
@@ -155,6 +155,8 @@ var pattern_assembler = function () {
 
     try {
       var markdownFileName = path.resolve(patternlab.config.paths.source.patterns, currentPattern.subdir, currentPattern.fileName + ".md");
+      changes_hunter.checkLastModified(currentPattern, markdownFileName);
+
       var markdownFileContents = fs.readFileSync(markdownFileName, 'utf8');
 
       var markdownObject = markdown_parser.parse(markdownFileContents);

core/lib/pattern_graph.js

@@ -55,6 +55,21 @@ var nodeName =
 
 PatternGraph.prototype = {
 
+  /**
+   * Synchronizes the graph nodes with the set of all known patterns.
+   * For instance when a pattern is deleted or moved, it might still have a node from the serialized
+   * JSON, but there is no source pattern.
+   *
+   * @see {@link https://github.com/pattern-lab/patternlab-node/issues/580|Issue #580}
+   */
+  sync: function () {
+    // Remove any patterns that are in the graph data, but that haven't been discovered when
+    // walking all patterns iteratively
+    const nodesToRemove = this.nodes().filter(n => !this.patterns.has(n));
+    nodesToRemove.forEach(n => this.remove(n));
+    return nodesToRemove;
+  },
+
   /**
    * Creates an independent copy of the graph where nodes and edges can be modified without
    * affecting the source.
@@ -362,7 +377,7 @@ PatternGraph.resolveJsonGraphFile = function (patternlab, file) {
 PatternGraph.loadFromFile = function (patternlab, file) {
   const jsonGraphFile = this.resolveJsonGraphFile(patternlab, file);
 
-  // File is fresh, so simply constuct an empty graph in memory
+  // File is fresh, so simply construct an empty graph in memory
   if (!fs.existsSync(jsonGraphFile)) {
     return PatternGraph.empty();
   }

core/lib/patternlab.js

@@ -438,10 +438,50 @@ var patternlab_engine = function (config) {
     return true;
   }
 
+  /**
+   * If a graph was serialized and then {@code deletePatternDir == true}, there is a mismatch in the
+   * pattern metadata and not all patterns might be recompiled.
+   * For that reason an empty graph is returned in this case, so every pattern will be flagged as
+   * "needs recompile". Otherwise the pattern graph is loaded from the meta data.
+   *
+   * @param patternlab
+   * @param {boolean} deletePatternDir When {@code true}, an empty graph is returned
+   * @return {PatternGraph}
+   */
+  function loadPatternGraph(deletePatternDir) {
+    // Sanity check to prevent problems when code is refactored
+    if (deletePatternDir) {
+      return PatternGraph.empty();
+    }
+    return PatternGraph.loadFromFile(patternlab);
+  }
+
   function buildPatterns(deletePatternDir) {
 
     patternlab.events.emit('patternlab-build-pattern-start', patternlab);
-    patternlab.graph = PatternGraph.loadFromFile(patternlab);
+
+    let graph = patternlab.graph = loadPatternGraph(deletePatternDir);
+
+    let graphNeedsUpgrade = !PatternGraph.checkVersion(graph);
+
+    if (graphNeedsUpgrade) {
+      plutils.log.info("Due to an upgrade, a complete rebuild is required and the public/patterns directory was deleted. " +
+        "Incremental build is available again on the next successful run.");
+
+      // Ensure that the freshly built graph has the latest version again.
+      patternlab.graph.upgradeVersion();
+    }
+
+    // Flags
+    let incrementalBuildsEnabled = !(deletePatternDir || graphNeedsUpgrade);
+
+    if (incrementalBuildsEnabled) {
+      plutils.log.info("Incremental builds enabled.");
+    } else {
+      // needs to be done BEFORE processing patterns
+      fs.removeSync(paths.public.patterns);
+      fs.emptyDirSync(paths.public.patterns);
+    }
 
     try {
       patternlab.data = buildPatternData(paths.source.data, fs);
@@ -511,34 +551,32 @@ var patternlab_engine = function (config) {
       cacheBuster: patternlab.cacheBuster
     });
 
-    let patternsToBuild = patternlab.patterns;
-
-    let graphNeedsUpgrade = !PatternGraph.checkVersion(patternlab.graph);
+    // If deletePatternDir == true or graph needs to be updated
+    // rebuild all patterns
+    let patternsToBuild = null;
 
-    // Incremental builds are enabled, but we cannot use them
-    if (!deletePatternDir && graphNeedsUpgrade) {
-      plutils.log.info("Due to an upgrade, a complete rebuild is required. " +
-        "Incremental build is available again on the next run.");
-
-      // Ensure that the freshly built graph has the latest version again.
-      patternlab.graph.upgradeVersion();
-    }
+    if (incrementalBuildsEnabled) {
+      // When the graph was loaded from file, some patterns might have been moved/deleted between runs
+      // so the graph data become out of sync
+      patternlab.graph.sync().forEach(n => {
+        plutils.log.info("[Deleted/Moved] " + n);
+      });
 
-    //delete the contents of config.patterns.public before writing
-    //Also if the serialized graph must be updated
-    if (deletePatternDir || graphNeedsUpgrade) {
-      fs.removeSync(paths.public.patterns);
-      fs.emptyDirSync(paths.public.patterns);
-    } else {
       // TODO Find created or deleted files
       let now = new Date().getTime();
-      var modified = pattern_assembler.find_modified_patterns(now, patternlab);
+      let modified = pattern_assembler.find_modified_patterns(now, patternlab);
 
       // First mark all modified files
       for (let p of modified) {
         p.compileState = CompileState.NEEDS_REBUILD;
       }
       patternsToBuild = patternlab.graph.compileOrder();
+    } else {
+      // build all patterns, mark all to be rebuilt
+      patternsToBuild = patternlab.patterns;
+      for (let p of patternsToBuild) {
+        p.compileState = CompileState.NEEDS_REBUILD;
+      }
     }
 
 

core/lib/ui_builder.js

@@ -568,7 +568,7 @@ var ui_builder = function () {
     output += 'var ishControls = {"ishControlsHide":' + JSON.stringify(patternlab.config.ishControlsHide) + '};' + eol;
 
     //navItems
-    output += 'var navItems = {"patternTypes": ' + JSON.stringify(patternlab.patternTypes) + ', "ishControlsHide": ' + JSON.stringify(patternlab.config.ishControlsHide) +'};' + eol;
+    output += 'var navItems = {"patternTypes": ' + JSON.stringify(patternlab.patternTypes) + ', "ishControlsHide": ' + JSON.stringify(patternlab.config.ishControlsHide) + '};' + eol;
 
     //patternPaths
     output += 'var patternPaths = ' + JSON.stringify(patternlab.patternPaths) + ';' + eol;