wip

Author: stuarthalloway

README.md

@@ -1,29 +1,27 @@
 clojure.test.generative
 ========================================
 
-Test data generation and execution harness. Very early days.
-This API will change. You have been warned.
-
+Generative test runner.
 
 Releases and Dependency Information
 ========================================
 
-Latest stable release: 0.1.4
+Latest stable release: 0.4.0
 
 * [All Released Versions](http://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.clojure%22%20AND%20a%3A%22test.generative%22)
 
 * [Development Snapshot Versions](https://oss.sonatype.org/index.html#nexus-search;gav~org.clojure~test.generative~~~)
 
 [Leiningen](https://github.com/technomancy/leiningen) dependency information:
 
-    [org.clojure/test.generative "0.1.4"]
+    [org.clojure/test.generative "0.4.0"]
 
 [Maven](http://maven.apache.org/) dependency information:
 
     <dependency>
       <groupId>org.clojure</groupId>
       <artifactId>test.generative</artifactId>
-      <version>0.1.4</version>
+      <version>0.4.0</version>
     </dependency>
 
 
@@ -41,18 +39,6 @@ and a validator:
 To generate test data, see the fns in the generators namespace. Note
 that these functions shadow a bunch of clojure.core names.
 
-You can run clojure.test and clojure.test.generative tests together
-from the c.t.g. runner:
-
-    (require '[clojure.test.generative.runner :as runner])
-    (runner/-main "src/test/clojure" "src/examples/clojure")
-
-Assertion support is currently minimal. There is an `is` macro,
-similar to clojure.test's, that provides rudimentaty contextual
-reporting.  You can also use plain assertions, or clojure.test
-validation forms such as `is` and `are`, or any other forms that throw
-exception on failure. No contextual reporting for these yet.
-
 You can configure the runner with Java system properties:
 
 <table>
@@ -65,14 +51,8 @@ You can configure the runner with Java system properties:
   <tr>
     <td>clojure.test.generative.msec</td><td>Desired test run duration</td>
   </tr>
-  <tr>
-    <td>clojure.test.generative.handlers</td><td>Comma-delimited list of handlers</td>
-  </tr>
 </table>
 
-The default handler prints test run to stdout, but others could
-e.g. put test events in a database.
-
 Developer Information
 ========================================
 
@@ -84,20 +64,6 @@ Developer Information
 
 * [Compatibility Test Matrix](http://build.clojure.org/job/test.generative-test-matrix/)
 
-Change Log
-====================
-
-* Release 0.1.5 (in development)
-  * Can now run tests under clojure.test
-  * Tests produce data events that can be consumed by arbitrary reporting tools
-  * Example reporting integration with logback. 
-  * Added `is` macro with more detailed reporting than `clojure.test/is`
-  * Removed collection based input generators. Input generators must be fns.
-  * Removed duplicate input check. Tests can be called multiple times with same input. 
-* Release 0.1.4 on 2012.01.03
-  * Initial version
-
-
 Copyright and License
 ========================================
 

pom.xml

@@ -59,19 +59,13 @@
     <dependency>
       <groupId>org.clojure</groupId>
       <artifactId>tools.namespace</artifactId>
-      <version>0.1.1</version>
+      <version>0.2.3</version>
     </dependency>
     <dependency>
       <groupId>org.clojure</groupId>
       <artifactId>data.generators</artifactId>
       <version>0.1.2</version>
     </dependency>
-    <dependency>
-      <groupId>ch.qos.logback</groupId>
-      <artifactId>logback-classic</artifactId>
-      <version>1.0.6</version>
-      <scope>provided</scope>
-    </dependency>
   </dependencies>
 
   <build>

src/main/clojure/clojure/test/generative/runner.clj

@@ -17,7 +17,7 @@
 
 (def ^:private config-mapping
      [["clojure.test.generative.threads"
-       [:threads]
+       [:nthreads]
        read-string
        (max 1 (dec (.availableProcessors (Runtime/getRuntime))))]
       ["clojure.test.generative.msec"
@@ -33,7 +33,6 @@
        (if (seq val)
          (assoc-in m path (coerce val))
          (assoc-in m path default))))
-   {}
    config-mapping))
 
 (def ^:private ^java.util.Random rnd (java.util.Random. (System/currentTimeMillis)))
@@ -43,10 +42,10 @@
   (locking rnd
     (.nextInt rnd)))
 
-(defprotocol TestContainer
+(defprotocol Testable
   (get-tests [_]))
 
-(extend-protocol TestContainer
+(extend-protocol Testable
   clojure.lang.Var 
   (get-tests
    [v]
@@ -65,23 +64,22 @@
    [m] m))
 
 
-(defn find-vars-in-namespaces
+(defn- find-vars-in-namespaces
   [& nses]
   (when nses
     (reduce (fn [v ns] (into v (vals (ns-interns ns)))) [] nses)))
 
-(defn find-vars-in-dirs
+(defn- find-vars-in-dirs
   [& dirs]
   (let [nses (mapcat #(ns/find-namespaces-in-dir (java.io.File. ^String %)) dirs)]
     (doseq [ns nses] (require ns))
     (apply find-vars-in-namespaces nses)))
 
-(defn run-one
+(defn- run-one
   "Run f (presumably for side effects) repeatedly on n threads,
    until msec has passed or somebody throws an exception.
    Returns as many status maps as seeds passed in."
-  [{:keys [test input-gen]} msec seeds]
-  (prn) (prn test)
+  [{:keys [test input-gen]} {:keys [msec seeds]}]
   (let [f (eval test)
         start (System/currentTimeMillis)
         futs (mapv
@@ -106,24 +104,24 @@
               seeds)]
     (map deref futs)))
 
-(defn run-n
-  "Run tests in parallel on nthreads, dividing msec equally between the tests."
-  [nthreads msec tests]
-  (mapcat #(run-one % (/ msec (count tests)) (repeatedly nthreads next-seed)) tests))
-
-(defn failed?
+(defn- failed?
   "Does test result indicate a failure?"
   [result]
   (contains? result :exception))
 
-(defn run-vars
-  "Designed for interactive use.  Prints results to *out* and throws
-   on first failure encountered."
-  [nthreads msec & test-containers]
-  (doseq [result (run-n nthreads msec (mapcat get-tests test-containers))]
-    (if (failed? result)
-      (throw (ex-info "Generative test failed" result))
-      (prn result))))
+(defn- run-n
+  "Run tests in parallel on nthreads, dividing msec equally between the tests.
+   Returns a list of maps of :iter, :seed, :test."
+  [{:keys [nthreads msec]} tests]
+  (mapcat #(run-one %
+                    {:msec (/ msec (count tests))
+                     :seeds (repeatedly nthreads next-seed)})
+          tests))
+
+(defn- prf
+  "Print and flush."
+  [s]
+  (print s) (flush))
 
 (defn dir-tests
   "Returns all tests in dirs"
@@ -134,22 +132,41 @@
          (apply find-vars-in-namespaces)
          (mapcat get-tests))))
 
+(defn inputs
+  "For interactive use.  Returns an infinite sequence of inputs for
+   a test."
+  [test]
+  ((:input-gen test)))
+
+(defn run
+  "Designed for interactive use.  Prints results to *out* and throws
+   on first failure encountered."
+  [nthreads msec & test-containers]
+  (doseq [result (run-n {:nthreads nthreads
+                         :msec msec}
+                        (mapcat get-tests test-containers))]
+    (if (failed? result)
+      (throw (ex-info "Generative test failed" result))
+      (prn result))))
+
 (defn run-suite
   "Designed for test suite use."
-  [{:keys [threads msec verbose]} tests]
-  (reduce
-   (fn [{:keys [failures iters tests]} result]
-     (if (or verbose (:exception result))
-       (do (prn) (prn result))
-       (print "."))
-     (when (:exception result)
-       (.printStackTrace ^Throwable (:exception result)))
-     (flush)
-     {:failures (+ failures (if (:exception result) 1 0))
-      :iters (+ iters (:iter result))
-      :tests (inc tests)})
-   {:failures 0 :iters 0 :tests 0}
-   (run-n threads msec tests)))
+  [{:keys [nthreads msec progress]} tests]
+  (let [progress (or progress #(prf "."))]
+    (reduce
+     (fn [{:keys [failures iters tests]} result]
+       (when (:exception result)
+         (.printStackTrace ^Throwable (:exception result)))
+       (if (:exception result)
+         (prn result)
+         (progress))
+       {:failures (+ failures (if (:exception result) 1 0))
+        :iters (+ iters (:iter result))
+        :tests (+ tests (/ 1 nthreads))})
+     {:failures 0 :iters 0 :tests 0}
+     (run-n {:nthreads nthreads
+             :msec msec}
+            tests))))
 
 (defn -main
   "Command line entry point. Calls System.exit!"