Skip to content

Commit f2c517b

Browse files
jpalharinijpalharini
committed
Combine code for reducer and converter
1 parent 29e997e commit f2c517b

7 files changed

Lines changed: 131 additions & 87 deletions

File tree

‎bin/lc-stress-tests.sh‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,30 @@ for size in "${struct_sizes[@]}"; do
1313
;;
1414
2|3)
1515
if (( JOB_COMPLETION_INDEX == 2)); then
16-
lein with-profile +bench,+fressian-new run -- "vec" ${size} false
16+
lein with-profile +bench,+fressian-convert run -- "vec" ${size} false
1717
else
18-
lein with-profile +bench,+fressian-new run -- "map" ${size} false
18+
lein with-profile +bench,+fressian-convert run -- "map" ${size} false
1919
fi
2020
;;
2121
4|5)
2222
if (( JOB_COMPLETION_INDEX == 4)); then
23-
lein with-profile +bench,+fressian-new run -- "vec" ${size} true
23+
lein with-profile +bench,+fressian-convert run -- "vec" ${size} true
2424
else
25-
lein with-profile +bench,+fressian-new run -- "map" ${size} true
25+
lein with-profile +bench,+fressian-convert run -- "map" ${size} true
26+
fi
27+
;;
28+
6|7)
29+
if (( JOB_COMPLETION_INDEX == 6)); then
30+
lein with-profile +bench,+fressian-reduce run -- "vec" ${size} false
31+
else
32+
lein with-profile +bench,+fressian-reduce run -- "map" ${size} false
33+
fi
34+
;;
35+
8|9)
36+
if (( JOB_COMPLETION_INDEX == 8)); then
37+
lein with-profile +bench,+fressian-reduce run -- "vec" ${size} true
38+
else
39+
lein with-profile +bench,+fressian-reduce run -- "map" ${size} true
2640
fi
2741
;;
2842
esac

‎dev/aux.clj‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
(ns aux
2-
(:require [clojure.string :as str])
3-
(:import [org.slf4j LoggerFactory]))
2+
(:require [clojure.core.async :as async]
3+
[clojure.string :as str])
4+
(:import [java.io ByteArrayInputStream ByteArrayOutputStream]
5+
[org.fressian FressianReader FressianWriter]
6+
[org.slf4j LoggerFactory]))
47

58
(defn get-fressian-version []
69
(let [libs (-> (System/getProperty "java.class.path")
@@ -12,8 +15,6 @@
1215
(re-find #".*fressian-(.*)\.jar")
1316
(second))))
1417

15-
16-
1718
(defn- log [level msg]
1819
`(let [logger# (LoggerFactory/getLogger "fressian")]
1920
(. logger# ~level (str ~msg))))
@@ -28,4 +29,56 @@
2829
(log 'warn msg))
2930

3031
(defmacro error [msg]
31-
(log 'error msg))
32+
(log 'error msg))
33+
34+
(defn gen-vec [size]
35+
(into [] (for [_ (range size)]
36+
(str (random-uuid)))))
37+
38+
(defn gen-map [size]
39+
(into {} (for [_ (range size)]
40+
{(str (random-uuid)) (str (random-uuid))})))
41+
42+
(defn fressian [data]
43+
(with-open [baos (ByteArrayOutputStream.)
44+
fw (FressianWriter. baos)]
45+
(.writeObject fw data)
46+
(.toByteArray baos)))
47+
48+
(defn defressian [data & [handlers]]
49+
(with-open [bais (ByteArrayInputStream. data)
50+
fr (FressianReader. bais handlers)]
51+
(.readObject fr)))
52+
53+
(defmacro measure-duration [& body]
54+
`(let [st# (System/nanoTime)]
55+
~@body
56+
(- (System/nanoTime) st#)))
57+
58+
(defn stress-test [{:keys [struct-type struct-size convert-list?] :as argmap}
59+
read-handlers]
60+
(let [gen-struct-fn (case (str struct-type)
61+
"map" gen-map
62+
"vec" gen-vec)
63+
fr-handlers (when convert-list? read-handlers)]
64+
(info (merge {:event :starting-stress-test} argmap))
65+
(let [work-chan (async/chan 1)
66+
timeout-chan (async/timeout (* 1000 60 20))
67+
; avoid submillisecond measurement
68+
struct-count (* 1000 (if (< struct-size 9)
69+
100 1))]
70+
(loop [runs 0
71+
proc-time 0]
72+
(let [structs (repeatedly struct-count #(gen-struct-fn struct-size))
73+
; put generated structs fressianed into the work channel as a single coll
74+
_ (async/put! work-chan (into [] (for [st structs] (fressian st))))
75+
[val port] (async/alts!! [work-chan timeout-chan])]
76+
(cond
77+
(= port timeout-chan)
78+
(info (merge {:event :stress-test-complete :runs runs :processing-time proc-time} argmap))
79+
80+
(= port work-chan)
81+
; measure time to read all structs in work chan
82+
(let [dur (measure-duration (doseq [fs val]
83+
(defressian fs fr-handlers)))]
84+
(recur (inc runs) (+ proc-time dur)))))))))

‎dev/simple_stress_test.clj‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎dev/simple_stress_test_converter.clj‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
(ns simple-stress-test-converter
2+
(:require [aux :as a]
3+
[clojure.edn :as edn]
4+
[metrics :as m])
5+
(:import [org.fressian.handlers ConvertList ILookup]))
6+
7+
(def custom-read-handlers
8+
(reify ILookup
9+
(valAt [_ k]
10+
(get {"fressian/list"
11+
(reify ConvertList
12+
(convertList [_ items]
13+
(vec items)))}
14+
k))))
15+
16+
(defn -main [& args]
17+
(let [parsed-args (map edn/read-string args)
18+
argmap (zipmap [:struct-type :struct-size :convert-list?] parsed-args)]
19+
(m/initialize-and-return-metrics)
20+
(a/stress-test argmap custom-read-handlers)
21+
(System/exit 0)))

‎dev/simple_stress_test_reducer.clj‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
(ns simple-stress-test-reducer
2+
(:require [aux :as a]
3+
[clojure.core.async :as async]
4+
[clojure.edn :as edn]
5+
[metrics :as m])
6+
(:import [java.io ByteArrayInputStream ByteArrayOutputStream]
7+
[org.fressian FressianReader FressianWriter]
8+
[org.fressian.handlers ILookup IReduceList]))
9+
10+
(def custom-read-handlers
11+
(reify ILookup
12+
(valAt [_ k]
13+
(get {"fressian/list" (reify IReduceList
14+
(init [_ len]
15+
(transient []))
16+
17+
(step [_ acc o]
18+
(conj! acc o))
19+
20+
(complete [_ acc]
21+
(persistent! acc)))}
22+
k))))
23+
24+
(defn -main [& args]
25+
(let [parsed-args (map edn/read-string args)
26+
argmap (zipmap [:struct-type :struct-size :convert-list?] parsed-args)]
27+
(m/initialize-and-return-metrics)
28+
(a/stress-test argmap custom-read-handlers)
29+
(System/exit 0)))

‎project.clj‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@
2424
:jvm-opts ["-server"
2525
"-Xms8g" "-Xmx8g"
2626
"-XX:+UseZGC"]}
27-
:fressian-new {:dependencies [[org.fressian/fressian "0.6.9-SNAPSHOT"]]
28-
:main simple-stress-test}
27+
:fressian-convert {:dependencies [[org.fressian/fressian "0.6.9-IConvertList"]]
28+
:main simple-stress-test-converter}
29+
:fressian-reduce {:dependencies [[org.fressian/fressian "0.6.9-IReduceList"]]
30+
:main simple-stress-test-reducer}
2931
:fressian-old {:dependencies [[org.fressian/fressian "0.6.8"]]
3032
:main simple-stress-test}}
3133

‎resources/k8s.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ kind: Job
33
metadata:
44
name: fressian-test
55
spec:
6-
completions: 6
6+
completions: 10
77
parallelism: 6
88
completionMode: Indexed
99
template:

0 commit comments

Comments
 (0)