Send a default User-Agent, as the JDK does

README.md

@@ -114,6 +114,12 @@ Restricted request headers behave as `java.net.http` does: setting
 `content-length`, `connection`, `host`, `upgrade` or `expect` on a request is an
 `IllegalArgumentException`, because the client owns them.
 
+A request whose caller sets no `User-Agent` names jolt, the way the JDK names
+itself: `Jolt/<version>` through `HttpURLConnection` (and so `jolt.http-client`),
+after the `http.agent` property when that is set, and `Jolt-http-client/<version>`
+through `java.net.http`. A `User-Agent` the caller sets is sent instead. Some
+APIs' CDNs refuse a request with none.
+
 ## Connections
 
 Connections are pooled and reused per origin. HTTP/1.1 is persistent by default,

src/jolt/http/core.clj

@@ -942,6 +942,35 @@
                                (or (not= "HTTP/1.0" version)
                                    (str/includes? conn-hdr "keep-alive"))))})))
 
+;; --- User-Agent --------------------------------------------------------------
+;; The JDK names itself on every request a caller leaves unnamed: HttpURLConnection
+;; as `Java/<version>`, after the http.agent property when that is set, and
+;; java.net.http as `Java-http-client/<version>`. Sending none at all is not
+;; neutral: a CDN in front of an API commonly answers a request with no
+;; User-Agent with a 403 bot-challenge page (Nous Research's inference API does),
+;; so code that works on the JVM failed under jolt. The two shims name jolt the
+;; same way, with jolt's version, since this is not the JDK's.
+(def ^:private runtime-version
+  (delay (let [v (str (try (jolt.host/jolt-version) (catch Throwable _ nil)))]
+           (if (str/blank? v) "dev" (str/replace v #"^v" "")))))
+
+(defn default-user-agent
+  "The User-Agent a request goes out with when its caller set none, for the
+  shim `kind`: :url-connection (HttpURLConnection, which clj-http-lite drives)
+  or :http-client (java.net.http)."
+  [kind]
+  (let [base (str (if (= kind :http-client) "Jolt-http-client/" "Jolt/") @runtime-version)
+        agent (when (= kind :url-connection) (System/getProperty "http.agent"))]
+    (if (str/blank? agent) base (str agent " " base))))
+
+(defn with-user-agent
+  "`headers`, [name value] pairs, with a User-Agent of `ua` added unless one,
+  of any case, is there already: a caller's own always wins."
+  [headers ua]
+  (if (some (fn [p] (= "user-agent" (str/lower-case (str (first p))))) headers)
+    headers
+    (conj (vec headers) ["User-Agent" ua])))
+
 (defn build-request
   "Serialise one HTTP/1.1 request. `absolute-target`, when given, replaces the
   origin-form path with a full URI — the request-target form a plain-http proxy

src/jolt/http/jdk.clj

@@ -526,7 +526,9 @@
   (let [ok (atom false)
         streaming (atom false)]
     (try
-      (core/s-write stream (core/build-request method url headers body
+      (core/s-write stream (core/build-request method url
+                                               (core/with-user-agent headers (core/default-user-agent :http-client))
+                                               body
                                                (when absolute? (core/spec-no-ref url))))
       (let [resp (core/read-response stream deadline method received
                                      (boolean stream-body?) (boolean body-unbounded?))]

src/jolt/http/platform.clj

@@ -102,7 +102,10 @@
                  (let [ok (atom false)
                        streaming (atom false)]
                    (try
-                     (s-write stream (build-request method url (tget conn :req-headers) body))
+                     (s-write stream (build-request method url
+                                                    (core/with-user-agent (tget conn :req-headers)
+                                                                          (core/default-user-agent :url-connection))
+                                                    body))
                      (let [r (read-response stream nil method received true)]
                        (if (:body-pull r)
                          (do (reset! streaming true)

test/jolt/http/core_test.clj

@@ -431,3 +431,15 @@
     (is (= [] (vec (core/concat-bas []))))
     (is (= [2 3] (vec (core/sub-ba (byte-array [1 2 3 4]) 1 3))))
     (is (= [] (vec (core/sub-ba (byte-array [1 2 3]) 2 2))))))
+
+(deftest a-request-names-its-client
+  (let [v (str/replace (str (jolt.host/jolt-version)) #"^v" "")]
+    (is (= (str "Jolt/" v) (core/default-user-agent :url-connection)) "as HttpURLConnection's Java/<version>")
+    (is (= (str "Jolt-http-client/" v) (core/default-user-agent :http-client)) "as java.net.http's Java-http-client/<version>"))
+  (is (= [["Accept" "*/*"] ["User-Agent" "Jolt/x"]] (core/with-user-agent [["Accept" "*/*"]] "Jolt/x")))
+  (testing "a caller's own wins, in any case"
+    (is (= [["user-agent" "mine"]] (core/with-user-agent [["user-agent" "mine"]] "Jolt/x")))
+    (is (= [["USER-AGENT" "mine"]] (core/with-user-agent [["USER-AGENT" "mine"]] "Jolt/x"))))
+  (is (str/includes? (core/ba->latin1 (core/build-request "GET" (core/parse-url "http://h/")
+                                                          (core/with-user-agent [] "Jolt/x") nil))
+                     "\r\nUser-Agent: Jolt/x\r\n")))

test/jolt/http/test_runner.clj

@@ -13,6 +13,7 @@
             [jolt.http.stream-shim-test]
             [jolt.http.core-test]
             [jolt.http.urlconnection-stream-test]
+            [jolt.http.user-agent-test]
             [clj-http.lite.links-test]
             [clj-http.lite.client-test]
             [clj-http.lite.integration-test]))
@@ -25,6 +26,7 @@
                        'jolt.http.stream-shim-test
                        'jolt.http.core-test
                        'jolt.http.urlconnection-stream-test
+                       'jolt.http.user-agent-test
                        'clj-http.lite.links-test
                        'clj-http.lite.client-test
                        'clj-http.lite.integration-test)]

test/jolt/http/test_server.clj

@@ -135,6 +135,8 @@
       (and (= m :head) (= uri "/head")) {:status 200 :body ""}
       (and (= m :get) (= uri "/content-type")) {:status 200 :body (or (get h "content-type") "")}
       (and (= m :get) (= uri "/header")) {:status 200 :body (or (get h "x-my-header") "")}
+      ;; what the client named itself as, "<none>" when it sent no User-Agent
+      (and (= m :get) (= uri "/user-agent")) {:status 200 :body (or (get h "user-agent") "<none>")}
       (and (= m :post) (= uri "/post")) {:status 200 :body (:body-raw req)}
       (and (= m :get) (= uri "/redirect")) {:status 302 :headers {"Location" "/get"} :body ""}
       (and (= m :get) (= uri "/error")) {:status 500 :body "o noes"}

test/jolt/http/user_agent_test.clj

@@ -0,0 +1,32 @@
+(ns jolt.http.user-agent-test
+  "What a request goes out with when its caller names no User-Agent, through
+  both shims, as the server sees it: the JDK always names itself, and a CDN
+  commonly refuses a request that doesn't (a 403 challenge page)."
+  (:require [clojure.string :as str]
+            [clojure.test :refer [deftest is testing use-fixtures]]
+            [jolt.http-client :as http]
+            [jolt.http.core :as core]
+            [jolt.http.test-server :as srv])
+  (:import [java.net URI]
+           [java.net.http HttpClient HttpRequest HttpResponse$BodyHandlers]))
+
+(def ^:private port 18130)
+(def ^:private base (str "http://localhost:" port))
+
+(use-fixtures :once (fn [t] (let [s (srv/start-plain port)] (try (t) (finally (srv/stop s))))))
+
+(defn- jdk-get [headers]
+  (let [b (reduce (fn [b [k v]] (.header b k v)) (HttpRequest/newBuilder (URI/create (str base "/user-agent"))) headers)]
+    (.body (.send (HttpClient/newHttpClient) (.build b) (HttpResponse$BodyHandlers/ofString)))))
+
+(deftest http-url-connection-names-jolt
+  (is (= (core/default-user-agent :url-connection) (:body (http/get (str base "/user-agent")))))
+  (is (str/starts-with? (:body (http/get (str base "/user-agent"))) "Jolt/"))
+  (testing "a caller's own is sent instead"
+    (is (= "mine/1" (:body (http/get (str base "/user-agent") {:headers {"User-Agent" "mine/1"}}))))
+    (is (= "mine/2" (:body (http/get (str base "/user-agent") {:headers {"user-agent" "mine/2"}}))))))
+
+(deftest java-net-http-names-jolt
+  (is (= (core/default-user-agent :http-client) (jdk-get [])))
+  (is (str/starts-with? (jdk-get []) "Jolt-http-client/"))
+  (is (= "mine/3" (jdk-get [["User-Agent" "mine/3"]])) "a caller's own is sent instead"))