diff --git a/README.md b/README.md index 5350c5b..8d00773 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,12 @@ Restricted request headers behave as `java.net.http` does: setting `content-length`, `connection`, `host`, `upgrade` or `expect` on a request is an `IllegalArgumentException`, because the client owns them. +A request whose caller sets no `User-Agent` names jolt, the way the JDK names +itself: `Jolt/` through `HttpURLConnection` (and so `jolt.http-client`), +after the `http.agent` property when that is set, and `Jolt-http-client/` +through `java.net.http`. A `User-Agent` the caller sets is sent instead. Some +APIs' CDNs refuse a request with none. + ## Connections Connections are pooled and reused per origin. HTTP/1.1 is persistent by default, diff --git a/src/jolt/http/core.clj b/src/jolt/http/core.clj index c8adb41..7ca87ba 100644 --- a/src/jolt/http/core.clj +++ b/src/jolt/http/core.clj @@ -942,6 +942,35 @@ (or (not= "HTTP/1.0" version) (str/includes? conn-hdr "keep-alive"))))}))) +;; --- User-Agent -------------------------------------------------------------- +;; The JDK names itself on every request a caller leaves unnamed: HttpURLConnection +;; as `Java/`, after the http.agent property when that is set, and +;; java.net.http as `Java-http-client/`. Sending none at all is not +;; neutral: a CDN in front of an API commonly answers a request with no +;; User-Agent with a 403 bot-challenge page (Nous Research's inference API does), +;; so code that works on the JVM failed under jolt. The two shims name jolt the +;; same way, with jolt's version, since this is not the JDK's. +(def ^:private runtime-version + (delay (let [v (str (try (jolt.host/jolt-version) (catch Throwable _ nil)))] + (if (str/blank? v) "dev" (str/replace v #"^v" ""))))) + +(defn default-user-agent + "The User-Agent a request goes out with when its caller set none, for the + shim `kind`: :url-connection (HttpURLConnection, which clj-http-lite drives) + or :http-client (java.net.http)." + [kind] + (let [base (str (if (= kind :http-client) "Jolt-http-client/" "Jolt/") @runtime-version) + agent (when (= kind :url-connection) (System/getProperty "http.agent"))] + (if (str/blank? agent) base (str agent " " base)))) + +(defn with-user-agent + "`headers`, [name value] pairs, with a User-Agent of `ua` added unless one, + of any case, is there already: a caller's own always wins." + [headers ua] + (if (some (fn [p] (= "user-agent" (str/lower-case (str (first p))))) headers) + headers + (conj (vec headers) ["User-Agent" ua]))) + (defn build-request "Serialise one HTTP/1.1 request. `absolute-target`, when given, replaces the origin-form path with a full URI — the request-target form a plain-http proxy diff --git a/src/jolt/http/jdk.clj b/src/jolt/http/jdk.clj index ef648ee..b5c08ee 100644 --- a/src/jolt/http/jdk.clj +++ b/src/jolt/http/jdk.clj @@ -526,7 +526,9 @@ (let [ok (atom false) streaming (atom false)] (try - (core/s-write stream (core/build-request method url headers body + (core/s-write stream (core/build-request method url + (core/with-user-agent headers (core/default-user-agent :http-client)) + body (when absolute? (core/spec-no-ref url)))) (let [resp (core/read-response stream deadline method received (boolean stream-body?) (boolean body-unbounded?))] diff --git a/src/jolt/http/platform.clj b/src/jolt/http/platform.clj index 1ee6c83..2199f3b 100644 --- a/src/jolt/http/platform.clj +++ b/src/jolt/http/platform.clj @@ -102,7 +102,10 @@ (let [ok (atom false) streaming (atom false)] (try - (s-write stream (build-request method url (tget conn :req-headers) body)) + (s-write stream (build-request method url + (core/with-user-agent (tget conn :req-headers) + (core/default-user-agent :url-connection)) + body)) (let [r (read-response stream nil method received true)] (if (:body-pull r) (do (reset! streaming true) diff --git a/test/jolt/http/core_test.clj b/test/jolt/http/core_test.clj index 7af6293..8ae7097 100644 --- a/test/jolt/http/core_test.clj +++ b/test/jolt/http/core_test.clj @@ -431,3 +431,15 @@ (is (= [] (vec (core/concat-bas [])))) (is (= [2 3] (vec (core/sub-ba (byte-array [1 2 3 4]) 1 3)))) (is (= [] (vec (core/sub-ba (byte-array [1 2 3]) 2 2)))))) + +(deftest a-request-names-its-client + (let [v (str/replace (str (jolt.host/jolt-version)) #"^v" "")] + (is (= (str "Jolt/" v) (core/default-user-agent :url-connection)) "as HttpURLConnection's Java/") + (is (= (str "Jolt-http-client/" v) (core/default-user-agent :http-client)) "as java.net.http's Java-http-client/")) + (is (= [["Accept" "*/*"] ["User-Agent" "Jolt/x"]] (core/with-user-agent [["Accept" "*/*"]] "Jolt/x"))) + (testing "a caller's own wins, in any case" + (is (= [["user-agent" "mine"]] (core/with-user-agent [["user-agent" "mine"]] "Jolt/x"))) + (is (= [["USER-AGENT" "mine"]] (core/with-user-agent [["USER-AGENT" "mine"]] "Jolt/x")))) + (is (str/includes? (core/ba->latin1 (core/build-request "GET" (core/parse-url "http://h/") + (core/with-user-agent [] "Jolt/x") nil)) + "\r\nUser-Agent: Jolt/x\r\n"))) diff --git a/test/jolt/http/test_runner.clj b/test/jolt/http/test_runner.clj index e9d9e63..8d40e62 100644 --- a/test/jolt/http/test_runner.clj +++ b/test/jolt/http/test_runner.clj @@ -13,6 +13,7 @@ [jolt.http.stream-shim-test] [jolt.http.core-test] [jolt.http.urlconnection-stream-test] + [jolt.http.user-agent-test] [clj-http.lite.links-test] [clj-http.lite.client-test] [clj-http.lite.integration-test])) @@ -25,6 +26,7 @@ 'jolt.http.stream-shim-test 'jolt.http.core-test 'jolt.http.urlconnection-stream-test + 'jolt.http.user-agent-test 'clj-http.lite.links-test 'clj-http.lite.client-test 'clj-http.lite.integration-test)] diff --git a/test/jolt/http/test_server.clj b/test/jolt/http/test_server.clj index 7727b9d..b424389 100644 --- a/test/jolt/http/test_server.clj +++ b/test/jolt/http/test_server.clj @@ -135,6 +135,8 @@ (and (= m :head) (= uri "/head")) {:status 200 :body ""} (and (= m :get) (= uri "/content-type")) {:status 200 :body (or (get h "content-type") "")} (and (= m :get) (= uri "/header")) {:status 200 :body (or (get h "x-my-header") "")} + ;; what the client named itself as, "" when it sent no User-Agent + (and (= m :get) (= uri "/user-agent")) {:status 200 :body (or (get h "user-agent") "")} (and (= m :post) (= uri "/post")) {:status 200 :body (:body-raw req)} (and (= m :get) (= uri "/redirect")) {:status 302 :headers {"Location" "/get"} :body ""} (and (= m :get) (= uri "/error")) {:status 500 :body "o noes"} diff --git a/test/jolt/http/user_agent_test.clj b/test/jolt/http/user_agent_test.clj new file mode 100644 index 0000000..e8e5f88 --- /dev/null +++ b/test/jolt/http/user_agent_test.clj @@ -0,0 +1,32 @@ +(ns jolt.http.user-agent-test + "What a request goes out with when its caller names no User-Agent, through + both shims, as the server sees it: the JDK always names itself, and a CDN + commonly refuses a request that doesn't (a 403 challenge page)." + (:require [clojure.string :as str] + [clojure.test :refer [deftest is testing use-fixtures]] + [jolt.http-client :as http] + [jolt.http.core :as core] + [jolt.http.test-server :as srv]) + (:import [java.net URI] + [java.net.http HttpClient HttpRequest HttpResponse$BodyHandlers])) + +(def ^:private port 18130) +(def ^:private base (str "http://localhost:" port)) + +(use-fixtures :once (fn [t] (let [s (srv/start-plain port)] (try (t) (finally (srv/stop s)))))) + +(defn- jdk-get [headers] + (let [b (reduce (fn [b [k v]] (.header b k v)) (HttpRequest/newBuilder (URI/create (str base "/user-agent"))) headers)] + (.body (.send (HttpClient/newHttpClient) (.build b) (HttpResponse$BodyHandlers/ofString))))) + +(deftest http-url-connection-names-jolt + (is (= (core/default-user-agent :url-connection) (:body (http/get (str base "/user-agent"))))) + (is (str/starts-with? (:body (http/get (str base "/user-agent"))) "Jolt/")) + (testing "a caller's own is sent instead" + (is (= "mine/1" (:body (http/get (str base "/user-agent") {:headers {"User-Agent" "mine/1"}})))) + (is (= "mine/2" (:body (http/get (str base "/user-agent") {:headers {"user-agent" "mine/2"}})))))) + +(deftest java-net-http-names-jolt + (is (= (core/default-user-agent :http-client) (jdk-get []))) + (is (str/starts-with? (jdk-get []) "Jolt-http-client/")) + (is (= "mine/3" (jdk-get [["User-Agent" "mine/3"]])) "a caller's own is sent instead"))