Skip to content

Commit 8ff06e3

Browse files
rpardinirpardini
committed
add mitmproxy/nginx-debug inspection capabilities
- avoid some caching for non-blob urls
1 parent 1486d69 commit 8ff06e3

4 files changed

Lines changed: 107 additions & 44 deletions

File tree

‎Dockerfile‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
# We start from my nginx fork which includes the proxy-connect module from tEngine
22
# Source is available at https://github.com/rpardini/nginx-proxy-connect-stable-alpine
3-
# Its equivalent to nginx:stable-alpine 1.14.0, with alpine 3.7
4-
FROM rpardini/nginx-proxy-connect-stable-alpine:latest
3+
FROM rpardini/nginx-proxy-connect-stable-alpine:nginx-1.14.0-alpine-3.8
54

65
# Add openssl, bash and ca-certificates, then clean apk cache -- yeah complain all you want.
7-
RUN apk add --update openssl bash ca-certificates && rm -rf /var/cache/apk/*
6+
# Also added deps for mitmproxy.
7+
RUN apk add --update openssl bash ca-certificates su-exec git g++ libffi libffi-dev libstdc++ openssl openssl-dev python3 python3-dev
8+
RUN LDFLAGS=-L/lib pip3 install mitmproxy
9+
RUN apk del --purge git g++ libffi-dev openssl-dev python3-dev && rm -rf /var/cache/apk/* && rm -rf ~/.cache/pip
10+
11+
# Required for mitmproxy
12+
ENV LANG=en_US.UTF-8
13+
14+
# Check the installed mitmproxy version
15+
RUN mitmproxy --version
816

917
# Create the cache directory and CA directory
1018
RUN mkdir -p /docker_mirror_cache /ca
@@ -27,13 +35,18 @@ RUN chmod +x /create_ca_cert.sh /entrypoint.sh
2735
# Clients should only use 3128, not anything else.
2836
EXPOSE 3128
2937

38+
# In debug mode, 8081 exposes the mitmweb interface.
39+
EXPOSE 8081
40+
3041
## Default envs.
3142
# A space delimited list of registries we should proxy and cache; this is in addition to the central DockerHub.
3243
ENV REGISTRIES="k8s.gcr.io gcr.io quay.io"
3344
# A space delimited list of registry:user:password to inject authentication for
3445
ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.registry:user:password"
3546
# Should we verify upstream's certificates? Default to true.
3647
ENV VERIFY_SSL="true"
48+
# Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
49+
ENV DEBUG="true"
3750

3851
# Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
3952
ENTRYPOINT ["/entrypoint.sh"]

‎create_ca_cert.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ if [ -f "$CA_CRT_FILE" ] ; then
3434
else
3535
logInfo "No CA was found. Generating one."
3636
logInfo "*** Please *** make sure to mount /ca as a volume -- if not, everytime this container starts, it will regenerate the CA and nothing will work."
37-
37+
3838
openssl genrsa -des3 -passout pass:foobar -out ${CA_KEY_FILE} 4096
3939

4040
logInfo "generate CA cert with key and self sign it: ${CAID}"
@@ -52,7 +52,7 @@ EOF
5252

5353
[[ ${DEBUG} -gt 0 ]] && logInfo "show the CA cert details"
5454
[[ ${DEBUG} -gt 0 ]] && openssl x509 -noout -text -in ${CA_CRT_FILE}
55-
55+
5656
echo 01 > ${CA_SRL_FILE}
5757

5858
fi
@@ -116,3 +116,6 @@ openssl x509 -req -days 365 -in web.csr -CA ia.crt -CAkey ia.key -out web.crt -p
116116

117117
logInfo "Concatenating fullchain.pem..."
118118
cat web.crt ia.crt ${CA_CRT_FILE} > fullchain.pem
119+
120+
logInfo "Concatenating fullchain_with_key.pem"
121+
cat fullchain.pem web.key > fullchain_with_key.pem

‎entrypoint.sh‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ for ONEREGISTRYIN in ${AUTH_REGISTRIES}; do
3636
done
3737

3838
echo "" > /etc/nginx/docker.verify.ssl.conf
39-
if [ "a$VERIFY_SSL" == "atrue" ]; then
39+
if [[ "a${VERIFY_SSL}" == "atrue" ]]; then
4040
cat << EOD > /etc/nginx/docker.verify.ssl.conf
4141
# We actually wanna be secure and avoid mitm attacks.
4242
# Fitting, since this whole thing is a mitm...
@@ -46,11 +46,36 @@ if [ "a$VERIFY_SSL" == "atrue" ]; then
4646
proxy_ssl_verify_depth 2;
4747
EOD
4848
echo "Upstream SSL certificate verification enabled."
49-
fi
49+
fi
50+
51+
# create default config for the caching layer to listen on 443.
52+
echo " listen 443 ssl default_server;" > /etc/nginx/caching.layer.listen
53+
echo "error_log /var/log/nginx/error.log warn;" > /etc/nginx/error.log.debug.warn
54+
55+
# normally use non-debug version of nginx
56+
NGINX_BIN="nginx"
57+
58+
if [[ "a${DEBUG}" == "atrue" ]]; then
59+
# in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
60+
echo " listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
61+
echo "error_log /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
62+
63+
# use debug binary
64+
NGINX_BIN="nginx-debug"
65+
66+
echo "Starting in DEBUG MODE."
67+
echo "Run mitmproxy with reverse pointing to the same certs..."
68+
mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
69+
--set keep_host_header=true --set ssl_insecure=true \
70+
--mode reverse:https://127.0.0.1:444 --listen-host 0.0.0.0 \
71+
--listen-port 443 --certs /certs/fullchain_with_key.pem \
72+
-w /ca/outfile &
73+
echo "Access mitmweb via http://127.0.0.1:8081/ "
74+
fi
5075

5176

5277
echo "Testing nginx config..."
53-
nginx -t
78+
${NGINX_BIN} -t
5479

5580
echo "Starting nginx! Have a nice day."
56-
nginx -g "daemon off;"
81+
${NGINX_BIN} -g "daemon off;"

‎nginx.conf‎

Lines changed: 57 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
user nginx;
22
worker_processes auto;
33

4-
error_log /var/log/nginx/error.log warn;
4+
# error log config comes from external file created by entrypoint, to toggle debug on/off.
5+
include /etc/nginx/error.log.debug.warn;
6+
57
pid /var/run/nginx.pid;
68

79
events {
@@ -13,28 +15,28 @@ http {
1315
include /etc/nginx/mime.types;
1416
default_type application/octet-stream;
1517

16-
# Use a debug-oriented logging format.
18+
# Use a debug-oriented logging format.
1719
log_format debugging '$remote_addr - $remote_user [$time_local] "$request" '
1820
'$status $body_bytes_sent '
1921
'"HOST: $host" "UPSTREAM: $upstream_addr" '
2022
'"UPSTREAM-STATUS: $upstream_status" '
2123
'"SSL-PROTO: $ssl_protocol" '
22-
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
24+
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
2325
'"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
2426
'"AUTH: $http_authorization" ' ;
25-
27+
2628
log_format tweaked '$upstream_cache_status [$time_local] "$uri" '
2729
'$status $body_bytes_sent '
2830
'"HOST:$host" '
2931
'"PROXY-HOST:$proxy_host" "UPSTREAM:$upstream_addr" ';
30-
32+
3133
keepalive_timeout 300;
3234
gzip off;
3335

3436
# The cache directory. This can get huge. Better to use a Docker volume pointing here!
3537
# Set to 32gb which should be enough
3638
proxy_cache_path /docker_mirror_cache levels=1:2 max_size=32g inactive=60d keys_zone=cache:10m use_temp_path=off;
37-
39+
3840
# Just in case you want to rewrite some hosts. Default maps directly.
3941
map $host $targetHost {
4042
hostnames;
@@ -48,7 +50,7 @@ http {
4850
include /etc/nginx/docker.auth.map;
4951
default "";
5052
}
51-
53+
5254
# Map to decide which hosts get directed to the caching portion.
5355
# This is automatically generated from the list of cached registries, plus a few fixed hosts
5456
# By default, we don't intercept, allowing free flow of non-registry traffic
@@ -57,13 +59,13 @@ http {
5759
include /etc/nginx/docker.intercept.map;
5860
default "$connect_host:443";
5961
}
60-
61-
map $dockerAuth $finalAuth {
62+
63+
map $dockerAuth $finalAuth {
6264
"" "$http_authorization"; # if empty, keep the original passed-in from the client
6365
default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
6466
}
65-
66-
67+
68+
6769
# These maps parse the original Host and URI from a /forcecache redirect.
6870
map $request_uri $realHost {
6971
~/forcecacheinsecure/([^:/]+)/originalwas(/.+) $1;
@@ -76,43 +78,44 @@ http {
7678
~/forcecachesecure/([^:/]+)/originalwas(/.+) $2;
7779
default "DID_NOT_MATCH_PATH";
7880
}
79-
80-
81+
82+
8183
# The proxy director layer, listens on 3128
8284
server {
8385
listen 3128;
8486
server_name _;
85-
87+
8688
# dont log the CONNECT proxy.
8789
access_log off;
88-
90+
8991
proxy_connect;
9092
proxy_connect_address $interceptedHost;
9193
proxy_max_temp_file_size 0;
92-
94+
9395
# We need to resolve the real names of our proxied servers.
9496
resolver 8.8.8.8 4.2.2.2 ipv6=off; # Avoid ipv6 addresses for now
9597

9698
# forward proxy for non-CONNECT request
9799
location / {
98100
return 403 "The docker caching proxy is working!";
99101
}
100-
102+
101103
location /ca.crt {
102-
alias /ca/ca.crt;
104+
alias /ca/ca.crt;
103105
}
104106

105107
# @TODO: add a dynamic root path that generates instructions for usage on docker clients
106108
}
107-
109+
108110

109111
# The caching layer
110112
server {
111113
# Listen on both 80 and 443, for all hostnames.
114+
# actually could be 443 or 444, depending on debug. this is now generated by the entrypoint.
112115
listen 80 default_server;
113-
listen 443 ssl default_server;
116+
include /etc/nginx/caching.layer.listen;
114117
server_name _;
115-
118+
116119
# Do some tweaked logging.
117120
access_log /var/log/nginx/access.log tweaked;
118121

@@ -136,17 +139,17 @@ http {
136139
if ($request_method = DELETE) {
137140
return 405 "DELETE method is not allowed";
138141
}
139-
142+
140143
proxy_read_timeout 900;
141-
142-
# Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
144+
145+
# Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
143146
# will wait for the first to finish instead of doing multiple upstream requests.
144147
proxy_cache_lock on;
145148
proxy_cache_lock_timeout 120s;
146149

147150
# Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
148-
proxy_cache_valid 200 301 302 307 60d;
149-
151+
proxy_cache_valid 200 301 302 307 60d;
152+
150153
# Some extra settings to maximize cache hits and efficiency
151154
proxy_force_ranges on;
152155
proxy_ignore_client_abort on;
@@ -155,13 +158,13 @@ http {
155158
# Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
156159
proxy_hide_header Set-Cookie;
157160
proxy_ignore_headers X-Accel-Expires Expires Cache-Control Set-Cookie;
158-
161+
159162
# Add the authentication info, if the map matched the target domain.
160163
proxy_set_header Authorization $finalAuth;
161-
164+
162165
# This comes from a include file generated by the entrypoint.
163166
include /etc/nginx/docker.verify.ssl.conf;
164-
167+
165168
# Some debugging info
166169
# add_header X-Docker-Caching-Proxy-Real-Host $realHost;
167170
# add_header X-Docker-Caching-Proxy-Real-Path $realPath;
@@ -176,23 +179,35 @@ http {
176179
# don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
177180
location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
178181
proxy_pass https://$targetHost;
182+
add_header X-Docker-Caching-Proxy-Debug-Cache "no:manifests";
179183
}
180184

181185
# don't cache mutable entity /v2/<name>/tags/list
182186
location ~ ^/v2/[^\/]+/tags/list {
183187
proxy_pass https://$targetHost;
188+
proxy_cache off;
189+
add_header X-Docker-Caching-Proxy-Debug-Cache "no:tagslist";
184190
}
185191

186192
# don't cache mutable entity /v2/_catalog
187193
location ~ ^/v2/_catalog$ {
188194
proxy_pass https://$targetHost;
195+
proxy_cache off;
196+
add_header X-Docker-Caching-Proxy-Debug-Cache "no:catalog";
189197
}
190-
191-
# force cache of the first hit which is always /v2/ - even for 401 unauthorized.
198+
199+
# dont cache the first hit which is always /v2/
192200
location = /v2/ {
193201
proxy_pass https://$targetHost;
194-
proxy_cache cache;
195-
proxy_cache_valid 200 301 302 307 401 60d;
202+
proxy_cache off;
203+
add_header X-Docker-Caching-Proxy-Debug-Cache "no:rootv2";
204+
}
205+
206+
# dont cache /token (done against auth servers)
207+
location = /token {
208+
proxy_pass https://$targetHost;
209+
proxy_cache off;
210+
add_header X-Docker-Caching-Proxy-Debug-Cache "no:token";
196211
}
197212

198213
# cache everything else
@@ -208,26 +223,33 @@ http {
208223
# We to it twice, one for http and another for https.
209224
proxy_redirect ~^https://([^:/]+)(/.+)$ https://docker.caching.proxy.internal/forcecachesecure/$1/originalwas$2;
210225
proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
226+
227+
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:everythingelse";
211228
}
212229

213230
# handling for the redirect case explained above, with https.
214231
# The $realHost and $realPath variables come from a map defined at the top of this file.
215232
location /forcecachesecure {
216233
proxy_pass https://$realHost$realPath;
217234
proxy_cache cache;
218-
235+
219236
# Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
220237
proxy_cache_key $proxy_host$uri;
238+
239+
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecachesecure";
240+
221241
}
222242

223243
# handling for the redirect case explained above, with http.
224244
# The $realHost and $realPath variables come from a map defined at the top of this file.
225245
location /forcecacheinsecure {
226246
proxy_pass http://$realHost$realPath;
227247
proxy_cache cache;
228-
248+
229249
# Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
230250
proxy_cache_key $proxy_host$uri;
251+
252+
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
231253
}
232254
}
233255
}

0 commit comments

Comments
 (0)