Reverse Proxy, Client IP Adresse in Nginx Logfile

krischanb

New Member
HAProxy community


Hi,
ich habe HAProxy als einen Reverse Proxy Konfiguriert. Nun würde ich gerne in der Nginx Logfile die IP des Client sehen. Dazu habe ich HAProxy wie folgt konfiguriert:

global
log /dev/log local0
log /dev/log local1 notice
chroot /var/lib/haproxy
stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
stats timeout 30s
user haproxy
group haproxy
daemon

defaults
log global
mode tcp
option tcplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
errorfile 504 /etc/haproxy/errors/504.http

frontend http-in
bind :80
mode http
option httplog

#acl is_letsencrypt path -i -m beg /.well-known/acme-challange/
acl is_letsencrypt path_beg -i /.well-known/

redirect scheme https code 301 if !{ ssl_fc } !is_letsencrypt
use_backend be_letsencrypt if is_letsencrypt

frontend https_in
bind *:443 transparent
mode tcp
option tcplog
option http-server-close
option forwardfor header X-Real-IP
http-request set-header X-Real-IP %[src]

acl tls req.ssl_hello_type 1
acl is_nextcloud req.ssl_sni -i my.domain.de

tcp-request inspect-delay 5s
tcp-request content accept if tls

use_backend be_nextcloud if is_nextcloud
use_backend be_openvpn if !{ req.ssl_hello_type 1 } !{ req.len 0 }

backend be_letsencrypt
mode http
server localhost 127.0.0.1:81


backend be_nextcloud
mode tcp
option ssl-hello-chk
option http-server-close
server localhost 127.0.0.1:82 send-proxy

backend be_openvpn
mode tcp
server localhost 127.0.0.1:4545

Meine Nginx server config:
upstream php-handler {
server unix:/run/php/php7.3-fpm.sock;
}
server {
listen 127.0.0.1:82 ssl http2 proxy_protocol;
listen [::]:82 ssl http2 proxy_protocol;
server_name localhost my.domain.de;

# Configure SSL
ssl on;

# Certificates used
ssl_certificate path_to_fullchain.pem;
ssl_certificate_key path_to_key.pem;

ssl_protocols TLSv1.2 TLSv1.3;

ssl_ciphers 'CHIPHER'

ssl_dhparam path_to_dhparams.pem

# Specifies a curve for ECDHE ciphers.
# High security, but will not work with Chrome:
#ssl_ecdh_curve secp521r1;
# Works with Windows (Mobile), but not with Android (DavDroid):
#ssl_ecdh_curve secp384r1;
# Works with Android (DavDroid):
ssl_ecdh_curve prime256v1;

# Server should determine the ciphers, not the client
ssl_prefer_server_ciphers on;

# OCSP Stapling
# fetch OCSP records from URL in ssl_certificate and cache them
ssl_stapling on;
ssl_stapling_verify on;
ssl_trusted_certificate path_to_fullchain.pem;
resolver 192.168.38.1;

# SSL session handling
ssl_session_timeout 24h;
ssl_session_cache shared:SSL:50m;
ssl_session_tickets off;

# Add headers to serve security related headers
#
# HSTS (ngx_http_headers_module is required)
# In order to be recoginzed by SSL test, there must be an index.hmtl in the server's root
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains" always;
add_header X-Content-Type-Options "nosniff" always;
# Usually this should be "DENY", but when hosting sites using frames, it has to be "SAMEORIGIN"
add_header Referrer-Policy "same-origin" always;
add_header X-XSS-Protection "1; mode=block" always;
add_header X-Robots-Tag none;
add_header X-Download-Options noopen;
add_header X-Permitted-Cross-Domain-Policies none;
add_header X-Frame-Options "SAMEORIGIN";

# Path to the root of your installation
root /var/www/;

location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}

# The following 2 rules are only needed for the user_webfinger app.
# Uncomment it if you're planning to use this app.
#rewrite ^/.well-known/host-meta /nextcloud/public.php?service=host-meta last;
#rewrite ^/.well-known/host-meta.json /nextcloud/public.php?service=host-meta-json last;

location = /.well-known/carddav {
return 301 $scheme://$host/nextcloud/remote.php/dav;
}

location = /.well-known/caldav {
return 301 $scheme://$host/nextcloud/remote.php/dav;
}
location /.well-known/acme-challenge { }

location ^~ /nextcloud {
# set max upload size
client_max_body_size 10G;
fastcgi_buffers 64 4K;

# Enable gzip but do not remove ETag headers
gzip on;
gzip_vary on;
gzip_comp_level 4;
gzip_min_length 256;
gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;

# Uncomment if your server is build with the ngx_pagespeed module
# This module is currently not supported.
#pagespeed off;

location /nextcloud {
rewrite ^ /nextcloud/index.php$uri;
}

location ~ ^/nextcloud/(?:build|tests|config|lib|3rdparty|templates|data)/ {
deny all;
}

location ~ ^/nextcloud/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}

location ~ ^/nextcloud/(?:index|remote|public|cron|core/ajax/update|status|ocs/v[12]|updater/.+|ocs-provider/.+|core/templates/40[34])\.php(?:$|/) {
include fastcgi_params;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
#Avoid sending the security headers twice
fastcgi_param modHeadersAvailable true;
fastcgi_param front_controller_active true;
fastcgi_pass php-handler;
fastcgi_intercept_errors on;

# Raise timeout values.
# This is especially important when the Nextcloud setup runs into timeouts (504 gateway errors)
fastcgi_read_timeout 600;
fastcgi_send_timeout 600;
fastcgi_connect_timeout 600;
fastcgi_request_buffering off;

# Pass PHP variables directly to PHP.
# This is usually done in the php.ini. For more flexibility, these variables are configured in the nginx config.
# All the PHP parameters have to be set in one fastcgi_param. When using more 'fastcgi_param PHP_VALUE' directives, the last one will override all the others.
fastcgi_param PHP_VALUE "open_basedir=/var/www:/tmp/:/var/nextcloud_data:/dev/urandom:/proc/meminfo
upload_max_filesize = 10G
post_max_size = 10G
max_execution_time = 3600
output_buffering = off";

# Make sure that the real IP of the remote host is passed to PHP.
#fastcgi_param REMOTE_ADDR $http_x_real_ip;
}

location ~ ^/nextloud/(?:updater|ocs-provider)(?:$|/) {
try_files $uri/ =404;
index index.php;
}

# Adding the cache control header for js and css files
# Make sure it is BELOW the PHP block
location ~* \.(?:css|js)$ {
try_files $uri /nextcloud/index.php$uri$is_args$args;
proxy_set_header Cache-Control "public, max-age=7200";
# Add headers to serve security related headers
# Again use 'proxy_set_header' (not 'add_header') as the headers have to be passed through a proxy.
proxy_set_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;";
proxy_set_header X-Content-Type-Options nosniff;
#proxy_set_header X-Frame-Options "SAMEORIGIN";
proxy_set_header X-XSS-Protection "1; mode=block";
proxy_set_header X-Robots-Tag none;
proxy_set_header X-Download-Options noopen;
proxy_set_header X-Permitted-Cross-Domain-Policies none;
# Optional: Don't log access to assets
access_log off;
}
location ~* \.(?:svg|gif|png|html|ttf|woff|ico|jpg|jpeg)$ {
try_files $uri /nextcloud/index.php$uri$is_args$args;
# Optional: Don't log access to other assets
access_log off;
}
}
}

Auszug aus nginx.conf:
log_format main '$http_x_real_ip - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log main;


Auszug aus nginx access.log
"- - - [26/Feb/2021:08:07:16 +0100] "GET / HTTP/2.0" 403 181 "-" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78..3904.108 Safari/537.36"

client hello captured mit tcpdump während Zugriff auf den Server:
Frame 4: 631 bytes on wire (5048 bits), 631 bytes captured (5048 bits)
Ethernet II, Src: 00:00:00_00:00:00 (00:00:00:00:00:00), Dst: 00:00:00_00:00:00 (00:00:00:00:00:00)
Destination: 00:00:00_00:00:00 (00:00:00:00:00:00)
Source: 00:00:00_00:00:00 (00:00:00:00:00:00)
Type: IPv4 (0x0800)
Internet Protocol Version 4, Src: 127.0.0.1, Dst: 127.0.0.1
Transmission Control Protocol, Src Port: 50396, Dst Port: 82, Seq: 1, Ack: 1, Len: 565
PROXY Protocol
PROXY v1 magic
Protocol: TCP4
Source Address: 109.40.1.110
Destination Address: 192.168.38.2
Source Port: 27031
Destination Port: 443
Transport Layer Security

Hat jemand eine Idee, wo mein Fehler liegt?
Thank you
Krischanb
 

d4f

Kaffee? Wo?
option forwardfor header X-Real-IP
http-request set-header X-Real-IP %[src]
Entferne den http-request set-header, dieser kann sich mit forwardfor beissen und ist unnötig.
Ich nehme an du testest auf HTTPS-Port, da HAProxy auf Port80 nicht für X-Real-IP konfiguriert ist?

Zwei Kuriositäten am Rand:
- Ich empfehle X-Forwarded-For und nicht X-Real-IP zu verwenden. Viele Applikationen unterstützen ersteres nativ was das Leben vereinfachen kann.
- Warum benutzt du nicht Nginx direkt als Proxy für die anderen Dienste?
 

krischanb

New Member
Hi,

option forwardfor header X-Real-IP
habe ich durch
option forwardfor header X-Forwarded-For
ersetzt und
http-request set-header X-Real-IP %[src]
entfernt, leider ohne Veränderung im Logfile nach Neustart von HAProxy und testweise auch Nginx.

- - - [05/Mar/2021:15:22:55 +0100] "GET /nextcloud/csrftoken HTTP/2.0" 200 122 "https://my.domain.de/nextcloud/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"

im TCPDump steht die Source Address im Client Hello:
Frame 4: 628 bytes on wire (5024 bits), 628 bytes captured (5024 bits)
Ethernet II, Src: 00:00:00_00:00:00 (00:00:00:00:00:00), Dst: 00:00:00_00:00:00 (00:00:00:00:00:00)
Internet Protocol Version 4, Src: 127.0.0.1, Dst: 127.0.0.1
Transmission Control Protocol, Src Port: 51476, Dst Port: 82, Seq: 1, Ack: 1, Len: 562
PROXY Protocol
PROXY v1 magic
Protocol: TCP4
Source Address: 93.236.209.22
Destination Address: 192.168.38.2
Source Port: 61296
Destination Port: 443
Transport Layer Security

Ich nehme an du testest auf HTTPS-Port, da HAProxy auf Port80 nicht für X-Real-IP konfiguriert ist?
Bei der Frage stehe ich gerade auf dem Schlauch. Was meinst du mit Testen?
Warum benutzt du nicht Nginx direkt als Proxy für die anderen Dienste?
Hat sich so ergeben. Später soll das noch um einen LB erweritert werden.

Hast du noch eine weitere Idee?
Danke und Gruß
Krischanb
 

d4f

Kaffee? Wo?
Bei der Frage stehe ich gerade auf dem Schlauch. Was meinst du mit Testen?
Deine Konfiguration sieht aktuell X-Forwarded-For lediglich auf SSL vor soweit ich korrekt lese, ansonsten müsste es entweder in beiden Frontends oder im default enthalten sein. Das ist aktuell allerdings etwas schwer zu erkennen da die Konfiguration nicht leserlich (eingerückt) ist.
Allerdings läuft 443 auf Modus TCP so dass das nie zum Zuge kommt.

Hat sich so ergeben. Später soll das noch um einen LB erweritert werden.
Nginx beherrscht ebenfalls loadbalancing. Ich bevorzuge generell möglichst wenige unterschiedliche Produkte zu verwenden, aber das ist Geschmackssache.

im TCPDump steht die Source Address im Client Hello:
Du hast einen TCP Proxy hier, was du brauchst ist ein HTTP Proxy.
 

krischanb

New Member
Hi,
dann hatte ich dich doch richtig verstanden und du lagst richtig mit deiner Vermutung es ist schlicht in dem Teil untergegangen. Ich habe versucht deine Anmerkungen umzusetzen.

So sieht die aktuelle HAproxy Konfiguration aus:
global log /dev/log local0 log /dev/log local1 notice chroot /var/lib/haproxy stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners stats timeout 30s user haproxy group haproxy daemon # Default SSL material locations ##ca-base /etc/ssl/certs ##crt-base /etc/ssl/private # Default ciphers to use on SSL-enabled listening sockets. # For more information, see ciphers(1SSL). This list is from: # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # An alternative list with additional directives can be obtained from # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS ssl-default-bind-options no-sslv3 defaults log global mode http option httplog option dontlognull option forwardfor header X-Forwarded-For timeout connect 5000 timeout client 50000 timeout server 50000 errorfile 400 /etc/haproxy/errors/400.http errorfile 403 /etc/haproxy/errors/403.http errorfile 408 /etc/haproxy/errors/408.http errorfile 500 /etc/haproxy/errors/500.http errorfile 502 /etc/haproxy/errors/502.http errorfile 503 /etc/haproxy/errors/503.http errorfile 504 /etc/haproxy/errors/504.http frontend http-in bind :80 acl is_letsencrypt path_beg -i /.well-known/ redirect scheme https code 301 if !{ ssl_fc } !is_letsencrypt use_backend be_letsencrypt if is_letsencrypt frontend https_in bind *:443 ssl crt /etc/letsencrypt/my.domain.de/haproxy.pem option http-server-close acl tls req.ssl_hello_type 1 #acl is_nextcloud req.ssl_sni -i my.domain.de acl is_nextcloud path_beg -i /nextcloud/ tcp-request inspect-delay 5s tcp-request content accept if tls use_backend be_nextcloud if is_nextcloud #use_backend be_openvpn if !{ req.ssl_hello_type 1 } !{ req.len 0 } #use_backend be_nextcloud #default_backend be_openvpn backend be_letsencrypt server localhost 127.0.0.1:81 backend be_nextcloud server localhost 127.0.0.1:82 send-proxy backend be_openvpn server localhost 127.0.0.1:4545


So sieht jetzt Nginx Konfiguration aus:
upstream php-handler { server unix:/run/php/php7.3-fpm.sock; } server { listen 127.0.0.1:82 proxy_protocol; listen [::]:82 proxy_protocol; server_name localhost my.domain.de; # Path to the root of your installation root /var/www/; location = /robots.txt { allow all; log_not_found off; access_log off; } location = /.well-known/carddav { return 301 $scheme://$host/nextcloud/remote.php/dav; } location = /.well-known/caldav { return 301 $scheme://$host/nextcloud/remote.php/dav; } location /.well-known/acme-challenge { } location ^~ /nextcloud { # set max upload size client_max_body_size 10G; fastcgi_buffers 64 4K; # Enable gzip but do not remove ETag headers gzip on; gzip_vary on; gzip_comp_level 4; gzip_min_length 256; gzip_proxied expired no-cache no-store private no_last_modified no_etag auth; gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy; # Uncomment if your server is build with the ngx_pagespeed module # This module is currently not supported. #pagespeed off; location /nextcloud { rewrite ^ /nextcloud/index.php$uri; } location ~ ^/nextcloud/(?:build|tests|config|lib|3rdparty|templates|data)/ { deny all; } location ~ ^/nextcloud/(?:\.|autotest|occ|issue|indie|db_|console) { deny all; } location ~ ^/nextcloud/(?:index|remote|public|cron|core/ajax/update|status|ocs/v[12]|updater/.+|ocs-provider/.+|core/templates/40[34])\.php(?:$|/) { include fastcgi_params; fastcgi_split_path_info ^(.+\.php)(/.+)$; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; #Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; fastcgi_param front_controller_active true; fastcgi_pass php-handler; fastcgi_intercept_errors on; # Raise timeout values. # This is especially important when the Nextcloud setup runs into timeouts (504 gateway errors) fastcgi_read_timeout 600; fastcgi_send_timeout 600; fastcgi_connect_timeout 600; fastcgi_request_buffering off; # Pass PHP variables directly to PHP. # This is usually done in the php.ini. For more flexibility, these variables are configured in the nginx config. # All the PHP parameters have to be set in one fastcgi_param. When using more 'fastcgi_param PHP_VALUE' directives, the last one will override all the others. fastcgi_param PHP_VALUE "open_basedir=/var/www:/tmp/:/var/nextcloud_data:/dev/urandom:/proc/meminfo upload_max_filesize = 10G post_max_size = 10G max_execution_time = 3600 # Make sure that the real IP of the remote host is passed to PHP. #fastcgi_param REMOTE_ADDR $http_x_real_ip; } location ~ ^/nextloud/(?:updater|ocs-provider)(?:$|/) { try_files $uri/ =404; index index.php; } # Adding the cache control header for js and css files # Make sure it is BELOW the PHP block location ~* \.(?:css|js)$ { try_files $uri /nextcloud/index.php$uri$is_args$args; proxy_set_header Cache-Control "public, max-age=7200"; # Add headers to serve security related headers # Again use 'proxy_set_header' (not 'add_header') as the headers have to be passed through a proxy. proxy_set_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;"; proxy_set_header X-Content-Type-Options nosniff; #proxy_set_header X-Frame-Options "SAMEORIGIN"; proxy_set_header X-XSS-Protection "1; mode=block"; proxy_set_header X-Robots-Tag none; proxy_set_header X-Download-Options noopen; proxy_set_header X-Permitted-Cross-Domain-Policies none; # Optional: Don't log access to assets access_log off; } location ~* \.(?:svg|gif|png|html|ttf|woff|ico|jpg|jpeg)$ { try_files $uri /nextcloud/index.php$uri$is_args$args; # Optional: Don't log access to other assets access_log off; } } }

Das Ergebnis im Log ist unverändert. Oder muss ich den Eintrag in der Nginx.conf anpassen?
log_format main '$http_x_real_ip - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"';

Du hast einen TCP Proxy hier, was du brauchst ist ein HTTP Proxy.
Wenn ich alles auf http umstelle, bekomme ich ein Problem mit dem VPN. Dieser muss über TCP laufen. Wobei es müsste doch möglich sein, bei einem Teil SSL zu terminieren und bei einem anderen Teil passthrough zu definieren.

ich danke für deine Geduld sowie Hilfestellung und hoffe ich habe es diesmal übersichtlicher formatiert.

Gruß
Krischanb
 

krischanb

New Member
Hi,
so ich konnte das Problem lösen. Habe den Wald vor lauter Bäumen nicht gesehen.
Letztendlich lag der Fehler in:
log_format main '$http_x_real_ip - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"';

$http_x_real_ip ersetzt duch $http_x_forwarded_for löste das Problem.

HAProxy läuft nun als SNI Switch, die Backends recirclen den Traffic zu einem anderen Frontend, von denen derzeit eines TCP für OpenVPN und das andere HTTP für die restlichen Anwendungen bedient. So habe ich TCP und http auf einer öffentlichen IP auf Port 443.

Gruß und nochmals Danke für die Tipps.
Krischanb
 
Top