nginx.conf

# Nginx configuration optimized for use on Kubernetes
#
# See
# - https://www.revsys.com/12days/nginx-tuning/
# - https://www.nginx.com/blog/testing-performance-nginx-ingress-controller-kubernetes/
# - https://docs.cloudposse.com/kubernetes-optimization/tune-nginx

# Set the worker processes based on the number of CPU cores
# Setting to `auto` will calculate it automatically
worker_processes auto;

# Number of file descriptors used for Nginx
# The limit for the maximum descriptors on the server is usually set by the OS.
# If you don't set descriptors, then OS settings will be used which is by default 2000
worker_rlimit_nofile 102400;

events {
    # Optmize to serve many clients with each thread
    use epoll;

    # Accept as many connections as possible
    multi_accept on;

    # Increase the number of worker connections from the default to avoid the
    # warning "1024 worker_connections are not enough, reusing connections"
    # During stress testing we found that this limit got hit even when CPU usage
    # was very low. This limit is higher than normally suggested but same that used in 
    # https://www.nginx.com/blog/testing-performance-nginx-ingress-controller-kubernetes/
    worker_connections 100000;
}

http {
  # Logging to stderr and stdout enables better integration with
  # Docker and Kubernetes.
  error_log  /dev/stderr warn;
  access_log /dev/stdout;

  # Send all headers at once
  tcp_nopush on;

  # Don't buffer data when sending. Good for small data bursts in real time
  tcp_nodelay on;

  # Reduce the amount of data sent over network
  gzip on;
  gzip_min_length 10240;
  gzip_proxied expired no-cache no-store private auth;
  gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/json application/xml;
  gzip_disable msie6;

  # Allow the server to close connection on non responding client. This will free up memory
  reset_timedout_connection on;

  # Request time outs. Default 60
  client_body_timeout 120;
  send_timeout 120;

  # Tune keepalives to work with the GCP HTTP(S) Load Balancer
  # The 650 is not arbitrary. For more details see
  # https://blog.percy.io/tuning-nginx-behind-google-cloud-platform-http-s-load-balancer-305982ddb340
  keepalive_timeout 650;
  keepalive_requests 10000;

  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;

  # Get the real IP address of the client.
  # Necessary for correct functioning of request throttling,
  # basic auth exceptions etc.
  # Look for client IP in the X-Forwarded-For header,
  # ignore trusted IPs, set private IPs and load balancer IPs
  # as trusted IPs.
  # See https://blog.jayway.com/2014/03/28/how-to-get-the-client-ip-when-using-cloudfront-and-nginx/
  # and https://geko.cloud/forward-real-ip-to-a-nginx-behind-a-gcp-load-balancer/
  real_ip_header X-Forwarded-For;
  real_ip_recursive on;
  # Private IPs
  set_real_ip_from 10.0.0.0/8;
  # GCP Load Balancer private IPs
  set_real_ip_from 35.191.0.0/16;
  set_real_ip_from 130.211.0.0/22;
  # GCP Load Balancer public IPs
  set_real_ip_from 34.120.0.0/13;

  # Basic authentication
  # Usually off but may be used in staging.  
  # Turn off for private IPs (health checkers that
  # will fail otherwise)
  geo $auth {
    default ${BASIC_AUTH};
    10.0.0.0/8 off;
  }

  # Authentication header sent on the manager
  # Do not send the Basic auth header used to secure this service
  # but send on Token and Bearer auth headers.
  map $http_authorization $manager_auth_header {
    default $http_authorization;
    ~^Basic\s+ '';
  }

  server {
    # Listen on production port so in development can
    # run container using `--net host` for internal redirects
    listen 9000;

    # Basic authentication
    auth_basic $auth;
    auth_basic_user_file /etc/nginx/.htpasswd;

    # Do not merge slashes in URL paths 
    # e.g http://hub.stenci.la/open/github://user/repo/
    merge_slashes off;

    # Increase the upload file size limit
    client_max_body_size 100M;

    # Custom error page for file upload limit.
    error_page 413 /413.html;

    # Custom error page for other errors.
    # 500 errors may be caused by a failure upstream but can also include
    # configuration errors in this file.
    # 502 and 504 errors are usually caused by a failures of upstream
    # services (e.g. a timeout) but may not be reported by those services.
    # This custom error page avoids the default AND embeds Javascript to
    # trigger an exception report.
    error_page 500 502 504 /50x.html;

    location ~ ^/(413|50x).html {
      root /usr/share/nginx/html;
      internal;
    }

    # Access to the monitor's HTTP API and graphing 
    # interface
    location /internal/monitor {
      proxy_pass ${MONITOR_URL};
    }

    location @jobs-connect {
      # Internal proxy URL to jobs to be used with `X-Accel-Redirect`
      # to restrict access.
      internal;

      # Pass on the upgrade header for Websocket support
      proxy_http_version 1.1;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "Upgrade";

      # Reset the authorization header to avoid sending a non-JWT
      # token to the session. In the future, this may be used
      # to send an actual JWT with claims for the user.
      proxy_set_header Authorization "";
      proxy_pass_header Authorization;

      # Increase timeouts from the default 60s
      # The send and read timeout needs to be the longest to maintain an idle connection:
      # "If the proxied server does not receive/transmit anything within this time, 
      # the connection is closed."
      proxy_connect_timeout       5m;
      proxy_send_timeout          1h;
      proxy_read_timeout          1h;

      # This approach, using a named location and setting an additional
      # header `X-Accel-Redirect-URL` is necessary to be able to use HTTP
      # to connect because it will preserve HTTP methods other than GET i.e POST.
      # That may not necessary for WebSockets (?) but nonetheless it is useful to
      # be able to use HTTP to make requests to the job session (especially
      # for debugging).
      # See https://serverfault.com/a/838447
      set $url $upstream_http_x_accel_redirect_url;
      proxy_pass $url;
    }

    location @jobs-broker {
      # Internal proxy URL to the job queue broker to be used with `X-Accel-Redirect`
      # to restrict access.
      internal;

      # Until we set up a HTTP compatible messaging protocol e.g. STOMP
      # This just echos back the proxy_pass URL
      set $url $upstream_http_x_accel_redirect_url;
      return 302 $url;
    }

    location @account-content {
      # Internal reverse-proxy to account content URLs as returned
      # by the `manager`.
      internal;

      # Unset Authorization and Cookie for security reasons.
      proxy_set_header Authorization '';
      proxy_set_header Cookie '';

      # DNS resolver required for the following proxy pass
      # if it is to a remote domain.
      resolver ${RESOLVER_ADDRESS} ipv6=off;

      # Proxy it!
      set $url $upstream_http_x_accel_redirect_url;
      proxy_pass $url;

      # Handle any redirects from upstream
      proxy_intercept_errors on;
      error_page 301 302 307 = @handle-redirect;
    }

    location @handle-redirect {
      # Handle redirect responses
      internal;

      # DNS resolver
      resolver ${RESOLVER_ADDRESS} ipv6=off;

      # Proxy to the Location header
      set $location $upstream_http_location;
      proxy_pass $location;
    }

    location / {
      # Proxy everything else to the `manager` service
      
      # Due to getting the “upstream sent too big header while reading response header from upstream”
      # error the buffers are set based on the reccomendations at
      # https://ma.ttias.be/nginx-proxy-upstream-sent-big-header-reading-response-header-upstream/
      proxy_buffer_size          128k;
      proxy_buffers              4 256k;
      proxy_busy_buffers_size    256k;

      proxy_set_header Authorization $manager_auth_header;
      proxy_set_header Host $http_host;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header X-Forwarded-For $remote_addr;
      proxy_set_header X-Scheme $scheme;
      proxy_pass http://${MANAGER_HOST};
    }
  }

  server {
    # Serve account content:
    #  Production: `<account>.stencila.io/<project>` 
    #  Staging: `<account>.account-test.stenci.la/<project>` 

    listen 9000;
    server_name *.stencila.io *.account-test.stenci.la;

    # Robots, favicon, static assets (needed for error pages)
    # just get proxied through.
    location ~ ^/(robots.txt|favicon.ico|static) {
      proxy_set_header Host $http_host;
      proxy_pass http://${MANAGER_HOST}$request_uri;
    }

    # Proxy all other requests get "rewritten" to the
    # manager's `/content` URL.
    location / {
      proxy_set_header Host $http_host;
      proxy_pass http://${MANAGER_HOST}/content$request_uri;
    }

    location @account-content {
      # Internal reverse-proxy to account content URLs as returned
      # by the `manager`.
      internal;

      # Unset Authorization and Cookie for security reasons.
      proxy_set_header Authorization '';
      proxy_set_header Cookie '';

      # DNS resolver required for the following proxy pass
      # if it is to a remote domain.
      resolver ${RESOLVER_ADDRESS} ipv6=off;

      # Proxy it!
      set $url $upstream_http_x_accel_redirect_url;
      proxy_pass $url;

      # Handle any redirects from upstream
      proxy_intercept_errors on;
      error_page 301 302 307 = @handle-redirect;
    }

    location @handle-redirect {
      # Handle redirect responses
      internal;

      # DNS resolver
      resolver ${RESOLVER_ADDRESS} ipv6=off;

      # Proxy to the Location header
      set $location $upstream_http_location;
      proxy_pass $location;
    }
  }
}