Oauth proxy and monitoring init

47 files changed, 812 insertions, 77 deletions

diff --git a/.gitignore b/.gitignore
index a757fe8..d789ea5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@
 .env
 secrets.enc
 secrets.pwd
+venv
+.venv
diff --git a/deploy.yml b/deploy.yml
index 94e0bbc..cd5f26f 100644
--- a/deploy.yml
+++ b/deploy.yml
@@ -65,3 +65,6 @@
 
 - name: mon
   ansible.builtin.import_playbook: playbooks/mon.yml
+
+- name: coffee
+  ansible.builtin.import_playbook: playbooks/coffee.yml
diff --git a/group_vars/all.yml b/group_vars/all.yml
index a285422..d1c7a24 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -11,7 +11,7 @@ ansible_user: serve
 # -- <networking> --
 loadbalancer_ip: "10.128.0.200"
 homelab_network: "10.128.0.0/16"
-swarm_network: "10.0.0.0/16"
+swarm_network: "10.0.0.0/8"
 docker_network: "172.16.0.0/12"
 headnet_network: "100.64.0.0/10"
 rfc1918_cgnat_networks:
@@ -29,6 +29,8 @@ headscale_nodes_domain: "in.{{ domain }}"
 mail_domain: "mail.{{ domain }}"
 oci_domain: "oci.{{ domain }}"
 passwd_domain: "passwd.{{ domain }}"
+oauth_proxy_domain: "fwdauth.{{ domain }}"
+outbound_domain: "outbound.{{ domain }}"
 # -- </shared_services> --
 
 # -- <docker> --
@@ -56,6 +58,10 @@ homelab_build: false
 deployment_time: "{{ now(utc=true,fmt='%s') }}"
 # -- </unique_deployment> --
 
+# -- <groups> --
+admins: "coffee_admins@{{ idm_domain }}"
+# -- </groups> --
+
 # -- <keys> --
 me_lizcoffee_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDRHu3h9mDjQyFbojcxGKW0hPUDfgUmb2WCzd4Dv2qISM3GGt9LjD8o0IbWRNaTf5UyId5lu7wNHtygs5ZDfUVnlfxrI1CmoExuqkYFjy+R9Cu0x1J2w7+MrKPBd5akLCuKTTnXbyv79T0tLb07rCpGHojW8HH6wdDtg0siVqsPqZVTjg7WGbBYqiqlA5p8s+V9xN1q8lTOZrRI0PdgoU8W+1oIr9OHSG1ZeUBQx60izTEwMnWBxY2aA8SQolIVvsJCcMMc/EAnaz/rdJ5IkeqXGslIhUI7WCPHnPWN8CSdwMOLi5BNaOAK7Y2FkfKTUlO7I52BL87Cl3YpMxR0mTDrfSJTSp0B3ZAbUIXDA7biSh04YLwGQVI799vcyJf355A60btPaiuiBgI0am3h0WxnOACg7K6eV023EiUQ24UjlQ8pufHcJ1oDW8v6LHlp/atCWOl9KQIun9UUg8DD8/BLPprc0wzAV6Nco0ZIedouxZuUhduYYvUrLJ+ICpaZg6oPGitVJPIgyyI+WTfjRN4WTj/Z3Yhuj0RqF8b5ea4FNWuJtfF724t7SVnZsYlZGSCqL8gaEzbIATVe3THn5VwbK+S4ELD/9W6MOd6aZcTOK2yP3jlwjcjnW8sLuX+2qNwtSVVa4o5VsRZU40Da+3flzoBsyUwSE3H2PsFPH29lIQ== lizzy@yubikey"
 # -- </keys> --
@@ -68,11 +74,24 @@ mesh:
     forward_dns: true
     split_vpn_dns_to: "10.128.0.44"
     private_records: []
+    public_healthchecks: []
+    private_healthchecks: []
   liz:
     gateway: "{{ loadbalancer_ip }}"
     domain: "{{ domain }}"
     forward_dns: false
     split_vpn_dns_to: "{{ loadbalancer_ip }}"
+    public_healthchecks:
+      - "https://{{ domain }}"
+      - "https://{{ idm_domain }}/status"
+      - "https://{{ headscale_host }}/health"
+      - "https://fwdauth.{{ domain }}/oauth2/sign_in"
+      - "https://test.{{ domain }}/"
+    private_healthchecks:
+      - "https://bin.{{ domain }}"
+      - "https://ci.{{ domain }}"
+      - "https://notes.{{ domain }}"
+      - "https://passwd.{{ domain }}/alive"
     private_records:
       - type: "A"
         name: "piplup.{{ domain }}"
@@ -97,9 +116,6 @@ mesh:
         name: "bin.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
-        name: "ci.{{ domain }}"
-        ip: "{{ loadbalancer_ip }}"
-      - type: "A"
         name: "idm.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
@@ -124,19 +140,51 @@ mesh:
         name: "src.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
+        name: "fwdauth.{{ domain }}"
+        ip: "{{ loadbalancer_ip }}"
+      - type: "A"
         name: "swarm.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
         name: "traefik.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
-        name: "piplup.pocket.{{ domain }}"
-        ip: "10.128.0.101"
-      - type: "A"
-        name: "togepi.pocket.{{ domain }}"
-        ip: "10.128.0.102"
+        name: "prometheus.{{ domain }}"
+        ip: "{{ loadbalancer_ip }}"
       - type: "A"
-        name: "roton.pocket.{{ domain }}"
-        ip: "10.128.0.103"
+        name: "mon.{{ domain }}"
+        ip: "{{ loadbalancer_ip }}"
 # -- </mesh> --
 
+# -- <logo> --
+
+logo: |
+  --|                                                        |--
+  --|                    ~ welcome to ~                      |--
+  --|                                                        |--
+  --|   .-.   _                       .--. .--.              |--
+  --|   :.:  :_;                     : .-': .-'              |--
+  --|   :.:  .-..---.     .--.  .--. : `; : `;.--.  .--.     |--
+  --|   :.:_ : :`-'_.' _ '  ..'' .; :: :  : :' '_.'' '_.'    |--
+  --|   `.__;:_;`.___;:_;`.__.'`.__.':_;  :_;`.__.'`.__.'    |--
+  --|                                                        |--
+  --|        ~₊˚⊹ ⋆˚✿˖°~   -────୨ৎ────-   ~₊˚⊹ ⋆˚✿˖°~        |--
+  --|   ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣀⣀⣀⣀⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                  |--
+  --|   ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⠤⠒⠉⠉⠉⣀⣂⣅⠬⡉⠭⢛⠿⢟⡶⣄⡀⠀⠀⠀⠀  we'll get brewing   |--
+  --|   ⠀⠀⠀⠀⠀⠀⠀⣠⠞⠁⠀⣄⢎⢩⢸⢉⣵⡖⢰⣶⣮⢹⣦⣡⢊⢻⡿⣦⠀⠀⠀  right away!         |--
+  --|   ⠀⠀⠀⠀⠀⠀⢠⡇⠀⠀⢎⠕⢭⢪⡶⠈⢿⣷⣿⠟⣋⣚⣯⣒⣣⡑⢨⢻⡇⠀⣀⣀⠀⠀⠀                  |--
+  --|   ⠀⠀⠀⠀⠀⣀⡼⣧⠀⠄⡊⢼⡩⣾⢌⠳⡜⣉⡠⡜⡞⣵⣊⡧⡠⠝⣣⡾⠁⠀⠻⠿⠗⠀⠀   /) /) (\ (\    |--
+  --|   ⠀⠀⠀⣢⣾⡟⣥⠻⣷⣌⡀⠬⡘⢅⡟⡇⡮⣷⡾⡿⢋⣉⢣⢔⣎⠿⠊⠀⠀⡴⣛⠆⠌⠀⠀  ( . .) (. . )   |--
+  --|   ⠀⢀⣶⡟⣡⣿⣿⣟⢯⣟⢿⣷⣶⣯⣬⣵⣾⣷⣶⡾⠧⠞⠓⠉⠀⠀⠀⢀⠘⠈⠀⠠⢘⡤⠀  ( づ ˚♡︎˖  ⊂ )   |--
+  --|   ⠄⣾⠏⣐⣛⡻⢿⣿⣯⣿⣿⣿⣾⣽⣛⣍⢃⡂⢄⠀⡀⠀⡀⠄⢂⠄⠡⢈⠒⡈⢒⠘⠴⢀⠀                  |--
+  --|   ⢰⣿⠀⠈⠻⣜⣄⠈⢙⣾⢿⣿⣿⣿⡿⣜⢣⡜⢢⠁⠄⡐⢠⢉⠂⠌⠀⡀⠄⠐⡀⠄⠐⠀⢐      ___         |--
+  --|   ⠸⣟⠀⡐⡅⠈⠑⠀⠊⠝⠈⢖⡿⠿⣿⣾⡱⢊⠅⡌⡰⢌⢆⠣⠈⢀⠐⠀⠄⠂⠠⡈⠠⣈⡧     (...)        |--
+  --|   ⠀⢿⣆⠱⣘⣧⣤⣀⣀⡀⢒⡥⣑⢨⠒⡰⠯⠾⡼⠶⠙⢈⠀⣀⠂⡄⢂⣁⢢⣑⣶⡽⣳⠟⠁  _    \ _        |--
+  --|   ⠀⠀⠻⣧⡜⢹⣿⣿⣿⣿⣿⣿⣿⣿⣷⣦⣴⡀⡀⠀⠛⠺⢿⣶⣿⣾⣷⣿⣿⣿⢟⣵⠏⠀⠀ ('>    <')       |--
+  --|   ⠀⠀⠀⠈⠿⣶⣉⠻⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣧⣤⢀⠀⠀⠈⠉⠙⠻⣯⡷⠟⠁⠀⠀⠀ (v)    (v)       |--
+  --|   ⠀⠀⠀⠀⠀⠈⠙⠿⣶⣽⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣞⣤⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀\(__w  w__)/      |--
+  --|   ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠛⠛⠛⠿⠿⠿⠿⠿⠿⠛⠛⠛⠉⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                  |--
+  --|                                                        |--
+  --|                                                        |--
+
+# -- </logo> --
diff --git a/group_vars/bin.yml b/group_vars/bin.yml
index 8f0701e..3cf546a 100644
--- a/group_vars/bin.yml
+++ b/group_vars/bin.yml
@@ -1,4 +1,4 @@
 ---
 
-bin_domain: bin.liz.coffee
+bin_domain: bin.{{ domain }}
 bin_base: "{{ swarm_base }}/bin"
diff --git a/group_vars/coffee.yml b/group_vars/coffee.yml
new file mode 100644
index 0000000..90ddc7b
--- /dev/null
+++ b/group_vars/coffee.yml
@@ -0,0 +1,4 @@
+---
+
+coffee_domain: coffee.liz.coffee
+coffee_base: "{{ swarm_base }}/coffee"
diff --git a/group_vars/mon.yml b/group_vars/mon.yml
index 51566f2..1d0944e 100644
--- a/group_vars/mon.yml
+++ b/group_vars/mon.yml
@@ -2,3 +2,5 @@
 
 mon_domain: mon.liz.coffee
 mon_base: "{{ swarm_base }}/mon"
+
+prometheus_domain: prometheus.liz.coffee
diff --git a/group_vars/outbound.yml b/group_vars/outbound.yml
index e9d7e94..14a6b22 100644
--- a/group_vars/outbound.yml
+++ b/group_vars/outbound.yml
@@ -4,10 +4,15 @@ headscale_url: 'https://{{ headscale_host }}'
 headscale_base_domain: '{{ headscale_nodes_domain }}'
 headscale_base: '/etc/docker/compose/headscale'
 headscale_port: '8080'
+headscale_metrics_port: '5577'
 headscale_listen_addr: '0.0.0.0:{{ headscale_port }}'
+headscale_metrics_listen_addr: '0.0.0.0:{{ headscale_metrics_port }}'
 
-headscale_dns_for_connected_clients_1: '{{ loadbalancer_ip }}'
-headscale_dns_for_connected_clients_2: '1.0.0.1'
+headscale_dns_for_connected_clients:
+#  - '{{ mesh.lucina.gateway }}'
+  - '{{ mesh.liz.gateway }}'
+  - '1.0.0.1'
+  - '8.8.8.8'
 
 vpn_proxy_filter_container_name: 'headscale-proxy'
 proxy_base: '/etc/docker/compose/proxy'
diff --git a/group_vars/silverbullet.yml b/group_vars/silverbullet.yml
index d24cb47..4d4623e 100644
--- a/group_vars/silverbullet.yml
+++ b/group_vars/silverbullet.yml
@@ -2,3 +2,5 @@
 
 silverbullet_base: "{{ swarm_base }}/silverbullet"
 silverbullet_domain: "notes.{{ domain }}"
+
+notes_user_group: "notes_users@{{ idm_domain }}"
diff --git a/group_vars/traefik.yml b/group_vars/traefik.yml
index 35c1483..5e2a056 100644
--- a/group_vars/traefik.yml
+++ b/group_vars/traefik.yml
@@ -3,3 +3,7 @@
 certs_email: "{{ cloudflare_email }}"
 traefik_base: "{{ swarm_base }}/traefik"
 traefik_domain: "proxy.{{ domain }}"
+
+forwardauth_headers: "Set-Cookie,Authorization,X-Forwarded-User,X-Forwarded-Email,X-Forwarded-Preferred-Username,X-Forwarded-Groups,X-Forwarded-{{ oauth_proxy_super_secret_header }},X-Auth-Request-User,X-Auth-Request-Email,X-Auth-Request-Preferred-Username,X-Auth-Request-Groups,X-Auth-Request-{{ oauth_proxy_super_secret_header }}"
+
+oauth_proxy_group: "oauth_proxy_users@{{ idm_domain }}"
diff --git a/inventory b/inventory
index 5775ebe..2e636dd 100644
--- a/inventory
+++ b/inventory
@@ -74,3 +74,6 @@ swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connectio
 [mon]
 swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}'
 
+[coffee]
+swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}'
+
diff --git a/playbooks/coffee.yml b/playbooks/coffee.yml
new file mode 100644
index 0000000..3034273
--- /dev/null
+++ b/playbooks/coffee.yml
@@ -0,0 +1,7 @@
+---
+
+- name: coffee setup
+  hosts: coffee
+  become: true
+  roles:
+    - coffee
diff --git a/playbooks/roles/bin/templates/stacks/docker-compose.yml b/playbooks/roles/bin/templates/stacks/docker-compose.yml
index 5f99f8b..f1a86c4 100644
--- a/playbooks/roles/bin/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/bin/templates/stacks/docker-compose.yml
@@ -1,21 +1,21 @@
 services:
-  bin:
-    image: stonith404/pingvin-share
+  copyparty:
+    image: copyparty/ac:latest
+    user: "1000:1000"
     volumes:
-      - {{ bin_base }}/volumes/data:/data
+      - "{{ bin_base }}/volumes/share:/w:z"
+      - "{{ bin_base }}/volumes/conf:/cfg:z"
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
-      - TRUST_PROXY=true
-      - API_URL=https://{{ bin_domain }}
-      - DATA_DIRECTORY=/data
-      - DATABASE_URL=file:/data/pingvin-share.db?connection_limit=1
+      - LD_PRELOAD=/usr/lib/libmimalloc-secure.so.NOPE
+      - PYTHONUNBUFFERED=1
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:3000/api/configs"]
-      timeout: 3s
+      test: ["CMD-SHELL", "wget --spider -q 127.0.0.1:3923/?reset=/._"]
       interval: 1m
-      retries: 2
-      start_period: 10s
+      timeout: 2s
+      retries: 5
+      start_period: 15s
     networks:
       - proxy
     deploy:
@@ -34,7 +34,8 @@ services:
         - traefik.http.routers.bin.tls.certResolver=letsencrypt
         - traefik.http.routers.bin.rule=Host(`{{ bin_domain }}`)
         - traefik.http.routers.bin.entrypoints=websecure
-        - traefik.http.services.bin.loadbalancer.server.port=3000
+        - traefik.http.routers.bin.middlewares=oauth-verify
+        - traefik.http.services.bin.loadbalancer.server.port=3923
 
 networks:
   proxy:
diff --git a/playbooks/roles/bin/templates/volumes/conf/copyparty.conf b/playbooks/roles/bin/templates/volumes/conf/copyparty.conf
new file mode 100644
index 0000000..eaea0a6
--- /dev/null
+++ b/playbooks/roles/bin/templates/volumes/conf/copyparty.conf
@@ -0,0 +1,34 @@
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # enable multimedia indexing
+  ansi   # enable colors in log messages
+  #q      # disable logging for more performance
+
+  # if we are confident that we got the docker-network config correct
+  # (meaning copyparty is only accessible through traefik, and
+  #  traefik makes sure that all requests go through authelia),
+  # then accept X-Forwarded-For and IdP headers from any private IP:
+  xff-src: lan
+
+  idp-h-usr: x-auth-request-user
+  idp-h-grp: x-auth-request-groups
+  idp-h-key: x-auth-request-{{ oauth_proxy_super_secret_header }}
+
+[/]      # create a volume at "/" (the webroot), which will
+  /w
+  accs:
+    rw: *       # everyone gets read-access, but
+    rwmda: @{{ admins }}  # the group "su" gets read-write-move-delete-admin
+
+
+[/u/${u}]    # each user gets their own home-folder at /u/username
+  /w/u/${u}  # which will be "u/username" in the docker data volume
+  accs:
+    r: *              # read-access for anyone, and
+    rwmda: ${u}, @{{ admins }}  # read-write-move-delete-admin for that username + the "su" group
+
+
+[/u/${u}/priv]    # each user also gets a private area at /u/username/priv
+  /w/u/${u}/priv  # stored at DATAVOLUME/u/username/priv
+  accs:
+    rwmda: ${u}, @{{ admins }}  # read-write-move-delete-admin for that username + the "su" group
diff --git a/playbooks/roles/bin/templates/volumes/data/.gitkeep b/playbooks/roles/bin/templates/volumes/share/.gitkeep
index e69de29..e69de29 100644
--- a/playbooks/roles/bin/templates/volumes/data/.gitkeep
+++ b/playbooks/roles/bin/templates/volumes/share/.gitkeep
diff --git a/playbooks/roles/ci/templates/stacks/docker-compose.yml b/playbooks/roles/ci/templates/stacks/docker-compose.yml
index 1cc3a10..2a77205 100644
--- a/playbooks/roles/ci/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/ci/templates/stacks/docker-compose.yml
@@ -41,6 +41,7 @@ services:
         - traefik.http.routers.ci.tls=true
         - traefik.http.routers.ci.tls.certResolver=letsencrypt
         - traefik.http.routers.ci.rule=Host(`{{ ci_domain }}`)
+        - traefik.http.routers.ci.middlewares=oauth-verify
         - traefik.http.routers.ci.entrypoints=websecure
         - traefik.http.services.ci.loadbalancer.server.port=8080
 
diff --git a/playbooks/roles/coffee/tasks/main.yml b/playbooks/roles/coffee/tasks/main.yml
new file mode 100644
index 0000000..0e71be3
--- /dev/null
+++ b/playbooks/roles/coffee/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+
+- name: Deploy coffee
+  ansible.builtin.import_tasks: manage-docker-swarm-service.yml
+  vars:
+    service_name: coffee
+    template_render_dir: "../templates"
+    service_destination_dir: "{{ coffee_base }}"
diff --git a/playbooks/roles/coffee/templates/stacks/docker-compose.yml b/playbooks/roles/coffee/templates/stacks/docker-compose.yml
new file mode 100644
index 0000000..3bbf1d7
--- /dev/null
+++ b/playbooks/roles/coffee/templates/stacks/docker-compose.yml
@@ -0,0 +1,37 @@
+services:
+  coffee:
+    image: oci.liz.coffee/emprespresso/coffee:release
+    volumes:
+      - "{{ coffee_base }}/volumes/data:/data"
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+    networks:
+      - proxy
+    healthcheck:
+      test: ["CMD-SHELL", "curl", "--fail", "http://localhost:80"]
+      timeout: 15s
+      interval: 30s
+      retries: 3
+      start_period: 5s
+    deploy:
+      mode: replicated
+      update_config:
+        parallelism: 1
+        failure_action: rollback
+        order: start-first
+        delay: 5s
+        monitor: 30s
+      replicas: 1
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=proxy
+        - traefik.http.routers.coffee.tls=true
+        - traefik.http.routers.coffee.tls.certResolver=letsencrypt
+        - traefik.http.routers.coffee.rule=Host(`{{ coffee_domain }}`)
+        - traefik.http.routers.coffee.entrypoints=websecure
+        - traefik.http.services.coffee.loadbalancer.server.port=80
+
+networks:
+  proxy:
+    external: true
diff --git a/playbooks/roles/mon/templates/volumes/data/.gitkeep b/playbooks/roles/coffee/templates/volumes/data/.gitkeep
index e69de29..e69de29 100644
--- a/playbooks/roles/mon/templates/volumes/data/.gitkeep
+++ b/playbooks/roles/coffee/templates/volumes/data/.gitkeep
diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml
index 446db35..63eb60a 100644
--- a/playbooks/roles/common/tasks/main.yml
+++ b/playbooks/roles/common/tasks/main.yml
@@ -33,8 +33,8 @@
 ### SSH
 
 - name: Copy sshd_config
-  ansible.builtin.copy:
-    src: files/sshd_config
+  ansible.builtin.template:
+    src: templates/sshd_config
     dest: /etc/ssh/sshd_config
     owner: root
     group: root
@@ -43,8 +43,8 @@
     - Restart sshd
 
 - name: Copy authorized_keys
-  ansible.builtin.copy:
-    src: files/authorized_keys
+  ansible.builtin.template:
+    src: templates/authorized_keys
     dest: /home/{{ ansible_user }}/.ssh/authorized_keys
 
 ### UFW
diff --git a/playbooks/roles/common/files/authorized_keys b/playbooks/roles/common/templates/authorized_keys
index 82f2cbb..82f2cbb 100644
--- a/playbooks/roles/common/files/authorized_keys
+++ b/playbooks/roles/common/templates/authorized_keys
diff --git a/playbooks/roles/common/files/sshd_config b/playbooks/roles/common/templates/sshd_config
index 239a0c0..239a0c0 100644
--- a/playbooks/roles/common/files/sshd_config
+++ b/playbooks/roles/common/templates/sshd_config
diff --git a/playbooks/roles/kanidm/templates/volumes/data/server.toml b/playbooks/roles/kanidm/templates/volumes/data/server.toml
index afaf0f1..c4f320d 100644
--- a/playbooks/roles/kanidm/templates/volumes/data/server.toml
+++ b/playbooks/roles/kanidm/templates/volumes/data/server.toml
@@ -10,7 +10,5 @@ log_level = "info"
 domain = "{{ idm_domain }}"
 origin = "https://{{ idm_domain }}"
 
-# soon... once https://github.com/kanidm/kanidm/commit/b5cdf9dcf20114ed291700d99e8531226025f197 released >:D
-# x-forward-for = ["{{ swarm_network }}"]
 [http_client_address_info]
-x-forward-for-all-source-trusted = []
+x-forward-for = ["{{ swarm_network }}"]
diff --git a/playbooks/roles/keepalived/templates/keepalived.conf.j2 b/playbooks/roles/keepalived/templates/keepalived.conf.j2
index cb9c449..c02b54c 100644
--- a/playbooks/roles/keepalived/templates/keepalived.conf.j2
+++ b/playbooks/roles/keepalived/templates/keepalived.conf.j2
@@ -5,7 +5,7 @@ global_defs {
 
 vrrp_script chk_avail {
     script "{{ keepalived_healthcheck_script }}"
-    interval 1
+    interval 10
     weight 10
     rise 6
     fall 1
diff --git a/playbooks/roles/mail/templates/stacks/docker-compose.yml b/playbooks/roles/mail/templates/stacks/docker-compose.yml
index d7f8984..5e42461 100644
--- a/playbooks/roles/mail/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/mail/templates/stacks/docker-compose.yml
@@ -2,8 +2,8 @@ services:
   roundcube:
     image: roundcube/roundcubemail:latest
     volumes:
-      - {{ mail_base }}/volumes/data/roundcube/db:/var/roundcube/db
-      - {{ mail_base }}/volumes/data/roundcube/config:/var/roundcube/config/
+      - "{{ mail_base }}/volumes/data/roundcube/db:/var/roundcube/db"
+      - "{{ mail_base }}/volumes/data/roundcube/config:/var/roundcube/config/"
     environment:
       - DEPLOYMENT_TIME={{ deployment_time }}
       - ROUNDCUBEMAIL_DB_TYPE=sqlite
@@ -36,7 +36,7 @@ services:
 
   mailserver:
     image: ghcr.io/docker-mailserver/docker-mailserver:latest
-    hostname: {{ mail_domain }}
+    hostname: "{{ mail_domain }}"
 {% if homelab_build %}
     command:
       - /scripts/wait-for-cert.sh
diff --git a/playbooks/roles/mon/templates/stacks/docker-compose.yml b/playbooks/roles/mon/templates/stacks/docker-compose.yml
index ff7269f..98332cc 100644
--- a/playbooks/roles/mon/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/mon/templates/stacks/docker-compose.yml
@@ -2,12 +2,14 @@ services:
   mon:
     image: twinproduction/gatus:latest
     volumes:
-      - {{ mon_base }}/volumes/data:/data
+      - "{{ mon_base }}/volumes/gatus/data:/data"
+      - "{{ mon_base }}/volumes/gatus/config:/config"
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
     networks:
       - proxy
+      - metrics
     deploy:
       mode: replicated
       update_config:
@@ -20,12 +22,43 @@ services:
       labels:
         - traefik.enable=true
         - traefik.swarm.network=proxy
+        - traefik.http.routers.mon.middlewares=oauth-verify
         - traefik.http.routers.mon.tls=true
         - traefik.http.routers.mon.tls.certResolver=letsencrypt
         - traefik.http.routers.mon.rule=Host(`{{ mon_domain }}`)
         - traefik.http.routers.mon.entrypoints=websecure
         - traefik.http.services.mon.loadbalancer.server.port=8080
 
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - "{{ mon_base }}/volumes/prometheus/config.yml:/etc/prometheus/prometheus.yml"
+    networks:
+      - proxy
+      - metrics
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+    deploy:
+      mode: replicated
+      replicas: 1
+      update_config:
+        parallelism: 1
+        order: start-first
+        failure_action: rollback
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=proxy
+        - traefik.http.routers.prometheus.tls=true
+        - traefik.http.routers.prometheus.tls.certResolver=letsencrypt
+        - traefik.http.routers.prometheus.rule=Host(`{{ prometheus_domain }}`)
+        - traefik.http.routers.prometheus.entrypoints=websecure
+        - traefik.http.services.prometheus.loadbalancer.server.port=9090
+
 networks:
   proxy:
     external: true
+  metrics:
+    name: metrics
+    driver: overlay
+    attachable: true
diff --git a/playbooks/roles/mon/templates/volumes/gatus/config/config.yml b/playbooks/roles/mon/templates/volumes/gatus/config/config.yml
new file mode 100644
index 0000000..2d1c0ef
--- /dev/null
+++ b/playbooks/roles/mon/templates/volumes/gatus/config/config.yml
@@ -0,0 +1,82 @@
+metrics: true
+
+endpoints:
+  - name: "HealthCheck"
+    url: "{{ healthchecks_io_ping }}"
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY] == pat(*OK*)"
+
+  - name: "Expiration For {{ domain }}"
+    url: "https://{{ domain }}"
+    interval: 30m
+    conditions:
+      - "[DOMAIN_EXPIRATION] > 720h"
+      - "[CERTIFICATE_EXPIRATION] > 240h"
+
+  - name: "LDAPS"
+    url: "tls://{{ idm_domain }}:3636"
+    interval: 5m
+    client:
+      timeout: 5s
+    conditions:
+      - "[CONNECTED] == true"
+      - "[CERTIFICATE_EXPIRATION] > 48h"
+
+{% for port in [465,993] %}
+  - name: "mail on port {{ port }}"
+    group: "mail"
+    url: "tls://{{ mail_domain }}:{{ port }}"
+    interval: 5m
+    client:
+      timeout: 5s
+    conditions:
+      - "[CONNECTED] == true"
+      - "[CERTIFICATE_EXPIRATION] > 48h"
+{% endfor %}
+
+{% for user, m in mesh.items() %}
+{% for healthcheck in m.public_healthchecks %}
+  - name: "healthcheck {{ user }} pub {{ healthcheck }} 200"
+    group: "{{ user }}_pub"
+    url: "{{ healthcheck }}"
+    interval: 1m
+    conditions:
+      - "[STATUS] == 200"
+{% endfor %}
+{% for healthcheck in m.private_healthchecks %}
+  - name: "healthcheck {{ user }} priv {{ healthcheck }}"
+    url: "{{ healthcheck }}"
+    group: "{{ user }}_priv"
+    interval: 1m
+    conditions:
+      - "[STATUS] == 200"
+  - name: "healthcheck {{ user }} pub {{ healthcheck }} 403"
+    group: "{{ user }}_priv"
+    url: "{{ healthcheck }}"
+    client:
+      dns-resolver: "tcp://1.1.1.1:53"
+    interval: 1m
+    conditions:
+      - "[STATUS] == 403"
+{% endfor %}
+{% for record in m.private_records %}
+  - name: "DNS Check [{{ record.name }}_{{ record.type }}]"
+    group: "{{ user }}_dns_private"
+    url: "{{ m.gateway }}"
+    interval: 5m
+    dns:
+      query-name: "{{ record.name }}"
+      query-type: "{{ record.type }}"
+    conditions:
+      - "[BODY] == {{ record.ip }}"
+      - "[DNS_RCODE] == NOERROR"
+
+  - name: "PING {{ record.name }}"
+    group: "{{ user }}_dns_private"
+    url: "icmp://{{ record.name }}"
+    conditions:
+      - "[CONNECTED] == true"
+{% endfor %}
+{% endfor %}
diff --git a/playbooks/roles/mon/templates/volumes/gatus/data/.gitkeep b/playbooks/roles/mon/templates/volumes/gatus/data/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/playbooks/roles/mon/templates/volumes/gatus/data/.gitkeep
diff --git a/playbooks/roles/mon/templates/volumes/prometheus/config.yml b/playbooks/roles/mon/templates/volumes/prometheus/config.yml
new file mode 100644
index 0000000..be59f7f
--- /dev/null
+++ b/playbooks/roles/mon/templates/volumes/prometheus/config.yml
@@ -0,0 +1,39 @@
+global:
+  scrape_interval: 20s
+
+scrape_configs:
+ - job_name: prometheus
+   static_configs:
+    - targets:
+       - mon_prometheus:9090
+
+ - job_name: gatus
+   static_configs:
+    - targets:
+       - mon_mon:8080
+
+ - job_name: oauth-proxy
+   static_configs:
+    - targets:
+       - traefik_oauth2-proxy:5577
+
+ - job_name: traefik
+   static_configs:
+    - targets:
+       - traefik_traefik:5577
+
+ - job_name: headscale
+   static_configs:
+    - targets:
+       - "{{ headscale_host }}:443"
+   basic_auth:
+    username: '{{ metrics_htpasswd_user }}'
+    password: '{{ metrics_htpasswd_passwd }}'
+
+ - job_name: outbound
+   static_configs:
+    - targets:
+       - "{{ outbound_domain }}:443"
+   basic_auth:
+    username: '{{ metrics_htpasswd_user }}'
+    password: '{{ metrics_htpasswd_passwd }}'
diff --git a/playbooks/roles/nginx_proxy/templates/docker-compose.yml b/playbooks/roles/nginx_proxy/templates/docker-compose.yml
index 33b3243..ee44e45 100644
--- a/playbooks/roles/nginx_proxy/templates/docker-compose.yml
+++ b/playbooks/roles/nginx_proxy/templates/docker-compose.yml
@@ -19,9 +19,11 @@ services:
       # src
       - "2222:2222"
     volumes:
-      - /var/run/docker.sock:/tmp/docker.sock:ro
-      - {{ nginx_proxy_base }}/certs:/etc/nginx/certs
-      - {{ nginx_proxy_base }}/toplevel.conf.d:/etc/nginx/toplevel.conf.d
+      - "/var/run/docker.sock:/tmp/docker.sock:ro"
+      - "{{ nginx_proxy_base }}/certs:/etc/nginx/certs"
+      - "{{ nginx_proxy_base }}/toplevel.conf.d:/etc/nginx/toplevel.conf.d"
+      - "{{ nginx_proxy_base }}/stubs.conf:/etc/nginx/conf.d/stubs.conf"
+      - "{{ nginx_proxy_base }}/htpasswd:/etc/nginx/htpasswd"
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
@@ -33,6 +35,22 @@ services:
     labels:
       - com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy
 
+  nginx-prometheus-exporter:
+    image: nginx/nginx-prometheus-exporter
+    restart: always
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+      - VIRTUAL_HOST={{ outbound_domain }}
+      - VIRTUAL_PORT=9113
+      - LETSENCRYPT_HOST={{ outbound_domain }}
+    command:
+      - '--nginx.scrape-uri=http://nginx-proxy:81/nginx_status'
+    depends_on:
+      - nginx-proxy
+    networks:
+      - proxy
+
   nginx-acme-companion:
     image: nginxproxy/acme-companion
     depends_on:
@@ -40,7 +58,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - acme:/etc/acme.sh
-      - {{ nginx_proxy_base }}/certs:/etc/nginx/certs
+      - "{{ nginx_proxy_base }}/certs:/etc/nginx/certs"
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
diff --git a/playbooks/roles/nginx_proxy/templates/htpasswd/outbound.liz.coffee b/playbooks/roles/nginx_proxy/templates/htpasswd/outbound.liz.coffee
new file mode 100644
index 0000000..1bfc222
--- /dev/null
+++ b/playbooks/roles/nginx_proxy/templates/htpasswd/outbound.liz.coffee
@@ -0,0 +1 @@
+{{ metrics_htpasswd }}
diff --git a/playbooks/roles/nginx_proxy/templates/htpasswd/vpn.liz.coffee_7edfc244708a7b5c7d4b4385c178aa8e03afde7f b/playbooks/roles/nginx_proxy/templates/htpasswd/vpn.liz.coffee_7edfc244708a7b5c7d4b4385c178aa8e03afde7f
new file mode 100644
index 0000000..1bfc222
--- /dev/null
+++ b/playbooks/roles/nginx_proxy/templates/htpasswd/vpn.liz.coffee_7edfc244708a7b5c7d4b4385c178aa8e03afde7f
@@ -0,0 +1 @@
+{{ metrics_htpasswd }}
diff --git a/playbooks/roles/nginx_proxy/templates/stubs.conf b/playbooks/roles/nginx_proxy/templates/stubs.conf
new file mode 100644
index 0000000..57765b7
--- /dev/null
+++ b/playbooks/roles/nginx_proxy/templates/stubs.conf
@@ -0,0 +1,9 @@
+server {
+    listen 81;
+    location /nginx_status {
+        stub_status;
+
+        access_log off;
+        allow all;
+    }
+}
diff --git a/playbooks/roles/outbound/templates/headscale/config/config.yaml b/playbooks/roles/outbound/templates/headscale/config/config.yaml
index 54657b2..078058e 100644
--- a/playbooks/roles/outbound/templates/headscale/config/config.yaml
+++ b/playbooks/roles/outbound/templates/headscale/config/config.yaml
@@ -7,7 +7,7 @@ listen_addr: '{{ headscale_listen_addr }}'
 # to keep this endpoint private to your internal
 # network
 #
-metrics_listen_addr: 127.0.0.1:9090
+metrics_listen_addr: '{{ headscale_metrics_listen_addr }}'
 
 # The Noise section includes specific configuration for the
 # TS2021 Noise protocol
@@ -122,9 +122,7 @@ dns:
   base_domain: "{{ headscale_base_domain }}"
   search_domains: []
   nameservers:
-    global:
-      - {{ headscale_dns_for_connected_clients_1 }}
-      - {{ headscale_dns_for_connected_clients_2 }}
+    global: {{ headscale_dns_for_connected_clients | tojson }}
     split:
 {% for user, m in mesh.items() %}
 {% if "split_vpn_dns_to" in m %}
diff --git a/playbooks/roles/outbound/templates/headscale/docker-compose.yml b/playbooks/roles/outbound/templates/headscale/docker-compose.yml
index 515630c..463db70 100644
--- a/playbooks/roles/outbound/templates/headscale/docker-compose.yml
+++ b/playbooks/roles/outbound/templates/headscale/docker-compose.yml
@@ -12,16 +12,19 @@ services:
     networks:
       - proxy
     environment:
-      - DEPLOYMENT_TIME={{ deployment_time }}
-      - VIRTUAL_HOST={{ headscale_host }}
-      - VIRTUAL_PORT={{ headscale_port }}
-      - LETSENCRYPT_HOST={{ headscale_host }}
+      DEPLOYMENT_TIME: "{{ deployment_time }}"
+      VIRTUAL_HOST_MULTIPORTS: |-
+        {{ headscale_host }}:
+          "/":
+            port: {{ headscale_port }}
+          "/metrics":
+            port: {{ headscale_metrics_port }}
 {% if homelab_build %}
     healthcheck:
       disable: true
 {% else %}
     healthcheck:
-      test: ["CMD", "wget", "-qO", "-", "http://localhost:{{ headscale_port }}/health"] 
+      test: ["CMD", "wget", "-qO", "-", "http://localhost:{{ headscale_port }}/health"]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -34,11 +37,12 @@ services:
     networks:
       - proxy
     environment:
+      - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
       - VIRTUAL_HOST={{ headscale_host }}
       - VIRTUAL_PORT={{ headscale_port }}
       - LETSENCRYPT_HOST={{ headscale_host }}
-      - VIRTUAL_PATH=/web/ 
+      - VIRTUAL_PATH=/web/
       - VIRTUAL_DEST=/web/
 
 networks:
diff --git a/playbooks/roles/outbound/templates/proxy/docker-compose.yml b/playbooks/roles/outbound/templates/proxy/docker-compose.yml
index c754cdc..654c5da 100644
--- a/playbooks/roles/outbound/templates/proxy/docker-compose.yml
+++ b/playbooks/roles/outbound/templates/proxy/docker-compose.yml
@@ -66,4 +66,3 @@ networks:
     driver: bridge
   proxy:
     external: true
-
diff --git a/playbooks/roles/outbound/templates/proxy/nginx/conf.d/coffee.conf b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/coffee.conf
new file mode 100644
index 0000000..5fa47be
--- /dev/null
+++ b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/coffee.conf
@@ -0,0 +1,19 @@
+server {
+    listen 80;
+    server_name liz.coffee;
+
+    real_ip_header X-Forwarded-For;
+    real_ip_recursive on;
+    set_real_ip_from {{ docker_network }};
+
+    location / {
+        proxy_pass https://{{ loadbalancer_ip }};
+        proxy_ssl_verify off;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
diff --git a/playbooks/roles/outbound/templates/proxy/nginx/conf.d/fwdauth.conf b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/fwdauth.conf
new file mode 100644
index 0000000..a2696bf
--- /dev/null
+++ b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/fwdauth.conf
@@ -0,0 +1,19 @@
+server {
+    listen 80;
+    server_name fwdauth.liz.coffee;
+
+    real_ip_header X-Forwarded-For;
+    real_ip_recursive on;
+    set_real_ip_from {{ docker_network }};
+
+    location / {
+        proxy_pass https://{{ loadbalancer_ip }};
+        proxy_ssl_verify off;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
diff --git a/playbooks/roles/silverbullet/templates/stacks/docker-compose.yml b/playbooks/roles/silverbullet/templates/stacks/docker-compose.yml
index 4175c4e..6593398 100644
--- a/playbooks/roles/silverbullet/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/silverbullet/templates/stacks/docker-compose.yml
@@ -5,9 +5,8 @@ services:
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
-      - SB_USER={{ silverbullet_password }}
     volumes:
-      - {{ silverbullet_base }}/volumes/data:/space
+      - "{{ silverbullet_base }}/volumes/data:/space"
     networks:
       - proxy
     deploy:
@@ -23,6 +22,8 @@ services:
         - traefik.http.routers.silverbullet.tls=true
         - traefik.http.routers.silverbullet.tls.certResolver=letsencrypt
         - traefik.http.routers.silverbullet.rule=Host(`{{ silverbullet_domain }}`)
+        - traefik.http.routers.silverbullet.middlewares=oauth-verify,oauth-notes-users
+        - traefik.http.middlewares.oauth-notes-users.forwardAuth.address=http://oauth2-proxy:4180/oauth2/auth?allowed_groups={{ notes_user_group }}
         - traefik.http.routers.silverbullet.entrypoints=websecure
         - traefik.http.services.silverbullet.loadbalancer.server.port=3000
 
diff --git a/playbooks/roles/test/templates/stacks/docker-compose.yml b/playbooks/roles/test/templates/stacks/docker-compose.yml
index 52f220f..ef0372e 100644
--- a/playbooks/roles/test/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/test/templates/stacks/docker-compose.yml
@@ -2,7 +2,7 @@ services:
   test:
     image: traefik/whoami:latest
     volumes:
-      - {{ test_base }}/volumes/data:/data
+      - "{{ test_base }}/volumes/data:/data"
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
diff --git a/playbooks/roles/traefik/templates/stacks/docker-compose.yml b/playbooks/roles/traefik/templates/stacks/docker-compose.yml
index 6362b31..46f5503 100644
--- a/playbooks/roles/traefik/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/traefik/templates/stacks/docker-compose.yml
@@ -15,7 +15,7 @@ services:
       timeout: 5s
       retries: 10
     volumes:
-      - {{ traefik_base }}/volumes/headscale:/var/lib/tailscale
+      - "{{ traefik_base }}/volumes/headscale:/var/lib/tailscale"
       - /dev/net/tun:/dev/net/tun
     cap_add:
       - NET_ADMIN
@@ -53,10 +53,12 @@ services:
       - CF_DNS_API_TOKEN={{ cloudflare_dns_api_token }}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
-      - {{ traefik_base }}/stacks/traefik.yml:/traefik.yml
-      - {{ traefik_base }}/volumes/certs:/certs
+      - "{{ traefik_base }}/stacks/traefik.yml:/traefik.yml"
+      - "{{ traefik_base }}/volumes/certs:/certs"
     networks:
+      - metrics
       - proxy
+      - oauth_proxy
       - headnet
     deploy:
       mode: global
@@ -66,21 +68,91 @@ services:
         failure_action: rollback
         monitor: 2s # go go go.
       labels:
-        - traefik.enable=true
-        - traefik.http.routers.dashboard.rule=Host(`{{ traefik_domain }}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))
-        - traefik.http.routers.dashboard.service=api@internal
-        - traefik.http.routers.dashboard.tls=true
-        - traefik.http.routers.dashboard.tls.certresolver=letsencrypt
-        - traefik.http.routers.ping.rule=Host(`{{ traefik_domain }}`) && PathPrefix(`/ping`)
-        - traefik.http.routers.ping.service=ping@internal
-        - traefik.http.routers.ping.tls=true
-        - traefik.http.routers.ping.tls.certresolver=letsencrypt
-        - traefik.http.services.dashboard.loadbalancer.server.port=8080
-        - traefik.http.services.ping.loadbalancer.server.port=8080
+        - "traefik.enable=true"
+        - "traefik.http.routers.dashboard.rule=Host(`{{ traefik_domain }}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
+        - "traefik.http.routers.dashboard.service=api@internal"
+        - "traefik.http.routers.dashboard.tls=true"
+        - "traefik.http.routers.dashboard.tls.certresolver=letsencrypt"
+        - "traefik.http.services.dashboard.loadbalancer.server.port=8080"
+        - "traefik.http.routers.ping.rule=Host(`{{ traefik_domain }}`) && PathPrefix(`/ping`)"
+        - "traefik.http.routers.ping.service=ping@internal"
+        - "traefik.http.routers.ping.tls=true"
+        - "traefik.http.routers.ping.tls.certresolver=letsencrypt"
+        - "traefik.http.services.ping.loadbalancer.server.port=8080"
+
+        - "traefik.http.middlewares.auth-headers.headers.stsSeconds=315360000"
+        - "traefik.http.middlewares.auth-headers.headers.browserXssFilter=true"
+        - "traefik.http.middlewares.auth-headers.headers.contentTypeNosniff=true"
+        - "traefik.http.middlewares.auth-headers.headers.forceSTSHeader=true"
+        - "traefik.http.middlewares.auth-headers.headers.stsIncludeSubdomains=true"
+        - "traefik.http.middlewares.auth-headers.headers.stsPreload=true"
+        - "traefik.http.middlewares.auth-headers.headers.frameDeny=true"
+
+        - "traefik.http.middlewares.oauth-verify.forwardAuth.address=http://oauth2-proxy:4180"
+        - "traefik.http.middlewares.oauth-verify.forwardAuth.trustForwardHeader=true"
+        - "traefik.http.middlewares.oauth-verify.forwardAuth.authResponseHeaders={{ forwardauth_headers }}"
+
+        - "traefik.http.middlewares.oauth-verify-noredirect.forwardAuth.address=http://oauth2-proxy:4180/oauth2/auth"
+        - "traefik.http.middlewares.oauth-verify-noredirect.forwardAuth.trustForwardHeader=true"
+        - "traefik.http.middlewares.oauth-verify-noredirect.forwardAuth.authResponseHeaders={{ forwardauth_headers }}"
+
+  oauth2-proxy:
+    image: quay.io/oauth2-proxy/oauth2-proxy:latest
+    command: --alpha-config /conf/oauth_proxy_alpha.yml --config /conf/oauth_proxy.cfg
+    volumes:
+      - "{{ traefik_base }}/volumes/oauth2proxy:/conf"
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+    networks:
+      - oauth_cache
+      - proxy
+      - oauth_proxy
+      - metrics
+    deploy:
+      mode: replicated
+      update_config:
+        parallelism: 1
+        failure_action: rollback
+        order: start-first
+        delay: 5s
+      replicas: 1
+      labels:
+        "traefik.enable": "true"
+        "traefik.swarm.network": "proxy"
+        "traefik.http.routers.fwdauth.tls": "true"
+        "traefik.http.routers.fwdauth.tls.certResolver": "letsencrypt"
+        "traefik.http.routers.fwdauth.rule": "!Host(`{{ idm_domain }}`) && (PathPrefix(`/oauth2`) || Host(`{{ oauth_proxy_domain }}`))"
+        "traefik.http.routers.fwdauth.entrypoints": "websecure"
+        "traefik.http.routers.fwdauth.middlewares": "auth-headers"
+        "traefik.http.services.fwdauth.loadbalancer.server.port": "4180"
+  oauth2-cache:
+    image: redis:8-alpine
+    restart: always
+    command: redis-server --save 20 1 --loglevel warning
+    volumes:
+      - "{{ traefik_base }}/volumes/redis:/data"
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+    networks:
+      - oauth_cache
+    deploy:
+      mode: replicated
+      update_config:
+        parallelism: 1
+        failure_action: rollback
+        order: start-first
+        delay: 5s
+      replicas: 1
 
 networks:
+  metrics:
+    external: true
   proxy:
     name: proxy
     driver: overlay
     attachable: true
+  oauth_proxy:
+  oauth_cache:
   headnet:
diff --git a/playbooks/roles/traefik/templates/stacks/traefik.yml b/playbooks/roles/traefik/templates/stacks/traefik.yml
index 09588b3..68235e4 100644
--- a/playbooks/roles/traefik/templates/stacks/traefik.yml
+++ b/playbooks/roles/traefik/templates/stacks/traefik.yml
@@ -1,12 +1,18 @@
 ping: {}
-accessLog: {}
+accessLog:
+  format: "json"
 log:
-  level: INFO
+  level: info
+metrics:
+  prometheus:
+    entryPoint: metrics
 api:
   dashboard: true
-  insecure: true
-  debug: true
+  insecure: false
+  debug: false
 entryPoints:
+  metrics:
+    address: ":5577"
   web:
     address: ":80"
     forwardedHeaders:
@@ -41,4 +47,5 @@ certificatesResolvers:
       # caServer: https://acme-staging-v02.api.letsencrypt.org/directory # staging
       dnsChallenge:
         provider: cloudflare
-        delayBeforeCheck: 10
+        propagation:
+          delayBeforeChecks: 12s
diff --git a/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy.cfg b/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy.cfg
new file mode 100644
index 0000000..3c412de
--- /dev/null
+++ b/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy.cfg
@@ -0,0 +1,26 @@
+## OAuth2 Proxy Config File
+
+request_logging = true
+email_domains = "*"
+reverse_proxy = true
+redirect_url = "https://{{ oauth_proxy_domain }}/oauth2/callback"
+real_client_ip_header = "X-Forwarded-For"
+trusted_ips = "{{ homelab_network }}"
+
+## Cookie Settings
+cookie_name = "_oauth2_proxy"
+cookie_secret = "{{ oauth_proxy_cookie_secret }}"
+cookie_domains = [".{{ domain }}", "{{ domain }}"]
+whitelist_domains = [".{{ domain }}", "{{ domain }}"]
+cookie_expire = "24h"
+cookie_refresh = "1h"
+cookie_secure = true
+session_store_type = "redis"
+redis_connection_url = "redis://oauth2-cache"
+
+## Templating
+
+banner = "-"
+footer = "-"
+custom_sign_in_logo="-"
+custom_templates_dir="/conf/templates"
diff --git a/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy_alpha.yml b/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy_alpha.yml
new file mode 100644
index 0000000..0f1b1ab
--- /dev/null
+++ b/playbooks/roles/traefik/templates/volumes/oauth2proxy/oauth_proxy_alpha.yml
@@ -0,0 +1,75 @@
+injectRequestHeaders:
+- name: X-Forwarded-User
+  values:
+  - claim: user
+- name: X-Forwarded-Email
+  values:
+  - claim: email
+- name: X-Forwarded-Preferred-Username
+  values:
+  - claim: preferred_username
+- name: X-Forwarded-Groups
+  values:
+  - claim: groups
+- name: Authorization
+  values:
+  - claim: id_token
+    prefix: 'Bearer '
+- name: "X-Forwarded-{{ oauth_proxy_super_secret_header }}"
+  values:
+  - value: "{{ oauth_proxy_super_secret_header | b64encode }}"
+injectResponseHeaders:
+- name: X-Auth-Request-User
+  values:
+  - claim: user
+- name: X-Auth-Request-Email
+  values:
+  - claim: email
+- name: X-Auth-Request-Preferred-Username
+  values:
+  - claim: preferred_username
+- name: X-Auth-Request-Groups
+  values:
+  - claim: groups
+- name: "X-Auth-Request-{{ oauth_proxy_super_secret_header }}"
+  values:
+  - value: "{{ oauth_proxy_super_secret_header | b64encode }}"
+- name: Authorization
+  values:
+  - claim: id_token
+    prefix: 'Bearer '
+metricsServer:
+  BindAddress: 0.0.0.0:5577
+  SecureBindAddress: ""
+  TLS: null
+providers:
+- id: kanidm
+  name: "{{ domain }} <3"
+  provider: oidc
+  clientID: "{{ oauth_proxy_client_id }}"
+  clientSecret: "{{ oauth_proxy_client_secret }}"
+  allowedGroups:
+  - "{{ oauth_proxy_group }}"
+  code_challenge_method: "S256"
+  scope: "openid profile groups email"
+  oidcConfig:
+    issuerURL: "https://{{ idm_domain }}/oauth2/openid/{{ oauth_proxy_client_id }}"
+    insecureSkipNonce: false
+    insecureAllowUnverifiedEmail: false
+    extraAudiences:
+      - "{{ oauth_proxy_client_id }}"
+    audienceClaims:
+      - aud
+    userIDClaim: sub
+    emailClaim: email
+    groupsClaim: groups
+server:
+  BindAddress: 0.0.0.0:4180
+  SecureBindAddress: ""
+  TLS: null
+upstreamConfig:
+  upstreams:
+  - id: "traefik"
+    static: true
+    path: "/"
+    staticCode: 202
diff --git a/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/error.html b/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/error.html
new file mode 100644
index 0000000..d202d83
--- /dev/null
+++ b/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/error.html
@@ -0,0 +1,96 @@
+{{ '{{' }}define "error.html"{{ '}}' }}
+<!DOCTYPE html>
+<html lang="en" charset="utf-8">
+<head>
+  <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'><text x='0' y='14' font-size='16'>☕</text></svg>">
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+  <title>{{ '{{' }} .StatusCode {{ '}}' }} {{ '{{' }} .Title {{ '}}' }}</title>
+  <style>
+    :root {
+      --bg: #282828;
+      --bg-alt: #1d2021;
+      --fg: #ebdbb2;
+      --red: #fb4934;
+      --blue: #83a598;
+    }
+    html, body {
+      margin: 0; padding: 0;
+      height: 100%;
+      background-color: var(--bg);
+      color: var(--fg);
+      font-family: monospace;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
+    .container {
+      background-color: var(--bg-alt);
+      border: 2px solid var(--red);
+      padding: 3.5rem;
+      border-radius: 6px;
+      max-width: 1000px;
+      width: 90%;
+      box-shadow: 0 0 8px rgba(0,0,0,0.5);
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+    }
+    .status {
+      font-size: 6rem;
+      margin: 0;
+      color: var(--red);
+      line-height: 1;
+    }
+    .title {
+      font-size: 1.5rem;
+      margin: 0.5rem 0 1rem;
+    }
+    .message {
+      background-color: var(--bg);
+      border: 1px solid var(--blue);
+      padding: 1rem;
+      width: 100%;
+      white-space: pre-wrap;
+      margin-bottom: 1rem;
+    }
+    .button {
+      width: 100%;
+      padding: 0.75rem;
+      background-color: var(--red);
+      color: var(--bg);
+      border: none;
+      text-transform: uppercase;
+      font-weight: bold;
+      cursor: pointer;
+      transition: background 0.2s;
+      margin-top: 0.5rem;
+    }
+    .button:hover {
+      background-color: #cc241d;
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <div class="status">{{ '{{' }} .StatusCode {{ '}}' }}</div>
+    <div class="title">{{ '{{' }} .Title {{ '}}' }}</div>
+    {{ '{{' }} if or .Message .RequestID {{ '}}' }}
+    <div class="message">
+      {{ '{{' }} if .Message {{ '}}' }}
+      {{ '{{' }} .Message {{ '}}' }}
+      {{ '{{' }} end {{ '}}' }}
+      {{ '{{' }} if .RequestID {{ '}}' }}
+      Request ID: {{ '{{' }} .RequestID {{ '}}' }}
+      {{ '{{' }} end {{ '}}' }}
+    </div>
+    {{ '{{' }} end {{ '}}' }}
+    {{ '{{' }} if .Redirect {{ '}}' }}
+    <form method="GET" action="{{ '{{' }} .Redirect {{ '}}' }}" style="width:100%;">
+      <button type="submit" class="button">Go Back</button>
+    </form>
+    {{ '{{' }} end {{ '}}' }}
+  </div>
+</body>
+</html>
+{{ '{{' }}end{{ '}}' }}
diff --git a/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/sign_in.html b/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/sign_in.html
new file mode 100644
index 0000000..17d3718
--- /dev/null
+++ b/playbooks/roles/traefik/templates/volumes/oauth2proxy/templates/sign_in.html
@@ -0,0 +1,72 @@
+{{ '{{' }}define "sign_in.html"{{ '}}' }}
+<!DOCTYPE html>
+<html lang="en" charset="utf-8">
+<head>
+  <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'><text x='0' y='14' font-size='16'>☕</text></svg>">
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+  <title>Sign In</title>
+  <style>
+    * {
+      font-family: 'monospace';
+    }
+    :root {
+      --bg: #282828;
+      --bg-alt: #1d2021;
+      --fg: #ebdbb2;
+      --green: #b8bb26;
+      --yellow: #fabd2f;
+    }
+    html, body {
+      margin: 0; padding: 0;
+      height: 100%;
+      background-color: var(--bg);
+      color: var(--fg);
+      font-family: monospace;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
+    .container {
+      background-color: var(--bg-alt);
+      border: 2px solid var(--green);
+      padding: 3.5rem;
+      border-radius: 6px;
+      max-width: 1000px;
+      width: 90%;
+      box-shadow: 0 0 8px rgba(0,0,0,0.5);
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+    }
+    .button {
+      width: 100%;
+      padding: 0.75rem;
+      background-color: var(--green);
+      color: var(--bg);
+      border: none;
+      text-transform: uppercase;
+      font-weight: bold;
+      cursor: pointer;
+      transition: background 0.2s;
+      margin-top: 1rem;
+    }
+    .button:hover {
+      background-color: var(--yellow);
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <pre class="logo">{{ logo }}</pre>
+    <form method="GET" action="{{ '{{' }} .ProxyPrefix {{ '}}' }}/start" style="width: 100%; display: flex; flex-direction: column;">
+      <input type="hidden" name="rd" value="{{ '{{' }} .Redirect {{ '}}' }}">
+      {{ '{{' }} if .SignInMessage {{ '}}' }}
+      <p>{{ '{{' }} .SignInMessage {{ '}}' }}</p>
+      {{ '{{' }} end {{ '}}' }}
+      <button type="submit" class="button">Sign in with {{ '{{' }} .ProviderName {{ '}}' }}</button>
+    </form>
+  </div>
+</body>
+</html>
+{{ '{{' }}end{{ '}}' }}
diff --git a/playbooks/roles/traefik/templates/volumes/redis/.gitkeep b/playbooks/roles/traefik/templates/volumes/redis/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/playbooks/roles/traefik/templates/volumes/redis/.gitkeep
diff --git a/secrets.txt b/secrets.txt
index 26af908..89b7245 100644
--- a/secrets.txt
+++ b/secrets.txt
@@ -5,11 +5,9 @@ cloudflare_dns_api_token
 cloudflare_email
 cloudflare_zone_id
 ceph_secret
-pihole_webpwd
 headscale_oidc_secret
 headscale_user_auth_key
 kanboard_ldap_password
-silverbullet_password
 ses_smtp_user_name
 ses_smtp_password
 email_ldap_api_token
@@ -21,3 +19,10 @@ ci_user_registry_password_argon_encoded
 passwd_client_id
 passwd_client_secret
 passwd_master_password
+oauth_proxy_client_id
+oauth_proxy_client_secret
+oauth_proxy_cookie_secret
+oauth_proxy_super_secret_header
+metrics_htpasswd_user
+metrics_htpasswd_passwd
+metrics_htpasswd