CI. Fuck.

25 files changed, 195 insertions, 34 deletions

diff --git a/create.py b/create.py
index c3fc5dd..e99a342 100755
--- a/create.py
+++ b/create.py
@@ -97,7 +97,7 @@ class HomelabDns:
             mesh = yaml.safe_load("\n".join(lines[mesh_start:mesh_end]))
             mesh['liz']['private_records'] += [{"type": "A", "name": dns_prefix + ".{{ domain }}", "ip": "{{ loadbalancer_ip }}"}]
             new_mesh = yaml.dump(mesh)
-	    new_lines = lines[0:mesh_start] + [self.start_delimiter, yaml.dump(mesh)] + [mesh_end:]
+            new_lines = lines[0:mesh_start] + [self.start_delimiter, yaml.dump(mesh)] + lines[mesh_end:]
 
         if not new_lines:
             return False
@@ -293,7 +293,7 @@ def main():
     if args.internal:
         logger.info("Configuring internal DNS via LabDNS...")
         dns = HomelabDns(
-            group_vars=Config.MESH_GROUP_VARS
+            group_vars=Config.MESH_GROUP_VARS,
             start_delimiter="# -- <mesh> --\n",
             end_delimiter="# -- </mesh> --\n"
         )
diff --git a/deploy.yml b/deploy.yml
index dc91530..94e0bbc 100644
--- a/deploy.yml
+++ b/deploy.yml
@@ -62,3 +62,6 @@
 
 - name: oci
   ansible.builtin.import_playbook: playbooks/oci.yml
+
+- name: mon
+  ansible.builtin.import_playbook: playbooks/mon.yml
diff --git a/group_vars/all.yml b/group_vars/all.yml
index 3648712..a285422 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -25,11 +25,16 @@ rfc1918_cgnat_networks:
 domain: "liz.coffee"
 idm_domain: "idm.{{ domain }}"
 headscale_host: "vpn.{{ domain }}"
+headscale_nodes_domain: "in.{{ domain }}"
 mail_domain: "mail.{{ domain }}"
 oci_domain: "oci.{{ domain }}"
 passwd_domain: "passwd.{{ domain }}"
 # -- </shared_services> --
 
+# -- <docker> --
+docker_gid: 995
+# -- </docker> --
+
 # -- <notifcation_email> --
 info_mail_user: "info"
 info_mail: "{{ info_mail_user }}@{{ domain }}"
@@ -61,13 +66,25 @@ mesh:
     gateway: "10.128.0.44"
     domain: "lucina.cloud"
     forward_dns: true
+    split_vpn_dns_to: "10.128.0.44"
     private_records: []
   liz:
     gateway: "{{ loadbalancer_ip }}"
     domain: "{{ domain }}"
     forward_dns: false
+    split_vpn_dns_to: "{{ loadbalancer_ip }}"
     private_records:
       - type: "A"
+        name: "piplup.{{ domain }}"
+        ip: "10.128.0.101"
+      - type: "A"
+        name: "togepi.{{ domain }}"
+        ip: "10.128.0.102"
+      - type: "A"
+        name: "roton.{{ domain }}"
+        ip: "10.128.0.103"
+
+      - type: "A"
         name: "oci.{{ domain }}"
         ip: "{{ loadbalancer_ip }}"
       - type: "A"
@@ -122,3 +139,4 @@ mesh:
         name: "roton.pocket.{{ domain }}"
         ip: "10.128.0.103"
 # -- </mesh> --
+
diff --git a/group_vars/mon.yml b/group_vars/mon.yml
new file mode 100644
index 0000000..51566f2
--- /dev/null
+++ b/group_vars/mon.yml
@@ -0,0 +1,4 @@
+---
+
+mon_domain: mon.liz.coffee
+mon_base: "{{ swarm_base }}/mon"
diff --git a/group_vars/nginx_proxy.yml b/group_vars/nginx_proxy.yml
index bd5a27a..5fb2a17 100644
--- a/group_vars/nginx_proxy.yml
+++ b/group_vars/nginx_proxy.yml
@@ -1,4 +1,4 @@
 ---
 
-certs_email: elizabeth@simponic.xyz
+certs_email: "me@{{ domain }}"
 nginx_proxy_base: "/etc/docker/compose/nginx-proxy"
diff --git a/group_vars/outbound.yml b/group_vars/outbound.yml
index 759872f..e9d7e94 100644
--- a/group_vars/outbound.yml
+++ b/group_vars/outbound.yml
@@ -1,7 +1,7 @@
 ---
 
 headscale_url: 'https://{{ headscale_host }}'
-headscale_base_domain: '{{ headscale_host }}'
+headscale_base_domain: '{{ headscale_nodes_domain }}'
 headscale_base: '/etc/docker/compose/headscale'
 headscale_port: '8080'
 headscale_listen_addr: '0.0.0.0:{{ headscale_port }}'
@@ -18,7 +18,7 @@ generate_auth_key: '{{ homelab_build }}'
 auth_key_expiration: '2y'
 auth_key_user: 'pocketmonsters'
 
-oauth_user_suffix: '@idm.{{ domain }}'
+oauth_user_suffix: '{{ idm_domain }}'
 
 # being in this list just means you'll have access to your own devices.
 # the vpn_users oauth claim decides whether or not you're authorized to actually
diff --git a/inventory b/inventory
index a15b7ba..5775ebe 100644
--- a/inventory
+++ b/inventory
@@ -71,3 +71,6 @@ swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connectio
 [oci]
 swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}'
 
+[mon]
+swarm-one ansible_host=10.128.0.201 ansible_user=serve         ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}'
+
diff --git a/playbooks/mon.yml b/playbooks/mon.yml
new file mode 100644
index 0000000..dc05b10
--- /dev/null
+++ b/playbooks/mon.yml
@@ -0,0 +1,7 @@
+---
+
+- name: mon setup
+  hosts: mon
+  become: true
+  roles:
+    - mon
diff --git a/playbooks/roles/ci/tasks/main.yml b/playbooks/roles/ci/tasks/main.yml
index cd0c220..fb3d3f7 100644
--- a/playbooks/roles/ci/tasks/main.yml
+++ b/playbooks/roles/ci/tasks/main.yml
@@ -3,6 +3,8 @@
 - name: Deploy ci
   ansible.builtin.import_tasks: manage-docker-swarm-service.yml
   vars:
+    service_owner: "1000"
+    file_mode: "755"
     service_name: ci
     template_render_dir: "../templates"
     service_destination_dir: "{{ ci_base }}"
diff --git a/playbooks/roles/ci/templates/stacks/docker-compose.yml b/playbooks/roles/ci/templates/stacks/docker-compose.yml
index c62fdd5..1cc3a10 100644
--- a/playbooks/roles/ci/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/ci/templates/stacks/docker-compose.yml
@@ -2,11 +2,17 @@
 
 services:
   worker:
-    image: oci.liz.coffee/emprespresso/ci-worker:release
+    image: oci.liz.coffee/emprespresso/ci_worker:release
     volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/run/docker.sock:/var/run/docker.sock:ro
       - {{ ci_base }}/volumes/laminar:/var/lib/laminar/
       - /var/lib/laminar/cfg # don't overwrite cfg jobs & scripts
+    healthcheck:
+      test: ["CMD-SHELL", "/usr/bin/laminarc show-jobs"]
+      timeout: 15s
+      interval: 10s
+      retries: 3
+      start_period: 5s
     environment:
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
@@ -15,6 +21,8 @@ services:
       - BW_CLIENTSECRET={{ passwd_client_secret }}
       - BW_PASSWORD={{ passwd_master_password }}
       - LAMINAR_BIND_RPC=*:9997
+      - LAMINAR_ARCHIVE_URL=https://{{ ci_domain }}
+      - LAMINAR_KEEP_RUNDIRS=5
     networks:
       - ci
       - proxy
@@ -36,13 +44,19 @@ services:
         - traefik.http.routers.ci.entrypoints=websecure
         - traefik.http.services.ci.loadbalancer.server.port=8080
 
-  cihooks:
-    image: oci.liz.coffee/emprespresso/ci-hooks:release
+  server:
+    image: oci.liz.coffee/emprespresso/ci_server:release
     environment:
       - LAMINAR_HOST=worker:9997
-      - LAMINAR_URL=worker:9997
+      - LAMINAR_URL=https://{{ ci_domain }}
       - TZ={{ timezone }}
       - DEPLOYMENT_TIME={{ deployment_time }}
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://localhost:9000/health"]
+      timeout: 15s
+      interval: 10s
+      retries: 3
+      start_period: 5s
     networks:
       - ci
     deploy:
diff --git a/playbooks/roles/ci/templates/volumes/laminar/archive/.gitkeep b/playbooks/roles/ci/templates/volumes/laminar/archive/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/playbooks/roles/ci/templates/volumes/laminar/archive/.gitkeep
diff --git a/playbooks/roles/ci/templates/volumes/laminar/run/.gitkeep b/playbooks/roles/ci/templates/volumes/laminar/run/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/playbooks/roles/ci/templates/volumes/laminar/run/.gitkeep
diff --git a/playbooks/roles/common/files/authorized_keys b/playbooks/roles/common/files/authorized_keys
index 60edc04..82f2cbb 100644
--- a/playbooks/roles/common/files/authorized_keys
+++ b/playbooks/roles/common/files/authorized_keys
@@ -1,2 +1,3 @@
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPnLAE5TrdYF8QWCSkvgUp15XKcwQJ9393a/CghSo8dG serve@ansible
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINkjxFI9i17i1MQXZUBl99OP7nRURHGFItPaCqkUUQJw serve@ci
 {{ me_lizcoffee_key }}
diff --git a/playbooks/roles/docker/tasks/main.yml b/playbooks/roles/docker/tasks/main.yml
index a156e4e..b99437d 100644
--- a/playbooks/roles/docker/tasks/main.yml
+++ b/playbooks/roles/docker/tasks/main.yml
@@ -25,6 +25,12 @@
         {{ ansible_distribution_release }} stable      
     state: present
 
+- name: Make docker group id deterministic
+  ansible.builtin.group:
+    name: docker
+    gid: "{{ docker_gid }}"
+    state: present
+
 - name: Install docker
   ansible.builtin.apt:
     name:
diff --git a/playbooks/roles/mon/tasks/main.yml b/playbooks/roles/mon/tasks/main.yml
new file mode 100644
index 0000000..c45b377
--- /dev/null
+++ b/playbooks/roles/mon/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+
+- name: Deploy mon
+  ansible.builtin.import_tasks: manage-docker-swarm-service.yml
+  vars:
+    service_name: mon
+    template_render_dir: "../templates"
+    service_destination_dir: "{{ mon_base }}"
diff --git a/playbooks/roles/mon/templates/stacks/docker-compose.yml b/playbooks/roles/mon/templates/stacks/docker-compose.yml
new file mode 100644
index 0000000..ff7269f
--- /dev/null
+++ b/playbooks/roles/mon/templates/stacks/docker-compose.yml
@@ -0,0 +1,31 @@
+services:
+  mon:
+    image: twinproduction/gatus:latest
+    volumes:
+      - {{ mon_base }}/volumes/data:/data
+    environment:
+      - TZ={{ timezone }}
+      - DEPLOYMENT_TIME={{ deployment_time }}
+    networks:
+      - proxy
+    deploy:
+      mode: replicated
+      update_config:
+        parallelism: 1
+        failure_action: rollback
+        order: start-first
+        delay: 5s
+        monitor: 30s
+      replicas: 1
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=proxy
+        - traefik.http.routers.mon.tls=true
+        - traefik.http.routers.mon.tls.certResolver=letsencrypt
+        - traefik.http.routers.mon.rule=Host(`{{ mon_domain }}`)
+        - traefik.http.routers.mon.entrypoints=websecure
+        - traefik.http.services.mon.loadbalancer.server.port=8080
+
+networks:
+  proxy:
+    external: true
diff --git a/playbooks/roles/mon/templates/volumes/data/.gitkeep b/playbooks/roles/mon/templates/volumes/data/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/playbooks/roles/mon/templates/volumes/data/.gitkeep
diff --git a/playbooks/roles/outbound/templates/headscale/config/acl.json b/playbooks/roles/outbound/templates/headscale/config/acl.json
index dcdd954..410de11 100644
--- a/playbooks/roles/outbound/templates/headscale/config/acl.json
+++ b/playbooks/roles/outbound/templates/headscale/config/acl.json
@@ -2,7 +2,7 @@
   "groups": {
     "group:vpn_admins": [
 {% for user in vpn_admins %}
-      "{{ user }}{{ oauth_user_suffix }}"{{ ", " if not loop.last else "" }}
+      "{{ user }}@{{ oauth_user_suffix }}"{{ ", " if not loop.last else "" }}
 {% endfor %}
     ]
   },
@@ -10,26 +10,26 @@
 {% for user in vpn_users %}
     {
       "action": "accept",
-      "src": ["{{ user }}{{ oauth_user_suffix }}"],
-      "dst": ["{{ user }}{{ oauth_user_suffix }}:*"]
+      "src": ["{{ user }}@{{ oauth_user_suffix }}"],
+      "dst": ["{{ user }}@{{ oauth_user_suffix }}:*"]
     },
 {% endfor %}
     {
       "action": "accept",
-      "src": ["{{ auth_key_user }}"],
-      "dst": ["{{ auth_key_user }}:*", "{{ loadbalancer_ip }}/32:*"]
+      "src": ["{{ auth_key_user }}@"],
+      "dst": ["{{ auth_key_user }}@:*", "{{ loadbalancer_ip }}/32:*"]
     },
 {% for user, m in mesh.items() %}
     {
       "action": "accept",
-      "src": ["{{ user }}{{ oauth_user_suffix }}"],
-      "dst": ["{{ m.gateway }}/32:*]"
+      "src": ["{{ user }}@{{ oauth_user_suffix }}"],
+      "dst": ["{{ m.gateway }}/32:*"]
     },
 {% endfor %}
     {
       "action": "accept",
       "src": ["group:vpn_admins"],
-      "dst": ["{{ loadbalancer_ip }}/32:*"]
+      "dst": [{% for user, m in mesh.items() %} "{{ m.gateway }}/32:*", {% endfor %} "{{ loadbalancer_ip }}/32:*"]
     }
   ]
 }
diff --git a/playbooks/roles/outbound/templates/headscale/config/config.yaml b/playbooks/roles/outbound/templates/headscale/config/config.yaml
index d3bff5a..54657b2 100644
--- a/playbooks/roles/outbound/templates/headscale/config/config.yaml
+++ b/playbooks/roles/outbound/templates/headscale/config/config.yaml
@@ -120,14 +120,18 @@ policy:
 dns:
   magic_dns: true
   base_domain: "{{ headscale_base_domain }}"
+  search_domains: []
   nameservers:
     global:
       - {{ headscale_dns_for_connected_clients_1 }}
       - {{ headscale_dns_for_connected_clients_2 }}
     split:
-      {{ domain }}:
-        - {{ loadbalancer_ip }}
-  search_domains: []
+{% for user, m in mesh.items() %}
+{% if "split_vpn_dns_to" in m %}
+      {{ m.domain }}:
+        - {{ m.split_vpn_dns_to }}
+{% endif %}
+{% endfor %}
 
 unix_socket: /var/run/headscale/headscale.sock
 unix_socket_permission: "0770"
@@ -151,7 +155,6 @@ oidc:
     - {{ domain }}
   allowed_groups:
     - vpn@{{ idm_domain }}
-  strip_email_domain: true
 
 # Logtail configuration
 # Logtail is Tailscales logging and auditing infrastructure, it allows the control panel
diff --git a/playbooks/roles/outbound/templates/proxy/nginx/conf.d/mon.conf b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/mon.conf
new file mode 100644
index 0000000..601e200
--- /dev/null
+++ b/playbooks/roles/outbound/templates/proxy/nginx/conf.d/mon.conf
@@ -0,0 +1,19 @@
+server {
+    listen 80;
+    server_name mon.liz.coffee;
+
+    real_ip_header X-Forwarded-For;
+    real_ip_recursive on;
+    set_real_ip_from {{ docker_network }};
+
+    location / {
+        proxy_pass https://{{ loadbalancer_ip }};
+        proxy_ssl_verify off;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
diff --git a/playbooks/roles/src/templates/stacks/docker-compose.yml b/playbooks/roles/src/templates/stacks/docker-compose.yml
index 3ac70f9..ec514b4 100644
--- a/playbooks/roles/src/templates/stacks/docker-compose.yml
+++ b/playbooks/roles/src/templates/stacks/docker-compose.yml
@@ -1,5 +1,37 @@
 services:
-  # TODO: own cgit fork
+  frontend:
+    image: oci.liz.coffee/emprespresso/cgit:release
+    volumes:
+      - {{ src_base }}/volumes/data/repos:/srv/git:ro
+    environment:
+      CGIT_TITLE: '{{ src_domain }}'
+      CGIT_DESC: '<3 {{ domain }}'
+      CGIT_VROOT: '/cgit'
+      CGIT_SECTION_FROM_STARTPATH: 1
+      CGIT_MAX_REPO_COUNT: 100
+    networks:
+      - proxy
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://localhost"]
+      timeout: 15s
+      interval: 30s
+      retries: 3
+      start_period: 5s
+    deploy:
+      mode: replicated
+      update_config:
+        parallelism: 1
+        failure_action: rollback
+        order: start-first
+        monitor: 10s
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=proxy
+        - traefik.http.routers.src.tls=true
+        - traefik.http.routers.src.tls.certResolver=letsencrypt
+        - traefik.http.routers.src.rule=Host(`{{ src_domain }}`)
+        - traefik.http.routers.src.entrypoints=websecure
+        - traefik.http.services.src.loadbalancer.server.port=80
 
   src:
     image: charmcli/soft-serve
diff --git a/playbooks/roles/src/templates/volumes/soft-serve/hooks/update b/playbooks/roles/src/templates/volumes/soft-serve/hooks/update
index a97e5f9..c209b41 100755
--- a/playbooks/roles/src/templates/volumes/soft-serve/hooks/update
+++ b/playbooks/roles/src/templates/volumes/soft-serve/hooks/update
@@ -41,22 +41,23 @@ refname="$1"
 _oldrev="$2"
 rev="$3"
 
-function post_trigger_ci_jobs() {
-    local host="cihooks"
+post_trigger_ci_jobs() {
+    local host="ci_server"
     local port="9000"
     local path="/job"
     local json_payload=$(printf '{"type": "ci_pipeline", "arguments": {"remote": "%s", "rev": "%s", "refname": "%s"}}' "$1" "$2" "$3")
-  
+
     echo "> $json_payload"
-  
-    which curl 2&>/dev/null || apk add -q curl
 
-    curl --silent --show-error -X POST \
-         -H "Content-Type: application/json" \ -H "Connection: close" \
+    which curl 2&>/dev/null || apk add -q curl
+    curl -X POST \
+         -H "Content-Type: application/json" \
+         -H "Connection: close" \
          -d "$json_payload" \
+         --no-progress-meter \
          "http://$host:$port$path"
 
-    echo "... Done!"
+    echo "... Done"
 }
 # -- </continuous_integration> --
 
diff --git a/tasks/copy-rendered-templates-recursive.yml b/tasks/copy-rendered-templates-recursive.yml
index 2b83834..3b6055a 100644
--- a/tasks/copy-rendered-templates-recursive.yml
+++ b/tasks/copy-rendered-templates-recursive.yml
@@ -14,6 +14,7 @@
     path: "{{ tempdir.path }}/{{ item.path | dirname }}"
     state: directory
     mode: "{{ mode | default('0755') }}"
+    owner: "{{ owner }}"
   with_filetree: "{{ render_dir }}"
   when: item.state == "file"
 
@@ -55,6 +56,7 @@
     src: "{{ item.src }}"
     dest: "{{ tempdir.path }}/{{ item.path }}"
     mode: "{{ mode | default('0755') }}"
+    owner: "{{ owner }}"
   loop: "{{ text_files }}"
 
 - name: Copy binary files directly
@@ -64,6 +66,7 @@
     src: "{{ item.src }}"
     dest: "{{ tempdir.path }}/{{ item.path }}"
     mode: "{{ mode | default('0644') }}"
+    owner: "{{ owner }}"
   loop: "{{ binary_files }}"
 
 - name: Sync rendered and copied files to remote host
@@ -72,16 +75,19 @@
   ansible.builtin.synchronize:
     src: "{{ tempdir.path }}/"
     dest: "{{ tempdir.path }}/"
+    owner: true
+    group: true
     recursive: true
 
 - name: Ensure destination exists
   ansible.builtin.file:
     path: "{{ destination_dir }}"
+    owner: "{{ owner }}"
     state: directory
 
-- name: Copy files to final destination
+- name: Copy files to final destination, preserving ownership stuff
   ansible.builtin.command:
-    cmd: bash -c 'cp -r {{ tempdir.path }}/* {{ destination_dir }}/'
+    cmd: bash -c 'cp -rp {{ tempdir.path }}/* {{ destination_dir }}/'
 
 - name: Remove local temporary directory
   delegate_to: localhost
diff --git a/tasks/manage-docker-compose-service.yml b/tasks/manage-docker-compose-service.yml
index 1910b0f..d53bc94 100644
--- a/tasks/manage-docker-compose-service.yml
+++ b/tasks/manage-docker-compose-service.yml
@@ -3,6 +3,8 @@
 - name: "Copy rendered templates for {{ service_name }}"
   ansible.builtin.import_tasks: copy-rendered-templates-recursive.yml
   vars:
+    owner: "{{ service_owner | default('1000') }}"
+    mode: "{{ file_mode | default('777') }}"
     render_dir: "{{ template_render_dir }}"
     destination_dir: "{{ service_destination_dir }}"
 
diff --git a/tasks/manage-docker-swarm-service.yml b/tasks/manage-docker-swarm-service.yml
index 3d01e1c..811ec8b 100644
--- a/tasks/manage-docker-swarm-service.yml
+++ b/tasks/manage-docker-swarm-service.yml
@@ -3,7 +3,8 @@
 - name: "Copy rendered templates for {{ service_name }}"
   ansible.builtin.import_tasks: copy-rendered-templates-recursive.yml
   vars:
-    mode: "0777"
+    owner: "{{ service_owner | default('1000') }}"
+    mode: "{{ file_mode | default('777') }}"
     render_dir: "{{ template_render_dir }}"
     destination_dir: "{{ service_destination_dir }}"