diff options
27 files changed, 349 insertions, 126 deletions
@@ -12,5 +12,14 @@ - name: Outbound ansible.builtin.import_playbook: playbooks/deploy-outbound.yml +- name: Ceph mountpoints + ansible.builtin.import_playbook: playbooks/deploy-ceph-mount.yml + - name: Swarm - ansible.builtin.import_playbook: playbooks/deploy-swarm.yml + ansible.builtin.import_playbook: playbooks/deploy-swarm-cluster.yml + +- name: Traefik + ansible.builtin.import_playbook: playbooks/deploy-traefik.yml + +- name: Portainer + ansible.builtin.import_playbook: playbooks/deploy-portainer.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index 11235f5..197662f 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -6,3 +6,6 @@ rfc1918_cgnat_networks: - 172.16.0.0/12 - 192.168.0.0/16 - 100.64.0.0/10 + +timezone: "America/Los_Angeles" +traefik_domain: "sips.liz.coffee" diff --git a/group_vars/ceph.yml b/group_vars/ceph.yml new file mode 100644 index 0000000..5c985ca --- /dev/null +++ b/group_vars/ceph.yml @@ -0,0 +1,9 @@ +--- + +# ceph_secret: <keep it safe in the vault> +cephfs_name: cephfs +ceph_mon_host: "[v2:10.128.0.101:3300/0,v1:10.128.0.101:6789/0] [v2:10.128.0.103:3300/0,v1:10.128.0.103:6789/0] [v2:10.128.0.102:3300/0,v1:10.128.0.102:6789/0]" +ceph_fsid: "ee994518-d7f3-4a7b-b148-09dba7f3dd4d" +ceph_client_name: swarm + +ceph_base: "/mnt/ceph" diff --git a/group_vars/nginx-proxy.yml b/group_vars/nginx-proxy.yml index f23cf75..bd5a27a 100644 --- a/group_vars/nginx-proxy.yml +++ b/group_vars/nginx-proxy.yml @@ -1,3 +1,4 @@ --- certs_email: elizabeth@simponic.xyz +nginx_proxy_base: "/etc/docker/compose/nginx-proxy" diff --git a/group_vars/portainer.yml b/group_vars/portainer.yml new file mode 100644 index 0000000..aa7726c --- /dev/null +++ b/group_vars/portainer.yml @@ -0,0 +1,4 @@ +--- + +portainer_base: "{{ swarm_base }}/portainer" +portainer_host: "swarm.{{ traefik_domain }}" diff --git a/group_vars/swarm.yml b/group_vars/swarm.yml index 92e63bc..bf0744d 100644 --- a/group_vars/swarm.yml +++ b/group_vars/swarm.yml @@ -1,8 +1,3 @@ --- -# ceph_secret: <keep it safe in the vault> -cephfs_name: cephfs -ceph_mon_host: "[v2:10.128.0.101:3300/0,v1:10.128.0.101:6789/0] [v2:10.128.0.103:3300/0,v1:10.128.0.103:6789/0] [v2:10.128.0.102:3300/0,v1:10.128.0.102:6789/0]" -ceph_fsid: "ee994518-d7f3-4a7b-b148-09dba7f3dd4d" -ceph_client_name: swarm -ceph_mons: "10.128.0.101:6789/10.128.0.102:6789/10.128.0.103:6789" +swarm_base: "{{ ceph_base }}/docker" diff --git a/group_vars/traefik.yml b/group_vars/traefik.yml new file mode 100644 index 0000000..85d890b --- /dev/null +++ b/group_vars/traefik.yml @@ -0,0 +1,6 @@ +--- + +# super incredible processing servers +traefik_domain: sips.liz.coffee +certs_email: "{{ cloudflare_email }}" +traefik_base: "{{ swarm_base }}/traefik" @@ -14,7 +14,18 @@ outbound-one.liz.coffee ansible_user=serve ansible_connection=ssh ansible_becom outbound-one.liz.coffee ansible_user=serve ansible_connection=ssh ansible_become_password='{{ outbound_one_become_password }}' # outbound-two.liz.coffee ansible_user=serve ansible_connection=ssh ansible_become_password='{{ vpn_become_password }}' +[ceph] +swarm-one ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' +swarm-two ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' +swarm-three ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' + [swarm] swarm-one ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' swarm-two ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' swarm-three ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' + +[portainer] +swarm-one ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' + +[traefik] +swarm-one ansible_user=serve ansible_connection=ssh ansible_become_password='{{ swarm_become_password }}' diff --git a/playbooks/deploy-ceph-mount.yml b/playbooks/deploy-ceph-mount.yml new file mode 100644 index 0000000..de2dd5b --- /dev/null +++ b/playbooks/deploy-ceph-mount.yml @@ -0,0 +1,7 @@ +--- + +- name: Setup ceph + hosts: ceph + become: true + roles: + - ceph diff --git a/playbooks/deploy-portainer.yml b/playbooks/deploy-portainer.yml new file mode 100644 index 0000000..1eb71d4 --- /dev/null +++ b/playbooks/deploy-portainer.yml @@ -0,0 +1,7 @@ +--- + +- name: portainer setup + hosts: portainer + become: true + roles: + - portainer diff --git a/playbooks/deploy-swarm-cluster.yml b/playbooks/deploy-swarm-cluster.yml new file mode 100644 index 0000000..22dcdb7 --- /dev/null +++ b/playbooks/deploy-swarm-cluster.yml @@ -0,0 +1,34 @@ +--- + +- name: Configure Docker Swarm Firewall Rules + hosts: swarm + become: true + tasks: + - name: Enable Local Swarm Communications + loop: "{{ rfc1918_cgnat_networks }}" + community.general.ufw: + rule: allow + port: "2377" + from: "{{ item }}" + state: enabled + + - name: Enable Local Swarm Communications + loop: "{{ rfc1918_cgnat_networks }}" + community.general.ufw: + rule: allow + port: "9001" + from: "{{ item }}" + state: enabled + +- name: Setup swarm on init node + hosts: swarm[0] + become: true + roles: + - swarm-init + +- name: Join non-init nodes + hosts: swarm:!swarm[0] + become: true + roles: + - swarm-join + diff --git a/playbooks/deploy-swarm.yml b/playbooks/deploy-swarm.yml deleted file mode 100644 index fc4da39..0000000 --- a/playbooks/deploy-swarm.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- - -- name: Install Ceph - hosts: swarm - become: true - tasks: - - name: Install Ceph - ansible.builtin.apt: - name: - - ceph-common - - ceph-fuse - state: present - - # - name: Copy Ceph Secret - # ansible.builtin.copy: - # content: "{{ ceph_secret }}" - # dest: /etc/ceph/secret.key - - # ceph config generate-minimal-conf - - name: Copy Ceph Configuration - ansible.builtin.copy: - content: "[global]\n fsid = {{ ceph_fsid }}\n mon_host = {{ ceph_mon_host }}\n" - dest: /etc/ceph/ceph.conf - mode: '0644' - - # ceph fs authorize cephfs client.swarm / rw - - name: Copy Ceph Keyring - ansible.builtin.copy: - content: "[client.{{ ceph_client_name }}]\n key = {{ ceph_secret }}\n" - dest: "/etc/ceph/ceph.client.{{ ceph_client_name }}.keyring" - mode: '0600' - - - name: Adjust ceph mount perms - ansible.builtin.file: - path: /mnt/ceph - owner: root - group: root - state: directory - recurse: true - - - name: Mount Ceph on Boot - ansible.builtin.lineinfile: - path: /etc/fstab - regexp: ':/\s+/mnt\s+ceph' - line: "none /mnt/ceph fuse.ceph ceph.id={{ ceph_client_name }},_netdev,defaults 0 0" - create: true - mode: "0644" - - - name: Mount ceph now - ansible.builtin.shell: - cmd: "mount -a" - - - name: Adjust ceph mount perms for docker - ansible.builtin.file: - path: /mnt/ceph/docker - owner: root - group: docker - state: directory - recurse: true - -- name: Initial docker swarm fw rules - hosts: swarm - become: true - tasks: - - name: Enable local swarm comms - loop: "{{ rfc1918_cgnat_networks }}" - community.general.ufw: - rule: allow - port: "2377" - from: "{{ item }}" - state: "enabled" - -- name: Initial docker swarm init - hosts: swarm[0] - become: true - tasks: - - name: Check Docker Swarm status - ansible.builtin.shell: docker info --format '{{ "{{.Swarm.LocalNodeState}}" }}' - register: docker_swarm_status - changed_when: false - - - name: Initialize Docker Swarm - ansible.builtin.shell: - cmd: docker swarm init --advertise-addr {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} - when: "'inactive' in docker_swarm_status.stdout" - register: swarm_init - changed_when: "'Swarm initialized' in swarm_init.stdout" - - - name: Retrieve Docker Swarm manager token - ansible.builtin.shell: docker swarm join-token manager -q - register: manager_token - changed_when: false - -- name: Join remaining managers to Docker Swarm - hosts: swarm:!swarm[0] - become: true - tasks: - - name: Check Docker Swarm status before attempting to join - ansible.builtin.shell: docker info --format '{{ "{{.Swarm.LocalNodeState}}" }}' - register: docker_swarm_status - changed_when: false - - - name: Join Swarm as manager - ansible.builtin.shell: - cmd: docker swarm join --token {{ hostvars[groups['swarm'][0]]['manager_token'].stdout }} {{ hostvars[groups['swarm'][0]]['ansible_default_ipv4']['address'] }}:2377 - when: hostvars[groups['swarm'][0]]['manager_token'].stdout is defined and docker_swarm_status.stdout != "active" - register: swarm_join - changed_when: "'This node joined a swarm as a manager' in swarm_join.stdout" - - - name: Label Docker Swarm manager nodes - ansible.builtin.shell: - cmd: docker node update --label-add manager=true {{ ansible_hostname }} - when: swarm_join is changed - changed_when: false - diff --git a/playbooks/deploy-traefik.yml b/playbooks/deploy-traefik.yml new file mode 100644 index 0000000..e26cf4b --- /dev/null +++ b/playbooks/deploy-traefik.yml @@ -0,0 +1,7 @@ +--- + +- name: traefik setup + hosts: traefik + become: true + roles: + - traefik diff --git a/playbooks/roles/ceph/tasks/main.yml b/playbooks/roles/ceph/tasks/main.yml new file mode 100644 index 0000000..b554340 --- /dev/null +++ b/playbooks/roles/ceph/tasks/main.yml @@ -0,0 +1,44 @@ +--- + +- name: Install Ceph Packages + ansible.builtin.apt: + name: + - ceph-common + - ceph-fuse + state: present + +- name: Copy Ceph Configuration + ansible.builtin.copy: + content: | + [global] + fsid = {{ ceph_fsid }} + mon_host = {{ ceph_mon_host }} + dest: /etc/ceph/ceph.conf + mode: '0644' + +- name: Copy Ceph Keyring + ansible.builtin.copy: + content: | + [client.{{ ceph_client_name }}] + key = {{ ceph_secret }} + dest: "/etc/ceph/ceph.client.{{ ceph_client_name }}.keyring" + mode: '0600' + +- name: Ensure Ceph Base Exists + ansible.builtin.file: + path: "{{ ceph_base }}" + owner: root + group: root + state: directory + recurse: true + +- name: Mount Ceph on Boot + ansible.builtin.lineinfile: + path: /etc/fstab + regexp: '{{ ceph_base }}\w+fuse.ceph' + line: "none {{ ceph_base }} fuse.ceph ceph.id={{ ceph_client_name }},_netdev,defaults 0 0" + create: true + mode: "0644" + +- name: Mount Ceph Now + ansible.builtin.command: mount -a diff --git a/playbooks/roles/nginx-proxy/handlers/main.yml b/playbooks/roles/nginx-proxy/handlers/main.yml index 43302b5..98486dc 100644 --- a/playbooks/roles/nginx-proxy/handlers/main.yml +++ b/playbooks/roles/nginx-proxy/handlers/main.yml @@ -5,3 +5,5 @@ name: docker-compose@nginx-proxy state: restarted enabled: true + when: compose_mode is not defined or compose_mode != false + diff --git a/playbooks/roles/nginx-proxy/tasks/main.yml b/playbooks/roles/nginx-proxy/tasks/main.yml index 9c14072..50958e7 100644 --- a/playbooks/roles/nginx-proxy/tasks/main.yml +++ b/playbooks/roles/nginx-proxy/tasks/main.yml @@ -3,14 +3,14 @@ - name: Build nginx-proxy compose dirs ansible.builtin.file: state: directory - dest: '/etc/docker/compose/nginx-proxy/{{ item.path }}' + dest: '{{ nginx_proxy_base }}/{{ item.path }}' with_filetree: '../templates' when: item.state == 'directory' - name: Build nginx-proxy compose files ansible.builtin.template: src: '{{ item.src }}' - dest: '/etc/docker/compose/nginx-proxy/{{ item.path }}' + dest: '{{ nginx_proxy_base }}/{{ item.path }}' with_filetree: '../templates' when: item.state == 'file' notify: diff --git a/playbooks/roles/nginx-proxy/templates/docker-compose.yml b/playbooks/roles/nginx-proxy/templates/docker-compose.yml index fd49712..e0f56c4 100644 --- a/playbooks/roles/nginx-proxy/templates/docker-compose.yml +++ b/playbooks/roles/nginx-proxy/templates/docker-compose.yml @@ -9,7 +9,7 @@ services: - "443:443" volumes: - /var/run/docker.sock:/tmp/docker.sock:ro - - ./certs:/etc/nginx/certs + - {{ nginx_proxy_base }}/certs:/etc/nginx/certs networks: - proxy labels: @@ -22,7 +22,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - acme:/etc/acme.sh - - ./certs:/etc/nginx/certs + - {{ nginx_proxy_base }}/certs:/etc/nginx/certs environment: - "DEFAULT_EMAIL={{ certs_email }}" - "ACME_CHALLENGE=DNS-01" diff --git a/playbooks/roles/portainer/tasks/main.yml b/playbooks/roles/portainer/tasks/main.yml new file mode 100644 index 0000000..db9146d --- /dev/null +++ b/playbooks/roles/portainer/tasks/main.yml @@ -0,0 +1,19 @@ +--- + +- name: Build portainer compose dirs + ansible.builtin.file: + state: directory + dest: '{{ portainer_base }}/{{ item.path }}' + with_filetree: '../templates' + when: item.state == 'directory' + +- name: Build portainer compose files + ansible.builtin.template: + src: '{{ item.src }}' + dest: '{{ portainer_base }}/{{ item.path }}' + with_filetree: '../templates' + when: item.state == 'file' + +- name: Deploy Portainer stack + ansible.builtin.command: + cmd: "docker stack deploy -c {{ portainer_base }}/stacks/docker-compose.yml portainer" diff --git a/playbooks/roles/portainer/templates/stacks/docker-compose.yml b/playbooks/roles/portainer/templates/stacks/docker-compose.yml new file mode 100644 index 0000000..1a02cef --- /dev/null +++ b/playbooks/roles/portainer/templates/stacks/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.2' + +services: + agent: + image: portainer/agent:2.21.5 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/var/lib/docker/volumes + networks: + - agent_network + deploy: + mode: global + placement: + constraints: [node.platform.os == linux] + + portainer: + image: portainer/portainer-ce:alpine + command: -H tcp://tasks.agent:9001 --tlsskipverify + ports: + - "8000:8000" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - {{ portainer_base }}/volumes/data:/data + networks: + - proxy + - agent_network + deploy: + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] + labels: + - traefik.enable=true + - traefik.swarm.network=proxy + - traefik.http.routers.portainer.rule=Host(`{{ portainer_host }}`) + - traefik.http.routers.portainer.entrypoints=websecure + - traefik.http.routers.portainer.tls=true + - traefik.http.routers.portainer.tls.certResolver=letsencrypt + - traefik.http.services.portainer.loadbalancer.server.port=9000 + +networks: + proxy: + external: true + agent_network: + driver: overlay + attachable: true diff --git a/playbooks/roles/portainer/templates/volumes/data/.gitkeep b/playbooks/roles/portainer/templates/volumes/data/.gitkeep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/playbooks/roles/portainer/templates/volumes/data/.gitkeep diff --git a/playbooks/roles/swarm-init/tasks/main.yml b/playbooks/roles/swarm-init/tasks/main.yml new file mode 100644 index 0000000..19967e9 --- /dev/null +++ b/playbooks/roles/swarm-init/tasks/main.yml @@ -0,0 +1,19 @@ +--- + +- name: Check Docker Swarm Status + ansible.builtin.command: docker info --format '{{ "{{.Swarm.LocalNodeState}}" }}' + register: docker_swarm_status + changed_when: false + +- name: Initialize Docker Swarm if Inactive + ansible.builtin.command: + cmd: docker swarm init --advertise-addr "{{ ansible_default_ipv4.address }}" + when: docker_swarm_status.stdout == "inactive" + register: swarm_init + changed_when: '"Swarm initialized" in swarm_init.stdout' + +- name: Retrieve Docker Swarm Manager Token + ansible.builtin.command: docker swarm join-token manager -q + register: manager_token + changed_when: false + diff --git a/playbooks/roles/swarm-join/tasks/main.yml b/playbooks/roles/swarm-join/tasks/main.yml new file mode 100644 index 0000000..5fdb66f --- /dev/null +++ b/playbooks/roles/swarm-join/tasks/main.yml @@ -0,0 +1,21 @@ +--- + +- name: Check Docker Swarm Status + ansible.builtin.command: docker info --format '{{ "{{.Swarm.LocalNodeState}}" }}' + register: docker_swarm_status + changed_when: false + +- name: Join Swarm as Manager + ansible.builtin.command: + cmd: docker swarm join --token {{ hostvars[groups['swarm'][0]]['manager_token'].stdout }} {{ hostvars[groups['swarm'][0]]['ansible_default_ipv4']['address'] }}:2377 + when: + - hostvars[groups['swarm'][0]]['manager_token'].stdout is defined + - docker_swarm_status.stdout != "active" + register: swarm_join + changed_when: '"This node joined a swarm as a manager" in swarm_join.stdout' + +- name: Label Docker Swarm Manager Nodes + ansible.builtin.command: + cmd: docker node update --label-add manager=true {{ ansible_hostname }} + when: swarm_join is changed + changed_when: false diff --git a/playbooks/roles/traefik/tasks/main.yml b/playbooks/roles/traefik/tasks/main.yml new file mode 100644 index 0000000..c365f55 --- /dev/null +++ b/playbooks/roles/traefik/tasks/main.yml @@ -0,0 +1,19 @@ +--- + +- name: Build traefik compose dirs + ansible.builtin.file: + state: directory + dest: '{{ traefik_base }}/{{ item.path }}' + with_filetree: '../templates' + when: item.state == 'directory' + +- name: Build traefik compose files + ansible.builtin.template: + src: '{{ item.src }}' + dest: '{{ traefik_base }}/{{ item.path }}' + with_filetree: '../templates' + when: item.state == 'file' + +- name: Deploy Traefik stack + ansible.builtin.command: + cmd: "docker stack deploy -c {{ traefik_base }}/stacks/docker-compose.yml traefik" diff --git a/playbooks/roles/traefik/templates/stacks/docker-compose.yml b/playbooks/roles/traefik/templates/stacks/docker-compose.yml new file mode 100644 index 0000000..4504af9 --- /dev/null +++ b/playbooks/roles/traefik/templates/stacks/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.8' +services: + traefik: + image: traefik:v3 + ports: + - 80:80 + - 443:443 + environment: + - TZ={{ timezone }} + - CF_API_EMAIL={{ cloudflare_email }} + - CF_DNS_API_TOKEN={{ cloudflare_dns_api_token }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - {{ traefik_base }}/stacks/traefik.yml:/traefik.yml + - {{ traefik_base }}/volumes/certs:/certs + networks: + - proxy + deploy: + mode: global + placement: + constraints: [node.role == manager] + labels: + - traefik.enable=true + - traefik.http.routers.dashboard.rule=Host(`traefik.{{ traefik_domain }}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard/`)) + - traefik.http.routers.dashboard.service=api@internal + - traefik.http.routers.dashboard.tls=true + - traefik.http.routers.dashboard.tls.certresolver=letsencrypt + - traefik.http.routers.ping.rule=Host(`traefik.{{ traefik_domain }}`) && PathPrefix(`/ping`) + - traefik.http.routers.ping.service=ping@internal + - traefik.http.routers.ping.tls=true + - traefik.http.routers.ping.tls.certresolver=letsencrypt + - traefik.http.services.dashboard.loadbalancer.server.port=8080 + - traefik.http.services.ping.loadbalancer.server.port=8080 + +networks: + proxy: + name: proxy + driver: overlay + attachable: true diff --git a/playbooks/roles/traefik/templates/stacks/traefik.yml b/playbooks/roles/traefik/templates/stacks/traefik.yml new file mode 100644 index 0000000..a80c261 --- /dev/null +++ b/playbooks/roles/traefik/templates/stacks/traefik.yml @@ -0,0 +1,35 @@ +ping: {} +accessLog: {} +log: + level: DEBUG +api: + dashboard: true + insecure: true + debug: false +entryPoints: + web: + address: ":80" + http: + redirections: + entryPoint: + to: websecure + scheme: https + websecure: + address: ":443" +serversTransport: + insecureSkipVerify: true +providers: + swarm: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false + network: proxy +certificatesResolvers: + letsencrypt: + acme: + email: {{ certs_email }} + storage: /certs/acme.json + caServer: https://acme-v02.api.letsencrypt.org/directory + # caServer: https://acme-staging-v02.api.letsencrypt.org/directory # staging + dnsChallenge: + provider: cloudflare + delayBeforeCheck: 10 diff --git a/playbooks/roles/traefik/templates/volumes/certs/.gitkeep b/playbooks/roles/traefik/templates/volumes/certs/.gitkeep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/playbooks/roles/traefik/templates/volumes/certs/.gitkeep diff --git a/secrets.txt b/secrets.txt index fda8656..1b142e1 100644 --- a/secrets.txt +++ b/secrets.txt @@ -1,5 +1,6 @@ swarm_become_password outbound_one_become_password cloudflare_token +cloudflare_dns_api_token cloudflare_email ceph_secret |