From fa8f3f9465e87d499f7d6428323f496a884b7818 Mon Sep 17 00:00:00 2001 From: Elizabeth Alexander Hunt Date: Sat, 10 May 2025 16:57:03 -0700 Subject: initial commit --- .ci/ci.json | 3 ++ .ci/ci.ts | 82 +++++++++++++++++++++++++++++++++++++++++ .dockerignore | 8 ++++ Dockerfile | 25 +++++++++++++ deno.json | 3 ++ hooks/Dockerfile | 3 ++ hooks/deno.json | 4 ++ hooks/mod.ts | 57 ++++++++++++++++++++++++++++ model/deno.json | 5 +++ model/job.ts | 41 +++++++++++++++++++++ model/mod.ts | 2 + model/pipeline.ts | 76 ++++++++++++++++++++++++++++++++++++++ utils/deno.json | 5 +++ utils/env.ts | 5 +++ utils/mod.ts | 4 ++ utils/run.ts | 21 +++++++++++ utils/secret.ts | 46 +++++++++++++++++++++++ utils/validate_identifier.ts | 3 ++ worker/Dockerfile | 20 ++++++++++ worker/deno.json | 4 ++ worker/jobs/checkout_ci.run | 39 ++++++++++++++++++++ worker/mod.ts | 0 worker/scripts/ansible_playbook | 0 worker/scripts/build_image | 67 +++++++++++++++++++++++++++++++++ worker/scripts/fetch_code | 6 +++ worker/scripts/run_pipeline | 28 ++++++++++++++ 26 files changed, 557 insertions(+) create mode 100644 .ci/ci.json create mode 100644 .ci/ci.ts create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 deno.json create mode 100644 hooks/Dockerfile create mode 100644 hooks/deno.json create mode 100644 hooks/mod.ts create mode 100644 model/deno.json create mode 100644 model/job.ts create mode 100644 model/mod.ts create mode 100644 model/pipeline.ts create mode 100644 utils/deno.json create mode 100644 utils/env.ts create mode 100644 utils/mod.ts create mode 100644 utils/run.ts create mode 100644 utils/secret.ts create mode 100644 utils/validate_identifier.ts create mode 100644 worker/Dockerfile create mode 100644 worker/deno.json create mode 100644 worker/jobs/checkout_ci.run create mode 100644 worker/mod.ts create mode 100644 worker/scripts/ansible_playbook create mode 100644 worker/scripts/build_image create mode 100644 worker/scripts/fetch_code create mode 100644 worker/scripts/run_pipeline diff --git a/.ci/ci.json b/.ci/ci.json new file mode 100644 index 0000000..dc29f8d --- /dev/null +++ b/.ci/ci.json @@ -0,0 +1,3 @@ +{ + "pipeline": ".ci/ci.ts" +} diff --git a/.ci/ci.ts b/.ci/ci.ts new file mode 100644 index 0000000..c876762 --- /dev/null +++ b/.ci/ci.ts @@ -0,0 +1,82 @@ +#!/usr/bin/env -S deno run --allow-env + +import { + BuildDockerImageJob, + DefaultGitHookPipelineBuilder, +} from "@liz-ci/model"; +import { AnsiblePlaybookJob, FetchCodeJob } from "../model/job.ts"; + +const REGISTRY = "oci.liz.coffee"; +const NAMESPACE = "img"; +const IMG = "liz-ci"; + +const getPipeline = () => { + const gitHookPipeline = new DefaultGitHookPipelineBuilder(); + const branch = gitHookPipeline.getBranch(); + if (!branch) return gitHookPipeline.build(); + + const commonBuildArgs = { + registry: REGISTRY, + namespace: NAMESPACE, + imageTag: branch, + }; + + const ciPackageBuild: BuildDockerImageJob = { + type: "build_docker_image", + arguments: { + ...commonBuildArgs, + context: gitHookPipeline.getSourceDestination(), + repository: IMG, + buildTarget: IMG, + dockerfile: "Dockerfile", + }, + }; + gitHookPipeline.addStage({ + parallelJobs: [ciPackageBuild], + }); + + const subPackages = [ + "worker", + "hooks", + ].map((_package) => ({ + type: "build_docker_image", + arguments: { + ...commonBuildArgs, + repository: `${IMG}-${_package}`, + buildTarget: _package, + dockerfile: `${_package}/Dockerfile`, + }, + })); + gitHookPipeline.addStage({ + parallelJobs: subPackages, + }); + + const isRelease = branch === "release"; + if (!isRelease) { + return gitHookPipeline.build(); + } + + const fetchAnsibleCode: FetchCodeJob = { + type: "fetch_code", + arguments: { + remoteUrl: "ssh://src.liz.coffee:2222/infra", + checkout: "main", + path: "infra", + }, + }; + const thenDeploy: AnsiblePlaybookJob = { + type: "ansible_playbook", + arguments: { + playbooks: "playbooks/ci.yml", + }, + }; + [fetchAnsibleCode, thenDeploy].forEach((deploymentStage) => + gitHookPipeline.addStage({ parallelJobs: [deploymentStage] }) + ); + + return gitHookPipeline.build(); +}; + +if (import.meta.main) { + console.log(getPipeline().serialize()); +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bf4779a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +*.env +.git +.gitignore +**/Dockerfile +README.md +*.log +_build/ +**/node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3e1e6dd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM docker.io/library/alpine:3 AS laminar + +RUN apk add boost-dev build-base capnproto capnproto-dev cmake \ + git ninja rapidjson-dev sqlite-dev zlib zlib-dev +ADD --keep-git-dir https://github.com/ohwgiles/laminar.git \ + /opt/laminar/src +RUN cmake -B /opt/laminar/build -S /opt/laminar/src -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr \ + && cmake --build /opt/laminar/build \ + && cmake --install /opt/laminar/build --strip + +FROM denoland/deno:alpine as liz-ci + +RUN apk add --no-cache capnproto sqlite-libs zlib curl +COPY --from=laminar /usr/sbin/laminard /usr/sbin/laminard +COPY --from=laminar /usr/bin/laminarc /usr/bin/laminarc +COPY --from=laminar /usr/share/man/man8/laminard.8.gz /usr/share/man/man8/laminard.8.gz +COPY --from=laminar /usr/share/man/man1/laminarc.1.gz /usr/share/man/man1/laminarc.1.gz +COPY --from=laminar /etc/laminar.conf /etc/laminar.conf +COPY --from=laminar /usr/share/bash-completion/completions/laminarc /usr/share/bash-completion/completions/laminarc +COPY --from=laminar /usr/share/zsh/site-functions/_laminarc /usr/share/zsh/site-functions/_laminarc + +WORKDIR /app +COPY . /app + +ENTRYPOINT [ "/bin/sh", "-c" ] diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..1dd298e --- /dev/null +++ b/deno.json @@ -0,0 +1,3 @@ +{ + "workspace": ["./model", "./worker", "./hooks", "./utils"] +} diff --git a/hooks/Dockerfile b/hooks/Dockerfile new file mode 100644 index 0000000..a287e2f --- /dev/null +++ b/hooks/Dockerfile @@ -0,0 +1,3 @@ +FROM oci.liz.coffee/img/liz-ci:release as hooks + +CMD [ "deno", "run", "--allow-env", "--allow-net", "/app/hooks/mod.ts" ] diff --git a/hooks/deno.json b/hooks/deno.json new file mode 100644 index 0000000..c4e8fca --- /dev/null +++ b/hooks/deno.json @@ -0,0 +1,4 @@ +{ + "name": "@liz-ci/hooks", + "exports": "./mod.ts" +} diff --git a/hooks/mod.ts b/hooks/mod.ts new file mode 100644 index 0000000..f432b72 --- /dev/null +++ b/hooks/mod.ts @@ -0,0 +1,57 @@ +#!/usr/bin/env -S deno run --allow-env --allow-net + +import { getStdout, validateIdentifier } from "@liz-ci/utils"; + +const addr = { port: 9000, hostname: "0.0.0.0" }; + +Deno.serve(addr, async (req) => { + const { pathname } = new URL(req.url); + if (pathname === "/health") { + try { + await getStdout(["laminarc", "show-jobs"]); + return new Response("think im healthy. lets get to work.", { + status: 200, + }); + } catch (e) { + console.error(e); + return new Response("i need to eat more vegetables -.-", { status: 500 }); + } + } + + if (req.method !== "POST") { + return new Response("invalid method", { + status: 405, + }); + } + + if (pathname === "/checkout_ci") { + const { remote, rev, refname } = await req.json(); + if (![remote, rev, refname].every(validateIdentifier)) { + return new Response("invalid request", { + status: 400, + }); + } + + try { + const laminar = await getStdout([ + "laminarc", + "queue", + "checkout_ci", + `remote="${remote}"`, + `rev="${rev}"`, + `refname="${refname}"`, + ]); + console.log(`successful ci queue :D\n` + laminar); + return new Response(laminar, { + status: 200, + }); + } catch (e) { + console.error(e); + return new Response("womp womp D:", { + status: 500, + }); + } + } + + return new Response("ahhhh idkkkk", { status: 404 }); +}); diff --git a/model/deno.json b/model/deno.json new file mode 100644 index 0000000..d22242e --- /dev/null +++ b/model/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@liz-ci/model", + "version": "0.1.0", + "exports": "./mod.ts" +} diff --git a/model/job.ts b/model/job.ts new file mode 100644 index 0000000..96e0959 --- /dev/null +++ b/model/job.ts @@ -0,0 +1,41 @@ +export type JobArgT = Record; +export interface Job { + readonly type: string; + readonly arguments: JobArgT; +} + +export interface FetchCodeJobProps extends JobArgT { + readonly remoteUrl: string; + readonly checkout: string; + readonly path: string; +} + +export interface FetchCodeJob { + readonly type: "fetch_code"; + readonly arguments: FetchCodeJobProps; +} + +export interface BuildDockerImageJobProps extends JobArgT { + readonly registry: string; + readonly namespace: string; + readonly repository: string; + readonly imageTag: string; + + readonly context: string; + readonly dockerfile: string; + readonly buildTarget: string; +} + +export interface BuildDockerImageJob extends Job { + readonly type: "build_docker_image"; + readonly arguments: BuildDockerImageJobProps; +} + +export interface AnsiblePlaybookJobProps extends JobArgT { + readonly playbooks: string; +} + +export interface AnsiblePlaybookJob extends Job { + readonly type: "ansible_playbook"; + readonly arguments: AnsiblePlaybookJobProps; +} diff --git a/model/mod.ts b/model/mod.ts new file mode 100644 index 0000000..944ab7d --- /dev/null +++ b/model/mod.ts @@ -0,0 +1,2 @@ +export * from "./job.ts"; +export * from "./pipeline.ts"; diff --git a/model/pipeline.ts b/model/pipeline.ts new file mode 100644 index 0000000..06361a4 --- /dev/null +++ b/model/pipeline.ts @@ -0,0 +1,76 @@ +import type { FetchCodeJob, Job } from "./mod.ts"; + +export interface Pipeline { + getStages(): ReadonlyArray; + serialize(): string; +} + +export interface PipelineStage { + readonly parallelJobs: Array; +} + +export interface PipelineBuilder { + addStage(stage: PipelineStage): PipelineBuilder; + build(): Pipeline; +} + +export class PipelineImpl implements Pipeline { + constructor(private readonly serialJobs: ReadonlyArray) {} + + public getStages() { + return this.serialJobs; + } + + public serialize() { + return JSON.stringify(this.serialJobs); + } + + public static from(s: string): Pipeline { + return new PipelineImpl(JSON.parse(s)); + } +} + +abstract class BasePipelineBuilder implements PipelineBuilder { + protected readonly stages: Array = []; + + public addStage(stage: PipelineStage): PipelineBuilder { + this.stages.push(stage); + return this; + } + + public build() { + return new PipelineImpl(this.stages); + } +} + +export class DefaultGitHookPipelineBuilder extends BasePipelineBuilder { + constructor( + private readonly remoteUrl = Deno.env.get("remote")!, + rev = Deno.env.get("rev")!, + private readonly ref = Deno.env.get("ref")!, + ) { + super(); + + this.addStage({ + parallelJobs: [ + { + type: "fetch_code", + arguments: { + remoteUrl, + checkout: rev, + path: this.getSourceDestination(), + }, + }, + ], + }); + } + + public getSourceDestination() { + return this.remoteUrl.split("/").at(-1) ?? "src"; + } + + public getBranch(): string | undefined { + const branchRefPrefix = "refs/heads/"; + return this.ref.split(branchRefPrefix).at(1); + } +} diff --git a/utils/deno.json b/utils/deno.json new file mode 100644 index 0000000..b85c47f --- /dev/null +++ b/utils/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@liz-ci/utils", + "version": "0.1.0", + "exports": "./mod.ts" +} diff --git a/utils/env.ts b/utils/env.ts new file mode 100644 index 0000000..c0cf447 --- /dev/null +++ b/utils/env.ts @@ -0,0 +1,5 @@ +export const getRequiredEnv = (name: string): string => { + const value = Deno.env.get(name); + if (!value) throw new Error(`${name} environment variable is required`); + return value; +}; diff --git a/utils/mod.ts b/utils/mod.ts new file mode 100644 index 0000000..93457f0 --- /dev/null +++ b/utils/mod.ts @@ -0,0 +1,4 @@ +export * from "./env.ts"; +export * from "./run.ts"; +export * from "./secret.ts"; +export * from "./validate_identifier.ts"; diff --git a/utils/run.ts b/utils/run.ts new file mode 100644 index 0000000..f3ce3d3 --- /dev/null +++ b/utils/run.ts @@ -0,0 +1,21 @@ +export const getStdout = async ( + cmd: string[] | string, + options: Deno.CommandOptions = {}, +): Promise => { + const [exec, ...args] = (typeof cmd === "string") ? cmd.split(" ") : cmd; + const command = new Deno.Command(exec, { + args, + stdout: "piped", + stderr: "piped", + ...options, + }); + + const { code, stdout, stderr } = await command.output(); + + const stdoutText = new TextDecoder().decode(stdout); + const stderrText = new TextDecoder().decode(stderr); + + if (code !== 0) throw new Error(`Command failed: ${cmd}\n${stderrText}`); + + return stdoutText; +}; diff --git a/utils/secret.ts b/utils/secret.ts new file mode 100644 index 0000000..9847aa6 --- /dev/null +++ b/utils/secret.ts @@ -0,0 +1,46 @@ +import { getRequiredEnv, getStdout } from "./mod.ts"; + +export class BitwardenSession { + private readonly sessionInitializer: Promise; + + constructor(server = getRequiredEnv("BW_SERVER")) { + ["BW_CLIENTID", "BW_CLIENTSECRET"].forEach(getRequiredEnv); + + this.sessionInitializer = getStdout( + `bw config server ${server} --quiet`, + ).then(() => getStdout(`bw login --apikey --quiet`)) + .then(() => getStdout(`bw unlock --passwordenv BW_PASSWORD --raw`)) + .then((session) => session.trim()); + } + + public async getItem( + secretName: string, + ): Promise { + return await this.sessionInitializer.then((session) => + getStdout(`bw list items`, { + env: { + BW_SESSION: session, + }, + }) + ).then((items) => JSON.parse(items)).then((items) => + items.find(({ name }: { name: string }) => name === secretName) + ); + } + + async close(): Promise { + return await this.sessionInitializer.then((session) => + getStdout(`bw lock`, { env: { BW_SESSION: session } }) + ).then(() => {}); + } +} + +export type LoginItem = { + login: { + username: string; + password: string; + }; +}; + +export type SecureNote = { + notes: string; +}; diff --git a/utils/validate_identifier.ts b/utils/validate_identifier.ts new file mode 100644 index 0000000..c8a5213 --- /dev/null +++ b/utils/validate_identifier.ts @@ -0,0 +1,3 @@ +export const validateIdentifier = (token: string) => { + return (/^[a-zA-Z0-9_\-:. ]+$/).test(token) && !token.includes(".."); +}; diff --git a/worker/Dockerfile b/worker/Dockerfile new file mode 100644 index 0000000..e3a8f7b --- /dev/null +++ b/worker/Dockerfile @@ -0,0 +1,20 @@ +FROM oci.liz.coffee/img/liz-ci:release as worker + +RUN addgroup docker +RUN adduser -SDh /var/lib/laminar -g 'Laminar' -G users -G docker laminar + +# Secret retrieval +RUN apk add nodejs npm jq +RUN npm install -g @bitwarden/cli + +# Image building / publishing jobs +RUN apk add docker-cli + +# Ansible playbooks +RUN apk add ansible-core openssh + +USER laminar +WORKDIR /var/lib/laminar +EXPOSE 8080 + +CMD [ "/usr/sbin/laminard" ] diff --git a/worker/deno.json b/worker/deno.json new file mode 100644 index 0000000..5636d0a --- /dev/null +++ b/worker/deno.json @@ -0,0 +1,4 @@ +{ + "name": "@liz-ci/worker", + "exports": "./mod.ts" +} diff --git a/worker/jobs/checkout_ci.run b/worker/jobs/checkout_ci.run new file mode 100644 index 0000000..d47697d --- /dev/null +++ b/worker/jobs/checkout_ci.run @@ -0,0 +1,39 @@ +#!/bin/bash +# usage: laminarc run ci remote="ssh://src.liz.coffee:2222/cgit" rev="" \ +# refname="refs/..." + +set -e + +RUN=`date +%s` +WORKING_DIR=`$PWD/$RUN` + +mkdir -p "$WORKING_DIR" && cd "$WORKING_DIR" + +checkout="$rev" path="tmpsrc" fetch_code.sh + +if [[ ! -e "$WORKING_DIR/tmpsrc/.ci/ci.json" ]]; then + echo "No Continuous Integration configured for $remote." + exit 0 +fi + +PIPELINE_GENERATOR_PATH=$(jq -r '.pipeline' "$WORKING_DIR/tmpsrc/.ci/ci.json") +if [[ "$PIPELINE_GENERATOR_PATH" == *".."* ]]; then + echo "Error: Path contains '..'" + exit 1 +fi + +docker run --rm \ + --network none \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + -v "$WORKING_DIR/tmpsrc/$PIPELINE_GENERATOR:/pipeline" \ + -e refname="$refname" \ + -e rev="$rev" \ + -e remote="$remote" \ + oci.liz.coffee/img/liz-ci:release \ + /pipeline \ + > "$WORKING_DIR/pipeline.json" + +rm -rf tmpsrc + +pipeline="$WORKING_DIR/pipeline.json" run_pipeline diff --git a/worker/mod.ts b/worker/mod.ts new file mode 100644 index 0000000..e69de29 diff --git a/worker/scripts/ansible_playbook b/worker/scripts/ansible_playbook new file mode 100644 index 0000000..e69de29 diff --git a/worker/scripts/build_image b/worker/scripts/build_image new file mode 100644 index 0000000..ba1ec8f --- /dev/null +++ b/worker/scripts/build_image @@ -0,0 +1,67 @@ +#!/usr/bin/env -S deno run --allow-env --allow-net + +import type { BuildDockerImageJobProps } from "@liz-ci/model"; +import { + BitwardenSession, + getRequiredEnv, + getStdout, + type LoginItem, +} from "@liz-ci/utils"; + +const args: BuildDockerImageJobProps = { + registry: getRequiredEnv("registry"), + namespace: getRequiredEnv("namespace"), + repository: getRequiredEnv("repository"), + imageTag: getRequiredEnv("imageTag"), + + context: getRequiredEnv("context"), + dockerfile: getRequiredEnv("dockerfile"), + buildTarget: getRequiredEnv("buildTarget"), +}; + +const bitwardenSession = new BitwardenSession(); +const { username: registryUsername, password: registryPassword } = + (await bitwardenSession.getItem(args.registry))?.login ?? {}; +if (!(registryUsername && registryPassword)) { + throw new Error("where's the login info bruh"); +} + +await getStdout( + [ + "docker", + "login", + "--username", + registryUsername, + "--password", + registryPassword, + args.registry, + ], +); + +const tag = + `${args.registry}/${args.namespace}/${args.repository}:${args.imageTag}`; +await getStdout( + [ + "docker", + "build", + "--target", + args.buildTarget, + "-t", + tag, + "-f", + `${args.dockerfile}`, + `${args.context}`, + ], + { + clearEnv: true, + env: {}, + }, +); + +await getStdout( + [ + "docker", + "push", + tag, + ], +); diff --git a/worker/scripts/fetch_code b/worker/scripts/fetch_code new file mode 100644 index 0000000..d45f6db --- /dev/null +++ b/worker/scripts/fetch_code @@ -0,0 +1,6 @@ +#!/bin/bash + +git clone "$remote" "$path" +cd "$path" +git reset --hard "$checkout" +cd - diff --git a/worker/scripts/run_pipeline b/worker/scripts/run_pipeline new file mode 100644 index 0000000..ad58573 --- /dev/null +++ b/worker/scripts/run_pipeline @@ -0,0 +1,28 @@ +#!/usr/bin/env -S deno --allow-env --allow-net --allow-read + +import { type Job, PipelineImpl } from "@liz-ci/model"; +import { getRequiredEnv, getStdout, validateIdentifier } from "@liz-ci/utils"; + +const stages = await (Deno.readTextFile(getRequiredEnv("pipeline"))) + .then(PipelineImpl.from) + .then((pipeline) => pipeline.getStages()); + +const validateJob = (job: Job) => { + Object.entries(job.arguments).forEach((e) => { + if (!e.every(validateIdentifier)) { + throw new Error(`job of type ${job.type} has invalid entry ${e}`); + } + }); +}; + +for (const stage of stages) { + await Promise.all( + stage.parallelJobs.map((job) => { + validateJob(job); + + return getStdout(job.type, { + env: job.arguments, + }); + }), + ); +} -- cgit v1.2.3-70-g09d2