initial commit

4 files changed, 258 insertions, 274 deletions

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..4fc7fb9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,19 @@
+```
+sudo pacman -S base-devel
+
+cd /tmp
+git clone https://github.com/archlinux/archinstall
+
+python3 -m venv .venv
+source .venv/bin/activate
+pip3 install .
+
+cd -
+
+archinstall --config user_configration.json --plugin aur.py --verbose
+
+```
+
+1. fillout disk configuration
+2. fillout disk encryption
+3. fill out hostname
diff --git a/archinstall-aur.py b/archinstall-aur.py
deleted file mode 100644
index b032e85..0000000
--- a/archinstall-aur.py
+++ /dev/null
@@ -1,229 +0,0 @@
-import archinstall
-import re
-import glob
-import shutil
-import pathlib
-import logging
-import urllib.request
-
-__version__ = 0.2
-
-sudo_user = archinstall.arguments.get("aur-user", "aoffline_usr")
-try:
-    found_aur_user = archinstall.SysCommand(f"id {sudo_user}").exit_code == 0
-except:
-    found_aur_user = False
-
-AUR_USER_CREATED = False
-AUR_HELPER = None
-AUR_HELPERS = {
-    "yay-bin": "LANG=C yay --noprovides --answerdiff None --answerclean None --mflags '--noconfirm'"
-}
-
-
-def untar_file(file):
-    archinstall.log(
-        f"(runas {sudo_user}) /usr/bin/tar --directory /home/{sudo_user}/ -xvzf {file}",
-        level=logging.DEBUG,
-        fg="gray",
-    )
-    archinstall.storage["installation_session"].arch_chroot(
-        f"/usr/bin/tar --directory /home/{sudo_user}/ -xvzf {file}", run_as=sudo_user
-    )
-
-
-def download_file(url, destination, filename=""):
-    if not (dst := pathlib.Path(destination)).exists():
-        dst.mkdir(parents=True)
-
-    if dst.is_file():
-        return False
-
-    tmp_filename, headers = urllib.request.urlretrieve(url)
-    shutil.move(tmp_filename, f"{destination}/{filename}")
-
-    return True
-
-
-class Plugin:
-    def on_pacstrap(self, packages: list) -> list:
-        global AUR_USER_CREATED
-        global AUR_HELPER
-        global AUR_HELPERS
-
-        if type(packages) == str:
-            packages = packages.split(" ")
-
-        archinstall.log(
-            f"Identifying AUR packages in package list: {packages}",
-            level=logging.INFO,
-            fg="gray",
-        )
-        aur_packages = []
-        std_packages = []
-
-        # We'd like to use upstream or a local JSON database to lookup packages.
-        # But for now this is the lowest latency option that doesn't hog resources upstream.
-        for package in packages:
-            try:
-                if archinstall.SysCommand(f"pacman -Ss {package}").exit_code == 0:
-                    std_packages.append(package)
-                else:
-                    aur_packages.append(package)
-
-            except archinstall.lib.exceptions.SysCallError:
-                aur_packages.append(package)
-
-        mount_location = archinstall.storage["installation_session"].target
-
-        aur_packages = sorted(
-            aur_packages, key=lambda package: 0 if package in AUR_HELPERS else 1
-        )
-        for package in aur_packages:
-            if AUR_USER_CREATED is False:
-                archinstall.log(
-                    f"Setting up temporary AUR build user {sudo_user} and installing build tools for {aur_packages}",
-                    level=logging.INFO,
-                    fg="gray",
-                )
-                # We have to install fakeroot to the live medium as it's missing
-                # (wasn't ever really intended to build stuff..)
-                archinstall.storage["installation_session"].add_additional_packages(
-                    ["fakeroot", "base-devel"]
-                )
-
-                with open(f"{mount_location}/etc/sudoers.d/{sudo_user}", "w") as fh:
-                    # TODO: This could be tweaked to only contain the binaries needed, such as `makepkg` and `pacman -U`.
-                    # But it's done in the live environment, not the final installation..
-                    # So risks are low unless the user pre-enabled sshd with a login for said user.
-                    fh.write(f"{sudo_user} ALL=(ALL:ALL) NOPASSWD: ALL\n")
-
-                archinstall.log(f"Creating temporary build user {sudo_user}")
-                archinstall.storage["installation_session"].user_create(
-                    sudo_user, password="somethingrandom"
-                )
-                # archinstall.SysCommand(f"/usr/bin/useradd -m -N -s /bin/bash {sudo_user}")
-
-                AUR_USER_CREATED = True
-
-            archinstall.log(
-                f"Building AUR package {package}", level=logging.INFO, fg="yellow"
-            )
-
-            if AUR_HELPER is not None:
-                build_command = f"{AUR_HELPER} -S {package}"
-                try:
-                    archinstall.storage["installation_session"].arch_chroot(
-                        f'/bin/sh -c "{build_command}"', run_as=sudo_user
-                    )
-                    continue
-                except archinstall.lib.exceptions.SysCallError:
-                    pass
-
-            if not download_file(
-                f"https://aur.archlinux.org/cgit/aur.git/snapshot/{package}.tar.gz",
-                destination=f"{mount_location}/home/{sudo_user}/",
-                filename=f"{package}.tar.gz",
-            ):
-                archinstall.log(
-                    f"Could not retrieve {package} from: https://aur.archlinux.org/cgit/aur.git/snapshot/{package}.tar.gz",
-                    fg="red",
-                    level=logging.ERROR,
-                )
-                exit(1)
-
-            archinstall.storage["installation_session"].chown(
-                sudo_user, f"/home/{sudo_user}/{package}.tar.gz"
-            )
-            untar_file(f"/home/{sudo_user}/{package}.tar.gz")
-            with open(
-                f"{mount_location}/home/{sudo_user}/{package}/PKGBUILD", "r"
-            ) as fh:
-                PKGBUILD = fh.read()
-
-            # This regexp needs to accomodate multiple keys, as well as the logic below
-            gpgkeys = re.findall("validpgpkeys=\(.*\)", PKGBUILD)
-            if gpgkeys:
-                for key in gpgkeys:
-                    key = key[13:].strip("(')\"")
-                    archinstall.log(f"Adding GPG-key {key} to session for {sudo_user}")
-                    archinstall.storage["installation_session"].arch_chroot(
-                        f"/usr/bin/gpg --recv-keys {key}", run_as=sudo_user
-                    )
-
-            build_command = f'cd /home/{sudo_user}/{package}; makepkg --clean --force --cleanbuild --noconfirm --needed -s'
-            if (
-                build_handle := archinstall.storage["installation_session"].arch_chroot(
-                    f'/bin/bash -c "{build_command}"', run_as=sudo_user
-                )
-            ).exit_code != 0:
-                archinstall.log(build_handle, level=logging.ERROR)
-                archinstall.log(
-                    f"Could not build {package}, see traceback above. Continuing to avoid re-build needs for the rest of the run and re-runs.",
-                    fg="red",
-                    level=logging.ERROR,
-                )
-            else:
-                print(
-                    f"Looking for: {mount_location}/home/{sudo_user}/{package}/*.tar.zst"
-                )
-                if built_package := glob.glob(
-                    f"{mount_location}/home/{sudo_user}/{package}/*.tar.zst"
-                ):
-                    built_package = pathlib.Path(built_package[0]).name
-                    print(f"Found package: {built_package}")
-
-                    archinstall.storage["installation_session"].arch_chroot(
-                        f"/usr/bin/pacman --noconfirm -U /home/{sudo_user}/{package}/{built_package}"
-                    )
-                    shutil.rmtree(f"{mount_location}/home/{sudo_user}/{package}")
-                    pathlib.Path(
-                        f"{mount_location}/home/{sudo_user}/{package}.tar.gz"
-                    ).unlink()
-                    AUR_HELPER = AUR_HELPERS.get(package, AUR_HELPER)
-                else:
-                    archinstall.log(
-                        f"Could not locate {package}.tar.zst after build.",
-                        fg="red",
-                        level=logging.ERROR,
-                    )
-                    exit(1)
-
-        if AUR_USER_CREATED:
-            archinstall.log(f"Removing temporary build user {sudo_user}")
-
-            pathlib.Path(f"{mount_location}/etc/sudoers.d/{sudo_user}").unlink()
-
-            # TODO: These are only needed if we run Installation.Boot():
-            # Stop dirmngr and gpg-agent before removing home directory and running userdel
-            # archinstall.storage['installation_session'].arch_chroot(f"/usr/bin/systemctl --machine={sudo_user}@.host --user stop dirmngr.socket", run_as=sudo_user)
-            archinstall.storage["installation_session"].arch_chroot(
-                f"/usr/bin/gpgconf --kill gpg-agent", run_as=sudo_user
-            )
-            try:
-                archinstall.storage["installation_session"].arch_chroot(
-                    f"/usr/bin/killall -u {sudo_user}", run_as=sudo_user
-                )
-            except archinstall.lib.exceptions.SysCallError:
-                # We'll terminate our own running process and that's fine
-                pass
-            archinstall.storage["installation_session"].arch_chroot(
-                f"/usr/bin/userdel {sudo_user}"
-            )
-
-            shutil.rmtree(f"{mount_location}/home/{sudo_user}")
-
-            AUR_USER_CREATED = False
-
-        # Returns a curated list of packages that exludes any AUR packages.
-        # This allows installataion.pacstrap() to contain AUR packages,
-        # but won't handle them or try to install them since we remove those here.
-        return std_packages
-
-
-def dummy_example(*args, **kwargs):
-    pass
-
-
-# Example function injection
-archinstall.plugin_function = dummy_example
diff --git a/aur.py b/aur.py
new file mode 100644
index 0000000..2bea1d1
--- /dev/null
+++ b/aur.py
@@ -0,0 +1,235 @@
+import archinstall
+import re
+import shutil
+import logging
+import asyncio
+import os
+from pathlib import Path
+
+__version__ = 0.4
+
+SUDOERS_FMT = "etc/sudoers.d/{user}"
+
+
+class PipelineStep:
+    def cleanup(self):
+        pass
+
+
+class PackageClassifier(PipelineStep):
+    def split(self, packages):
+        aur, std = [], []
+        for pkg in packages:
+            try:
+                if (
+                    archinstall.lib.general.SysCommand(f"pacman -Ss {pkg}").exit_code
+                    == 0
+                ):
+                    std.append(pkg)
+                else:
+                    aur.append(pkg)
+            except archinstall.lib.exceptions.SysCallError:
+                aur.append(pkg)
+        return std, aur
+
+
+class UserManager(PipelineStep):
+    def __init__(self, user: str, mount_location: Path, installation):
+        self.user = user
+        self.mount_location = mount_location
+        self.installation = installation
+        self.user_created = False
+
+    def _run_chroot(self, command):
+        return self.installation.arch_chroot(command, run_as=self.user)
+
+    def create(self):
+        if self.user_created:
+            return
+
+        archinstall.lib.output.log(
+            f"Creating temporary user {self.user}", level=logging.INFO
+        )
+        self.installation.add_additional_packages(["fakeroot", "base-devel"])
+
+        sudoers_path = self.mount_location / SUDOERS_FMT.format(user=self.user)
+        sudoers_path.write_text(f"{self.user} ALL=(ALL:ALL) NOPASSWD: ALL\n")
+
+        password = archinstall.lib.models.users.Password(plaintext="somethingrandom")
+        user = archinstall.lib.models.users.User(
+            username=self.user, password=password, groups=[], sudo=False
+        )
+        self.installation.create_users([user])
+
+        home_path = self.mount_location / "home" / self.user
+        home_path.mkdir(parents=True, exist_ok=True)
+        self.installation.arch_chroot(
+            f"/usr/bin/chown {self.user}:{self.user} /home/{self.user}"
+        )
+
+        self.user_created = True
+
+    def cleanup(self):
+        if not self.user_created:
+            return
+
+        archinstall.lib.output.log(f"Cleaning up user {self.user}", level=logging.INFO)
+        try:
+            self._run_chroot("/usr/bin/gpgconf --kill gpg-agent")
+            self._run_chroot(f"/usr/bin/killall -u {self.user}")
+        except archinstall.lib.exceptions.SysCallError:
+            pass
+
+        self.installation.arch_chroot(f"/usr/bin/userdel {self.user}")
+        shutil.rmtree(self.mount_location / f"home/{self.user}", ignore_errors=True)
+        (self.mount_location / SUDOERS_FMT.format(user=self.user)).unlink(
+            missing_ok=True
+        )
+        self.user_created = False
+
+
+class PackageDownloader(PipelineStep):
+    def __init__(self, user: str, mount_location: Path, installation):
+        self.user = user
+        self.mount_location = mount_location
+        self.downloaded = []
+        self.installation = installation
+
+    def _run(self, command):
+        return self.installation.arch_chroot(command, run_as=self.user)
+
+    async def _download_package(self, package: str):
+        package_file = f"{package}.tar.gz"
+        url = f"https://aur.archlinux.org/cgit/aur.git/snapshot/{package_file}"
+        dest = f"/home/{self.user}/{package_file}"
+
+        try:
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None, lambda: self._run(f"/usr/bin/curl {url} > {dest}")
+            )
+            self.downloaded.append(package)
+        except Exception as e:
+            archinstall.lib.output.log(
+                f"Failed to download {package}: {e}", level=logging.ERROR, fg="red"
+            )
+
+    async def download(self, packages):
+        archinstall.log(f"Downloading AUR packages: {packages}", level=logging.INFO)
+        semaphore = asyncio.Semaphore(3)
+
+        async def sem_task(pkg):
+            async with semaphore:
+                await self._download_package(pkg)
+
+        await asyncio.gather(*(sem_task(pkg) for pkg in packages))
+
+    def cleanup(self):
+        for pkg in self.downloaded:
+            tar_path = self.mount_location / "home" / self.user / f"{pkg}.tar.gz"
+            tar_path.unlink(missing_ok=True)
+
+
+class PackageInstaller(PipelineStep):
+    def __init__(self, user: str, mount_location: Path, installation):
+        self.user = user
+        self.mount_location = mount_location
+        self.installation = installation
+        self.installed = []
+        self.installed_dirs = []
+
+    def _run(self, command):
+        return self.installation.arch_chroot(command, run_as=self.user)
+
+    def _untar(self, package: str):
+        self._run(
+            f"/usr/bin/tar --directory /home/{self.user}/ -xvzf /home/{self.user}/{package}.tar.gz"
+        )
+
+    def install(self, package: str):
+        self._untar(package)
+        build_dir = self.mount_location / "home" / self.user / package
+
+        try:
+            with (build_dir / "PKGBUILD").open() as fh:
+                content = fh.read()
+            gpgkeys = re.findall(r"validpgpkeys=\((.*)\)", content)
+            for key in gpgkeys:
+                key = key.strip("'\" ")
+                self._run(f"/usr/bin/gpg --recv-keys {key}")
+        except FileNotFoundError:
+            archinstall.lib.output.log(
+                f"Missing PKGBUILD for {package}", level=logging.ERROR, fg="red"
+            )
+            return
+
+        cmd = (
+            f"cd /home/{self.user}/{package} && makepkg --force --noconfirm --needed -si"
+        )
+        result = self._run(f'/bin/bash -c "{cmd}"')
+        if result.exit_code != 0:
+            archinstall.lib.output.log(
+                f"Build failed for {package}", level=logging.ERROR, fg="red"
+            )
+            return
+
+#        packages = list(build_dir.glob("*.tar.zst"))
+#        if not packages:
+#            archinstall.log(
+#                f"No built packages found for {package}", level=logging.ERROR, fg="red"
+#            )
+#            return
+#
+#        self._run(
+#            f"/usr/bin/pacman --noconfirm -U /home/{self.user}/{package}/{packages[0].name}"
+#        )
+#        self.installed.append(package)
+        self.installed_dirs.append(build_dir)
+
+    def cleanup(self):
+        for build_dir in self.installed_dirs:
+            shutil.rmtree(build_dir, ignore_errors=True)
+        self.installed_dirs.clear()
+
+
+class Plugin:
+    def __init__(self):
+        self.user = os.getenv("AUR_USER", "aoffline_usr")
+        self.lazy_initd = False
+
+    def _lazy_init(self):
+        self.installation = archinstall.lib.storage.storage["session"]
+        self.classifier = PackageClassifier()
+        self.mount_location = Path(self.installation.target)
+        self.usermgr = UserManager(self.user, self.mount_location, self.installation)
+        self.downloader = PackageDownloader(
+            self.user, self.mount_location, self.installation
+        )
+        self.installer = PackageInstaller(
+            self.user, self.mount_location, self.installation
+        )
+        self.lazy_initd = True
+
+    """
+    TODO: 
+    Use the nobody account to run makepkg.
+    Clone the AUR repo, chown it to nobody, then use sudo -u nobody makepkg to build it.
+    """
+    def on_pacstrap(self, packages: list[str]) -> list[str]:
+        if not self.lazy_initd:
+            self._lazy_init()
+
+        std, aur = self.classifier.split(packages)
+        if not aur:
+            return std
+
+        self.usermgr.create()
+        asyncio.run(self.downloader.download(aur))
+
+        for pkg in self.downloader.downloaded:
+            self.installer.install(pkg)
+
+        for step in reversed([self.installer, self.downloader, self.usermgr]):
+            step.cleanup()
+
+        return std
diff --git a/user_configuration.json b/user_configuration.json
index f6af4f4..c177030 100644
--- a/user_configuration.json
+++ b/user_configuration.json
@@ -5,9 +5,6 @@
   },
   "bootloader": "Grub",
   "custom_commands": [],
-  "disk_config": null,
-  "disk_encryption": null,
-  "hostname": "okabe",
   "kernels": ["linux"],
   "locale_config": {
     "kb_layout": "us",
@@ -44,46 +41,7 @@
         "https://mirrors.mit.edu/archlinux/$repo/os/$arch",
         "https://arch.hu.fo/archlinux/$repo/os/$arch",
         "https://zxcvfdsa.com/arch/$repo/os/$arch",
-        "https://mirror.theash.xyz/arch/$repo/os/$arch",
-        "https://mirror.clarkson.edu/archlinux/$repo/os/$arch",
-        "https://mirrors.bloomu.edu/archlinux/$repo/os/$arch",
-        "https://codingflyboy.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://coresite.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://forksystems.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://irltoolkit.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://mirror.fcix.net/archlinux/$repo/os/$arch",
-        "https://mnvoip.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://nnenix.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://nocix.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://ohioix.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://opencolo.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://ridgewireless.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://southfront.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://volico.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://ziply.mm.fcix.net/archlinux/$repo/os/$arch",
-        "https://america.mirror.pkgbuild.com/$repo/os/$arch",
-        "https://losangeles.mirror.pkgbuild.com/$repo/os/$arch",
-        "https://mirrors.vectair.net/archlinux/$repo/os/$arch",
-        "https://arch.mirror.k0.ae/$repo/os/$arch",
-        "https://mirror.zackmyers.io/archlinux/$repo/os/$arch",
-        "https://m.lqy.me/arch/$repo/os/$arch",
-        "https://mirror.adectra.com/archlinux/$repo/os/$arch",
-        "https://arch.goober.cloud/$repo/os/$arch",
-        "https://mirrors.bjg.at/arch/$repo/os/$arch",
-        "https://mirror.pilotfiber.com/archlinux/$repo/os/$arch",
-        "https://mirrors.iu13.net/archlinux/$repo/os/$arch",
-        "https://mirror.colonelhosting.com/archlinux/$repo/os/$arch",
-        "https://us.arch.niranjan.co/$repo/os/$arch",
-        "https://mirror.hasphetica.win/archlinux/$repo/os/$arch",
-        "https://arch-mirror.marcusspencer.xyz:4443/archlinux/$repo/os/$arch",
-        "https://us-mnz.soulharsh007.dev/archlinux/$repo/os/$arch",
-        "https://mirror.akane.network/archmirror/$repo/os/$arch",
-        "https://arch.miningtcup.me/$repo/os/$arch",
-        "https://mirrors.smeal.xyz/arch-linux/$repo/os/$arch",
-        "https://arch-mirror.brightlight.today/$repo/os/$arch",
-        "https://yonderly.org/mirrors/archlinux/$repo/os/$arch",
-        "https://mirrors.lahansons.com/archlinux/$repo/os/$arch",
-        "https://mirror.givebytes.net/archlinux/$repo/os/$arch"
+        "https://mirror.theash.xyz/arch/$repo/os/$arch"
       ]
     },
     "optional_repositories": []
@@ -112,9 +70,10 @@
     "docker-compose",
     "emacs-wayland",
     "flatpak",
+    "fuzzel",
     "git",
     "gnome-keyring",
-    "graphite-grub-theme-default-1080p",
+    "graphite-grub-theme",
     "libmpeg2",
     "librewolf-bin",
     "mpv",
@@ -174,7 +133,7 @@
     "mise",
     "mako"
   ],
-  "parallel downloads": 0,
+  "parallel downloads": 3,
   "profile_config": null,
   "services": ["bluetooth", "ly", "NetworkManager", "reflector", "polkit"],
   "swap": true,