diff options
| -rw-r--r-- | README.md | 19 | ||||
| -rw-r--r-- | archinstall-aur.py | 229 | ||||
| -rw-r--r-- | aur.py | 235 | ||||
| -rw-r--r-- | user_configuration.json | 49 |
4 files changed, 258 insertions, 274 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..4fc7fb9 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +``` +sudo pacman -S base-devel + +cd /tmp +git clone https://github.com/archlinux/archinstall + +python3 -m venv .venv +source .venv/bin/activate +pip3 install . + +cd - + +archinstall --config user_configration.json --plugin aur.py --verbose + +``` + +1. fillout disk configuration +2. fillout disk encryption +3. fill out hostname diff --git a/archinstall-aur.py b/archinstall-aur.py deleted file mode 100644 index b032e85..0000000 --- a/archinstall-aur.py +++ /dev/null @@ -1,229 +0,0 @@ -import archinstall -import re -import glob -import shutil -import pathlib -import logging -import urllib.request - -__version__ = 0.2 - -sudo_user = archinstall.arguments.get("aur-user", "aoffline_usr") -try: - found_aur_user = archinstall.SysCommand(f"id {sudo_user}").exit_code == 0 -except: - found_aur_user = False - -AUR_USER_CREATED = False -AUR_HELPER = None -AUR_HELPERS = { - "yay-bin": "LANG=C yay --noprovides --answerdiff None --answerclean None --mflags '--noconfirm'" -} - - -def untar_file(file): - archinstall.log( - f"(runas {sudo_user}) /usr/bin/tar --directory /home/{sudo_user}/ -xvzf {file}", - level=logging.DEBUG, - fg="gray", - ) - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/tar --directory /home/{sudo_user}/ -xvzf {file}", run_as=sudo_user - ) - - -def download_file(url, destination, filename=""): - if not (dst := pathlib.Path(destination)).exists(): - dst.mkdir(parents=True) - - if dst.is_file(): - return False - - tmp_filename, headers = urllib.request.urlretrieve(url) - shutil.move(tmp_filename, f"{destination}/{filename}") - - return True - - -class Plugin: - def on_pacstrap(self, packages: list) -> list: - global AUR_USER_CREATED - global AUR_HELPER - global AUR_HELPERS - - if type(packages) == str: - packages = packages.split(" ") - - archinstall.log( - f"Identifying AUR packages in package list: {packages}", - level=logging.INFO, - fg="gray", - ) - aur_packages = [] - std_packages = [] - - # We'd like to use upstream or a local JSON database to lookup packages. - # But for now this is the lowest latency option that doesn't hog resources upstream. - for package in packages: - try: - if archinstall.SysCommand(f"pacman -Ss {package}").exit_code == 0: - std_packages.append(package) - else: - aur_packages.append(package) - - except archinstall.lib.exceptions.SysCallError: - aur_packages.append(package) - - mount_location = archinstall.storage["installation_session"].target - - aur_packages = sorted( - aur_packages, key=lambda package: 0 if package in AUR_HELPERS else 1 - ) - for package in aur_packages: - if AUR_USER_CREATED is False: - archinstall.log( - f"Setting up temporary AUR build user {sudo_user} and installing build tools for {aur_packages}", - level=logging.INFO, - fg="gray", - ) - # We have to install fakeroot to the live medium as it's missing - # (wasn't ever really intended to build stuff..) - archinstall.storage["installation_session"].add_additional_packages( - ["fakeroot", "base-devel"] - ) - - with open(f"{mount_location}/etc/sudoers.d/{sudo_user}", "w") as fh: - # TODO: This could be tweaked to only contain the binaries needed, such as `makepkg` and `pacman -U`. - # But it's done in the live environment, not the final installation.. - # So risks are low unless the user pre-enabled sshd with a login for said user. - fh.write(f"{sudo_user} ALL=(ALL:ALL) NOPASSWD: ALL\n") - - archinstall.log(f"Creating temporary build user {sudo_user}") - archinstall.storage["installation_session"].user_create( - sudo_user, password="somethingrandom" - ) - # archinstall.SysCommand(f"/usr/bin/useradd -m -N -s /bin/bash {sudo_user}") - - AUR_USER_CREATED = True - - archinstall.log( - f"Building AUR package {package}", level=logging.INFO, fg="yellow" - ) - - if AUR_HELPER is not None: - build_command = f"{AUR_HELPER} -S {package}" - try: - archinstall.storage["installation_session"].arch_chroot( - f'/bin/sh -c "{build_command}"', run_as=sudo_user - ) - continue - except archinstall.lib.exceptions.SysCallError: - pass - - if not download_file( - f"https://aur.archlinux.org/cgit/aur.git/snapshot/{package}.tar.gz", - destination=f"{mount_location}/home/{sudo_user}/", - filename=f"{package}.tar.gz", - ): - archinstall.log( - f"Could not retrieve {package} from: https://aur.archlinux.org/cgit/aur.git/snapshot/{package}.tar.gz", - fg="red", - level=logging.ERROR, - ) - exit(1) - - archinstall.storage["installation_session"].chown( - sudo_user, f"/home/{sudo_user}/{package}.tar.gz" - ) - untar_file(f"/home/{sudo_user}/{package}.tar.gz") - with open( - f"{mount_location}/home/{sudo_user}/{package}/PKGBUILD", "r" - ) as fh: - PKGBUILD = fh.read() - - # This regexp needs to accomodate multiple keys, as well as the logic below - gpgkeys = re.findall("validpgpkeys=\(.*\)", PKGBUILD) - if gpgkeys: - for key in gpgkeys: - key = key[13:].strip("(')\"") - archinstall.log(f"Adding GPG-key {key} to session for {sudo_user}") - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/gpg --recv-keys {key}", run_as=sudo_user - ) - - build_command = f'cd /home/{sudo_user}/{package}; makepkg --clean --force --cleanbuild --noconfirm --needed -s' - if ( - build_handle := archinstall.storage["installation_session"].arch_chroot( - f'/bin/bash -c "{build_command}"', run_as=sudo_user - ) - ).exit_code != 0: - archinstall.log(build_handle, level=logging.ERROR) - archinstall.log( - f"Could not build {package}, see traceback above. Continuing to avoid re-build needs for the rest of the run and re-runs.", - fg="red", - level=logging.ERROR, - ) - else: - print( - f"Looking for: {mount_location}/home/{sudo_user}/{package}/*.tar.zst" - ) - if built_package := glob.glob( - f"{mount_location}/home/{sudo_user}/{package}/*.tar.zst" - ): - built_package = pathlib.Path(built_package[0]).name - print(f"Found package: {built_package}") - - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/pacman --noconfirm -U /home/{sudo_user}/{package}/{built_package}" - ) - shutil.rmtree(f"{mount_location}/home/{sudo_user}/{package}") - pathlib.Path( - f"{mount_location}/home/{sudo_user}/{package}.tar.gz" - ).unlink() - AUR_HELPER = AUR_HELPERS.get(package, AUR_HELPER) - else: - archinstall.log( - f"Could not locate {package}.tar.zst after build.", - fg="red", - level=logging.ERROR, - ) - exit(1) - - if AUR_USER_CREATED: - archinstall.log(f"Removing temporary build user {sudo_user}") - - pathlib.Path(f"{mount_location}/etc/sudoers.d/{sudo_user}").unlink() - - # TODO: These are only needed if we run Installation.Boot(): - # Stop dirmngr and gpg-agent before removing home directory and running userdel - # archinstall.storage['installation_session'].arch_chroot(f"/usr/bin/systemctl --machine={sudo_user}@.host --user stop dirmngr.socket", run_as=sudo_user) - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/gpgconf --kill gpg-agent", run_as=sudo_user - ) - try: - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/killall -u {sudo_user}", run_as=sudo_user - ) - except archinstall.lib.exceptions.SysCallError: - # We'll terminate our own running process and that's fine - pass - archinstall.storage["installation_session"].arch_chroot( - f"/usr/bin/userdel {sudo_user}" - ) - - shutil.rmtree(f"{mount_location}/home/{sudo_user}") - - AUR_USER_CREATED = False - - # Returns a curated list of packages that exludes any AUR packages. - # This allows installataion.pacstrap() to contain AUR packages, - # but won't handle them or try to install them since we remove those here. - return std_packages - - -def dummy_example(*args, **kwargs): - pass - - -# Example function injection -archinstall.plugin_function = dummy_example @@ -0,0 +1,235 @@ +import archinstall +import re +import shutil +import logging +import asyncio +import os +from pathlib import Path + +__version__ = 0.4 + +SUDOERS_FMT = "etc/sudoers.d/{user}" + + +class PipelineStep: + def cleanup(self): + pass + + +class PackageClassifier(PipelineStep): + def split(self, packages): + aur, std = [], [] + for pkg in packages: + try: + if ( + archinstall.lib.general.SysCommand(f"pacman -Ss {pkg}").exit_code + == 0 + ): + std.append(pkg) + else: + aur.append(pkg) + except archinstall.lib.exceptions.SysCallError: + aur.append(pkg) + return std, aur + + +class UserManager(PipelineStep): + def __init__(self, user: str, mount_location: Path, installation): + self.user = user + self.mount_location = mount_location + self.installation = installation + self.user_created = False + + def _run_chroot(self, command): + return self.installation.arch_chroot(command, run_as=self.user) + + def create(self): + if self.user_created: + return + + archinstall.lib.output.log( + f"Creating temporary user {self.user}", level=logging.INFO + ) + self.installation.add_additional_packages(["fakeroot", "base-devel"]) + + sudoers_path = self.mount_location / SUDOERS_FMT.format(user=self.user) + sudoers_path.write_text(f"{self.user} ALL=(ALL:ALL) NOPASSWD: ALL\n") + + password = archinstall.lib.models.users.Password(plaintext="somethingrandom") + user = archinstall.lib.models.users.User( + username=self.user, password=password, groups=[], sudo=False + ) + self.installation.create_users([user]) + + home_path = self.mount_location / "home" / self.user + home_path.mkdir(parents=True, exist_ok=True) + self.installation.arch_chroot( + f"/usr/bin/chown {self.user}:{self.user} /home/{self.user}" + ) + + self.user_created = True + + def cleanup(self): + if not self.user_created: + return + + archinstall.lib.output.log(f"Cleaning up user {self.user}", level=logging.INFO) + try: + self._run_chroot("/usr/bin/gpgconf --kill gpg-agent") + self._run_chroot(f"/usr/bin/killall -u {self.user}") + except archinstall.lib.exceptions.SysCallError: + pass + + self.installation.arch_chroot(f"/usr/bin/userdel {self.user}") + shutil.rmtree(self.mount_location / f"home/{self.user}", ignore_errors=True) + (self.mount_location / SUDOERS_FMT.format(user=self.user)).unlink( + missing_ok=True + ) + self.user_created = False + + +class PackageDownloader(PipelineStep): + def __init__(self, user: str, mount_location: Path, installation): + self.user = user + self.mount_location = mount_location + self.downloaded = [] + self.installation = installation + + def _run(self, command): + return self.installation.arch_chroot(command, run_as=self.user) + + async def _download_package(self, package: str): + package_file = f"{package}.tar.gz" + url = f"https://aur.archlinux.org/cgit/aur.git/snapshot/{package_file}" + dest = f"/home/{self.user}/{package_file}" + + try: + loop = asyncio.get_event_loop() + await loop.run_in_executor( + None, lambda: self._run(f"/usr/bin/curl {url} > {dest}") + ) + self.downloaded.append(package) + except Exception as e: + archinstall.lib.output.log( + f"Failed to download {package}: {e}", level=logging.ERROR, fg="red" + ) + + async def download(self, packages): + archinstall.log(f"Downloading AUR packages: {packages}", level=logging.INFO) + semaphore = asyncio.Semaphore(3) + + async def sem_task(pkg): + async with semaphore: + await self._download_package(pkg) + + await asyncio.gather(*(sem_task(pkg) for pkg in packages)) + + def cleanup(self): + for pkg in self.downloaded: + tar_path = self.mount_location / "home" / self.user / f"{pkg}.tar.gz" + tar_path.unlink(missing_ok=True) + + +class PackageInstaller(PipelineStep): + def __init__(self, user: str, mount_location: Path, installation): + self.user = user + self.mount_location = mount_location + self.installation = installation + self.installed = [] + self.installed_dirs = [] + + def _run(self, command): + return self.installation.arch_chroot(command, run_as=self.user) + + def _untar(self, package: str): + self._run( + f"/usr/bin/tar --directory /home/{self.user}/ -xvzf /home/{self.user}/{package}.tar.gz" + ) + + def install(self, package: str): + self._untar(package) + build_dir = self.mount_location / "home" / self.user / package + + try: + with (build_dir / "PKGBUILD").open() as fh: + content = fh.read() + gpgkeys = re.findall(r"validpgpkeys=\((.*)\)", content) + for key in gpgkeys: + key = key.strip("'\" ") + self._run(f"/usr/bin/gpg --recv-keys {key}") + except FileNotFoundError: + archinstall.lib.output.log( + f"Missing PKGBUILD for {package}", level=logging.ERROR, fg="red" + ) + return + + cmd = ( + f"cd /home/{self.user}/{package} && makepkg --force --noconfirm --needed -si" + ) + result = self._run(f'/bin/bash -c "{cmd}"') + if result.exit_code != 0: + archinstall.lib.output.log( + f"Build failed for {package}", level=logging.ERROR, fg="red" + ) + return + +# packages = list(build_dir.glob("*.tar.zst")) +# if not packages: +# archinstall.log( +# f"No built packages found for {package}", level=logging.ERROR, fg="red" +# ) +# return +# +# self._run( +# f"/usr/bin/pacman --noconfirm -U /home/{self.user}/{package}/{packages[0].name}" +# ) +# self.installed.append(package) + self.installed_dirs.append(build_dir) + + def cleanup(self): + for build_dir in self.installed_dirs: + shutil.rmtree(build_dir, ignore_errors=True) + self.installed_dirs.clear() + + +class Plugin: + def __init__(self): + self.user = os.getenv("AUR_USER", "aoffline_usr") + self.lazy_initd = False + + def _lazy_init(self): + self.installation = archinstall.lib.storage.storage["session"] + self.classifier = PackageClassifier() + self.mount_location = Path(self.installation.target) + self.usermgr = UserManager(self.user, self.mount_location, self.installation) + self.downloader = PackageDownloader( + self.user, self.mount_location, self.installation + ) + self.installer = PackageInstaller( + self.user, self.mount_location, self.installation + ) + self.lazy_initd = True + + """ + TODO: + Use the nobody account to run makepkg. + Clone the AUR repo, chown it to nobody, then use sudo -u nobody makepkg to build it. + """ + def on_pacstrap(self, packages: list[str]) -> list[str]: + if not self.lazy_initd: + self._lazy_init() + + std, aur = self.classifier.split(packages) + if not aur: + return std + + self.usermgr.create() + asyncio.run(self.downloader.download(aur)) + + for pkg in self.downloader.downloaded: + self.installer.install(pkg) + + for step in reversed([self.installer, self.downloader, self.usermgr]): + step.cleanup() + + return std diff --git a/user_configuration.json b/user_configuration.json index f6af4f4..c177030 100644 --- a/user_configuration.json +++ b/user_configuration.json @@ -5,9 +5,6 @@ }, "bootloader": "Grub", "custom_commands": [], - "disk_config": null, - "disk_encryption": null, - "hostname": "okabe", "kernels": ["linux"], "locale_config": { "kb_layout": "us", @@ -44,46 +41,7 @@ "https://mirrors.mit.edu/archlinux/$repo/os/$arch", "https://arch.hu.fo/archlinux/$repo/os/$arch", "https://zxcvfdsa.com/arch/$repo/os/$arch", - "https://mirror.theash.xyz/arch/$repo/os/$arch", - "https://mirror.clarkson.edu/archlinux/$repo/os/$arch", - "https://mirrors.bloomu.edu/archlinux/$repo/os/$arch", - "https://codingflyboy.mm.fcix.net/archlinux/$repo/os/$arch", - "https://coresite.mm.fcix.net/archlinux/$repo/os/$arch", - "https://forksystems.mm.fcix.net/archlinux/$repo/os/$arch", - "https://irltoolkit.mm.fcix.net/archlinux/$repo/os/$arch", - "https://mirror.fcix.net/archlinux/$repo/os/$arch", - "https://mnvoip.mm.fcix.net/archlinux/$repo/os/$arch", - "https://nnenix.mm.fcix.net/archlinux/$repo/os/$arch", - "https://nocix.mm.fcix.net/archlinux/$repo/os/$arch", - "https://ohioix.mm.fcix.net/archlinux/$repo/os/$arch", - "https://opencolo.mm.fcix.net/archlinux/$repo/os/$arch", - "https://ridgewireless.mm.fcix.net/archlinux/$repo/os/$arch", - "https://southfront.mm.fcix.net/archlinux/$repo/os/$arch", - "https://volico.mm.fcix.net/archlinux/$repo/os/$arch", - "https://ziply.mm.fcix.net/archlinux/$repo/os/$arch", - "https://america.mirror.pkgbuild.com/$repo/os/$arch", - "https://losangeles.mirror.pkgbuild.com/$repo/os/$arch", - "https://mirrors.vectair.net/archlinux/$repo/os/$arch", - "https://arch.mirror.k0.ae/$repo/os/$arch", - "https://mirror.zackmyers.io/archlinux/$repo/os/$arch", - "https://m.lqy.me/arch/$repo/os/$arch", - "https://mirror.adectra.com/archlinux/$repo/os/$arch", - "https://arch.goober.cloud/$repo/os/$arch", - "https://mirrors.bjg.at/arch/$repo/os/$arch", - "https://mirror.pilotfiber.com/archlinux/$repo/os/$arch", - "https://mirrors.iu13.net/archlinux/$repo/os/$arch", - "https://mirror.colonelhosting.com/archlinux/$repo/os/$arch", - "https://us.arch.niranjan.co/$repo/os/$arch", - "https://mirror.hasphetica.win/archlinux/$repo/os/$arch", - "https://arch-mirror.marcusspencer.xyz:4443/archlinux/$repo/os/$arch", - "https://us-mnz.soulharsh007.dev/archlinux/$repo/os/$arch", - "https://mirror.akane.network/archmirror/$repo/os/$arch", - "https://arch.miningtcup.me/$repo/os/$arch", - "https://mirrors.smeal.xyz/arch-linux/$repo/os/$arch", - "https://arch-mirror.brightlight.today/$repo/os/$arch", - "https://yonderly.org/mirrors/archlinux/$repo/os/$arch", - "https://mirrors.lahansons.com/archlinux/$repo/os/$arch", - "https://mirror.givebytes.net/archlinux/$repo/os/$arch" + "https://mirror.theash.xyz/arch/$repo/os/$arch" ] }, "optional_repositories": [] @@ -112,9 +70,10 @@ "docker-compose", "emacs-wayland", "flatpak", + "fuzzel", "git", "gnome-keyring", - "graphite-grub-theme-default-1080p", + "graphite-grub-theme", "libmpeg2", "librewolf-bin", "mpv", @@ -174,7 +133,7 @@ "mise", "mako" ], - "parallel downloads": 0, + "parallel downloads": 3, "profile_config": null, "services": ["bluetooth", "ly", "NetworkManager", "reflector", "polkit"], "swap": true, |