From 22aa0dcae25337108b592572c1363188dead9069 Mon Sep 17 00:00:00 2001 From: Kai Norman Clasen Date: Wed, 5 Jun 2024 10:45:24 +0200 Subject: [PATCH 1/3] feat: Add nix support Build `audible-cli` via nix _including_ all plugins! Added necessary code to create an AppImage for Linux + docker images via nix. Also add GitHub action logic to check whether the package continues to build via nix, including the AppImage and docker image. Currently limiting support to Linux x86, as this is the only local machine. --- .github/workflows/nix.yml | 36 ++++++++ .gitignore | 3 + flake.lock | 175 ++++++++++++++++++++++++++++++++++++++ flake.nix | 54 ++++++++++++ nix/default.nix | 115 +++++++++++++++++++++++++ nix/isbntools.nix | 42 +++++++++ nix/overlays.nix | 33 +++++++ 7 files changed, 458 insertions(+) create mode 100644 .github/workflows/nix.yml create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/default.nix create mode 100644 nix/isbntools.nix create mode 100644 nix/overlays.nix diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 0000000..8585027 --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,36 @@ +name: Nix-Test + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Checkout + uses: actions/checkout@v4 + - name: Run `nix fmt` + run: nix fmt -- --check * + - name: Run `flake checks` + run: nix flake check -L + - name: Create AppImage + run: nix build .#audible-cli-AppImage + - name: Test appimage + run: ./result decrypt --help + - name: Rename AppImage + if: startsWith(github.ref, 'refs/tags/') + run: cp ./result audible-cli.AppImage + - name: Release-AppImage + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + files: audible-cli.AppImage + # FUTURE: Push the docker container to an image registry + # see: https://github.com/containers/skopeo + # and usage in: https://github.com/seanrmurphy/nix-container-build-gha/blob/main/scripts/build_and_push_image.sh diff --git a/.gitignore b/.gitignore index 3183e33..441d232 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ /src/audible_cli/*.bak library.csv +result +*.AppImage + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..890089f --- /dev/null +++ b/flake.lock @@ -0,0 +1,175 @@ +{ + "nodes": { + "appimage-runtime": { + "flake": false, + "locked": { + "lastModified": 1652289700, + "narHash": "sha256-uxQBDy/JA7uEboTOUmGaZ2FAKY/0dQ9c0A0N8+J+a7I=", + "owner": "AppImageCrafters", + "repo": "appimage-runtime", + "rev": "6500a1ef68e039caba2ebab1c7ed74c2ea9e67a5", + "type": "github" + }, + "original": { + "owner": "AppImageCrafters", + "repo": "appimage-runtime", + "type": "github" + } + }, + "flake-compat": { + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "revCount": 57, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1656928814, + "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-appimage": { + "inputs": { + "appimage-runtime": "appimage-runtime", + "flake-compat": "flake-compat_2", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "squashfuse": "squashfuse" + }, + "locked": { + "lastModified": 1717557985, + "narHash": "sha256-D0Bts6935bgRilLnMln9aTA7/Y74HN6x7iJ0IN2HaZM=", + "owner": "ralismark", + "repo": "nix-appimage", + "rev": "46aef0153eb833e0b896b60ca2bdb6354b410e45", + "type": "github" + }, + "original": { + "owner": "ralismark", + "repo": "nix-appimage", + "type": "github" + } + }, + "nix-filter": { + "locked": { + "lastModified": 1710156097, + "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=", + "owner": "numtide", + "repo": "nix-filter", + "rev": "3342559a24e85fc164b295c3444e8a139924675b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "nix-filter", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1659526864, + "narHash": "sha256-XFzXrc1+6DZb9hBgHfEzfwylPUSqVFJbQPs8eOgYufU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "478f3cbc8448b5852539d785fbfe9a53304133be", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-22.05", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1716715802, + "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "nix-appimage": "nix-appimage", + "nix-filter": "nix-filter", + "nixpkgs": "nixpkgs_2", + "systems": "systems" + } + }, + "squashfuse": { + "flake": false, + "locked": { + "lastModified": 1655253282, + "narHash": "sha256-RIhDXzpmrYUOwj5OYzjWKJw0cwE+L3t/9pIkg/hFXA0=", + "owner": "vasi", + "repo": "squashfuse", + "rev": "d1d7ddafb765098b34239eacaf2f9abee1fbc27c", + "type": "github" + }, + "original": { + "owner": "vasi", + "repo": "squashfuse", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1680978846, + "narHash": "sha256-Gtqg8b/v49BFDpDetjclCYXm8mAnTrUzR0JnE2nv5aw=", + "owner": "nix-systems", + "repo": "x86_64-linux", + "rev": "2ecfcac5e15790ba6ce360ceccddb15ad16d08a8", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "x86_64-linux", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..069afe5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,54 @@ +{ + description = "Nix related tooling for a command line interface for audible. With the CLI you can download your Audible books, cover, chapter files & conver them."; + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + systems.url = "github:nix-systems/x86_64-linux"; + flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"; + nix-filter.url = "github:numtide/nix-filter"; + nix-appimage = { + url = "github:ralismark/nix-appimage"; + }; + }; + outputs = { + self, + nixpkgs, + systems, + nix-appimage, + flake-compat, + nix-filter, + } @ inputs: let + eachSystem = nixpkgs.lib.genAttrs (import systems); + pkgsFor = eachSystem (system: (nixpkgs.legacyPackages.${system}.extend self.overlays.default)); + in { + formatter = eachSystem (system: pkgsFor.${system}.alejandra); + checks = eachSystem (system: self.packages.${system}); + overlays = import ./nix/overlays.nix { + inherit inputs; + lib = nixpkgs.lib; # TODO: Understand this construct + }; + + packages = eachSystem (system: let + pkgs = pkgsFor.${system}; + in rec { + audible-cli = pkgs.audible-cli; + audible-cli-full = pkgs.audible-cli-full; + isbntools = pkgs.python3Packages.isbntools; + audible-cli-AppImage = inputs.nix-appimage.mkappimage.${system} { + drv = audible-cli-full; + name = audible-cli-full.name; + entrypoint = pkgs.lib.getExe audible-cli-full; + }; + audible-cli-docker = pkgs.dockerTools.buildLayeredImage { + name = audible-cli-full.pname; # `.name` has illegal docker tag format + tag = "latest"; + contents = [audible-cli-full]; + config = { + Entrypoint = [ + "${pkgs.lib.getExe audible-cli-full}" + ]; + }; + }; + default = audible-cli-full; + }); + }; +} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..2adb225 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,115 @@ +{ + lib, + nix-filter, + python3Packages, + ffmpeg_7-headless, + enable-plugin-decrypt ? true, + enable-plugin-goodreads-transform ? true, + enable-plugin-annotations ? true, + enable-plugin-image-urls ? true, + enable-plugin-listening-stats ? true, + version ? "git", +}: +# The core of the code was taken from nixpkgs and with special thanks to the +# upstream maintainer `jvanbruegge` +# https://github.com/NixOS/nixpkgs/blob/63c3a29ca82437c87573e4c6919b09a24ea61b0f/pkgs/by-name/au/audible-cli/package.nix +python3Packages.buildPythonApplication { + pname = "audible-cli"; + inherit version; + pyproject = true; + + src = nix-filter { + root = ./..; + include = + [ + # Include the "src" path relative to the root + "src" + "LICENSE" + "setup.cfg" + "README.md" # Required by `setup.cfg` + "setup.py" + "nix" + ] + ++ lib.optionals enable-plugin-annotations [ + "plugin_cmds/cmd_get-annotations.py" + ] + ++ lib.optionals enable-plugin-goodreads-transform [ + "plugin_cmds/cmd_goodreads-transform.py" + ] + ++ lib.optionals enable-plugin-image-urls [ + "plugin_cmds/cmd_image-urls.py" + ] + ++ lib.optionals enable-plugin-listening-stats [ + "plugin_cmds/cmd_listening-stats.py" + ] + ++ lib.optionals enable-plugin-decrypt [ + "plugin_cmds/cmd_decrypt.py" + ]; + }; + + # there is no real benefit of trying to make ffmpeg smaller, as headless + # only takes about 25MB, whereas Python takes >120MB. + dependencies = lib.optionals enable-plugin-decrypt [ffmpeg_7-headless]; + makeWrapperArgs = + lib.optionals + (enable-plugin-annotations || enable-plugin-goodreads-transform || enable-plugin-image-urls || enable-plugin-listening-stats || enable-plugin-decrypt) + ["--set AUDIBLE_PLUGIN_DIR $src/plugin_cmds"]; + + nativeBuildInputs = with python3Packages; [ + pythonRelaxDepsHook + setuptools + ]; + # FUTURE: Renable once shell completion is fixed! + # ++ [ + # installShellFiles + # ]; + + propagatedBuildInputs = + (with python3Packages; [ + aiofiles + audible + click + httpx + packaging + pillow + questionary + setuptools + tabulate + toml + tqdm + ]) + ++ lib.optionals enable-plugin-goodreads-transform [ + python3Packages.isbntools + ]; + + pythonRelaxDeps = [ + "httpx" + ]; + + # FUTURE: Fix fish code_completions & re-enable them + # postInstall = '' + # export PATH=$out/bin:$PATH + # installShellCompletion --cmd audible \ + # --bash <(source utils/code_completion/audible-complete-bash.sh) \ + # --fish <(source utils/code_completion/audible-complete-zsh-fish.sh) \ + # --zsh <(source utils/code_completion/audible-complete-zsh-fish.sh) + # ''; + + # upstream has no tests + doCheck = false; + # FUTURE: Add import tests for the different plugins! + + pythonImportsCheck = [ + "audible_cli" + ]; + + # passthru.updateScript = pkgs.nix-update-script {}; + + meta = { + description = "A command line interface for audible package. With the cli you can download your Audible books, cover, chapter files"; + license = lib.licenses.agpl3Only; + homepage = "https://github.com/mkb79/audible-cli"; + maintainers = with lib.maintainers; [kai-tub]; + mainProgram = "audible"; + }; +} diff --git a/nix/isbntools.nix b/nix/isbntools.nix new file mode 100644 index 0000000..a0c980a --- /dev/null +++ b/nix/isbntools.nix @@ -0,0 +1,42 @@ +{ + lib, + python3Packages, + fetchFromGitHub, + nix-update-script, +}: +python3Packages.buildPythonPackage rec { + pname = "isbntools"; + version = "4.3.29"; + pyproject = true; + + src = fetchFromGitHub { + owner = "xlcnd"; + repo = "isbntools"; + rev = "refs/tags/v${version}"; + hash = "sha256-s47y14YHL/ihAUCnneDcTlyVQj3rUgUnBLD2dPBGD/Y="; + }; + + nativeBuildInputs = with python3Packages; [ + setuptools + ]; + propagatedBuildInputs = with python3Packages; [ + isbnlib + ]; + + # FUTURE: Configure and enable the upstream tests! + doCheck = false; + + pythonImportsCheck = [ + "isbntools" + ]; + + passthru.updateScript = nix-update-script {}; + + meta = { + description = "A Python framework for 'all things ISBN' including metadata, descriptions, covers..."; + license = lib.licenses.lgpl3Plus; + homepage = "https://github.com/xlcnd/isbntools"; + changelog = "https://github.com/xlcnd/isbntools/tree/v${src.rev}/CHANGES.txt"; + maintainers = [lib.maintainers.kai-tub]; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix new file mode 100644 index 0000000..c95f2a9 --- /dev/null +++ b/nix/overlays.nix @@ -0,0 +1,33 @@ +{ + lib, + inputs, +}: let + mkDate = longDate: (lib.concatStringsSep "-" [ + (builtins.substring 0 4 longDate) + (builtins.substring 4 2 longDate) + (builtins.substring 6 2 longDate) + ]); +in { + default = final: prev: let + date = mkDate (inputs.self.lastModifiedDate or "19700101"); + in { + audible-cli = final.callPackage ./default.nix { + version = "0.3.2b3+date=${date}_${inputs.self.shortRev or "dirty"}"; + nix-filter = inputs.nix-filter.lib; + enable-plugin-decrypt = false; + enable-plugin-goodreads-transform = false; + enable-plugin-annotations = false; + enable-plugin-image-urls = false; + enable-plugin-listening-stats = false; + }; + audible-cli-full = final.callPackage ./default.nix { + version = "0.3.2b3+date=${date}_${inputs.self.shortRev or "dirty"}"; + nix-filter = inputs.nix-filter.lib; + }; + python3Packages = + prev.python3Packages + // { + isbntools = prev.callPackage ./isbntools.nix {}; + }; + }; +} From e90c30dcaaf3468e48877944d697dee2e06c1ea1 Mon Sep 17 00:00:00 2001 From: Kai Norman Clasen Date: Thu, 6 Jun 2024 21:54:47 +0200 Subject: [PATCH 2/3] internal: Add CI job to automatically deploy docker on tagged release --- .github/workflows/nix.yml | 11 +++++++---- flake.nix | 9 +++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 8585027..498a4a1 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -8,8 +8,9 @@ jobs: build: runs-on: ubuntu-latest permissions: - contents: write + contents: write # write required for action-gh-release id-token: write + packages: write # write required for pushing docker steps: - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main @@ -31,6 +32,8 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: files: audible-cli.AppImage - # FUTURE: Push the docker container to an image registry - # see: https://github.com/containers/skopeo - # and usage in: https://github.com/seanrmurphy/nix-container-build-gha/blob/main/scripts/build_and_push_image.sh + - name: Release Docker + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + nix run .#docker-pusher diff --git a/flake.nix b/flake.nix index 069afe5..f81c933 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,15 @@ }; }; default = audible-cli-full; + docker-pusher = pkgs.writeShellApplication { + name = "docker-pusher"; + runtimeInputs = [pkgs.skopeo]; + text = '' + nix build .#audible-cli-docker + DOCKER_REPOSITORY="docker://ghcr.io/mkb79/audible-cli" + skopeo --insecure-policy copy "docker-archive:result" "$DOCKER_REPOSITORY" + ''; + }; }); }; } From c404d804668ab8f8b15e8659c6af9c2326266457 Mon Sep 17 00:00:00 2001 From: Kai Norman Clasen Date: Thu, 6 Jun 2024 22:01:14 +0200 Subject: [PATCH 3/3] nix: Derive version string from _version.py Take advantage of the TOML-like structure of `_version.py` file to easily parse the given version string for the nix package version. --- nix/overlays.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/overlays.nix b/nix/overlays.nix index c95f2a9..8c6ce73 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -10,9 +10,12 @@ in { default = final: prev: let date = mkDate (inputs.self.lastModifiedDate or "19700101"); + # Take advantage of the 'accidentally' TOML-compatible file structure + _versionPy = builtins.fromTOML (builtins.readFile ../src/audible_cli/_version.py); + version = _versionPy.__version__; in { audible-cli = final.callPackage ./default.nix { - version = "0.3.2b3+date=${date}_${inputs.self.shortRev or "dirty"}"; + version = "${version}+date=${date}_${inputs.self.shortRev or "dirty"}"; nix-filter = inputs.nix-filter.lib; enable-plugin-decrypt = false; enable-plugin-goodreads-transform = false; @@ -21,7 +24,7 @@ in { enable-plugin-listening-stats = false; }; audible-cli-full = final.callPackage ./default.nix { - version = "0.3.2b3+date=${date}_${inputs.self.shortRev or "dirty"}"; + version = "${version}+date=${date}_${inputs.self.shortRev or "dirty"}"; nix-filter = inputs.nix-filter.lib; }; python3Packages =