From 4394b44c93c0e3680e49abb4a7f3c49748f50db9 Mon Sep 17 00:00:00 2001 From: matthewcroughan Date: Thu, 2 Mar 2023 17:37:15 +0000 Subject: [PATCH] invokeai: update to v2.3.1.post2 Co-authored-by: Max --- flake.lock | 8 +- flake.nix | 2 +- modules/aipython3/overlays.nix | 2 + packages/compel/default.nix | 27 +++ packages/diffusers/default.nix | 4 +- packages/safetensors/Cargo.lock | 378 +++++++++++++++++++++++++++++++ packages/safetensors/default.nix | 83 +++++++ projects/invokeai/package.nix | 31 ++- 8 files changed, 524 insertions(+), 11 deletions(-) create mode 100644 packages/compel/default.nix create mode 100644 packages/safetensors/Cargo.lock create mode 100644 packages/safetensors/default.nix diff --git a/flake.lock b/flake.lock index 616bd60..3952d25 100644 --- a/flake.lock +++ b/flake.lock @@ -119,16 +119,16 @@ "invokeai-src": { "flake": false, "locked": { - "lastModified": 1672601215, - "narHash": "sha256-YnT8bvWB7aqK+J8zf2uem3yJI0ehPki+IGN7Xg1+Z2Q=", + "lastModified": 1677475057, + "narHash": "sha256-REtyVcyRgspn1yYvB4vIHdOrPRZRNSSraepHik9MfgE=", "owner": "invoke-ai", "repo": "InvokeAI", - "rev": "c3f82d44815eff47169818d8671fa7ff862fe202", + "rev": "650f4bb58ceca458bff1410f35cd6d6caad399c6", "type": "github" }, "original": { "owner": "invoke-ai", - "ref": "v2.2.5", + "ref": "v2.3.1.post2", "repo": "InvokeAI", "type": "github" } diff --git a/flake.nix b/flake.nix index 67d0642..58a8d01 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,7 @@ url = "github:NixOS/nixpkgs/nixos-unstable"; }; invokeai-src = { - url = "github:invoke-ai/InvokeAI/v2.2.5"; + url = "github:invoke-ai/InvokeAI/v2.3.1.post2"; flake = false; }; koboldai-src = { diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 2c9912a..329d8ef 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -21,6 +21,8 @@ pkgs: { callPackage = final.callPackage; rmCallPackage = path: args: rm (callPackage path args); in { + safetensors = callPackage ../../packages/safetensors { }; + compel = callPackage ../../packages/compel { }; apispec-webframeworks = callPackage ../../packages/apispec-webframeworks { }; pydeprecate = callPackage ../../packages/pydeprecate { }; taming-transformers-rom1504 = diff --git a/packages/compel/default.nix b/packages/compel/default.nix new file mode 100644 index 0000000..a05dae4 --- /dev/null +++ b/packages/compel/default.nix @@ -0,0 +1,27 @@ +{ buildPythonPackage, fetchPypi, lib, setuptools, transformers, diffusers, torch }: + +buildPythonPackage rec { + pname = "compel"; + version = "0.1.7"; + format = "pyproject"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-JP+PX0yENTNnfsAJ/hzgIA/cr/RhIWV1GEa1rYTdlnc="; + }; + + propagatedBuildInputs = [ + setuptools + diffusers + transformers + torch + ]; + +# # TODO FIXME + doCheck = false; + + meta = { + description = "A prompting enhancement library for transformers-type text embedding systems"; + homepage = "https://github.com/damian0815/compel"; + }; +} diff --git a/packages/diffusers/default.nix b/packages/diffusers/default.nix index 580bf63..6b7e005 100644 --- a/packages/diffusers/default.nix +++ b/packages/diffusers/default.nix @@ -34,14 +34,14 @@ buildPythonPackage rec { pname = "diffusers"; - version = "0.7.2"; + version = "0.13.1"; disabled = isPy27; format = "pyproject"; src = fetchPypi { inherit pname version; - sha256 = "sha256-+4FP/RUMxvRwOAuMalIRgad76y9EE00qrS5M2Kos7Q4="; + sha256 = "sha256-KbZ6HvNNn1sFw+qwpJZpfaskllLSmNp0JVuKpPxPjkw="; }; propagatedBuildInputs = [ diff --git a/packages/safetensors/Cargo.lock b/packages/safetensors/Cargo.lock new file mode 100644 index 0000000..1febf05 --- /dev/null +++ b/packages/safetensors/Cargo.lock @@ -0,0 +1,378 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "safetensors" +version = "0.2.8" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "safetensors-python" +version = "0.2.8" +dependencies = [ + "libloading", + "memmap2", + "pyo3", + "safetensors", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" diff --git a/packages/safetensors/default.nix b/packages/safetensors/default.nix new file mode 100644 index 0000000..74beb1a --- /dev/null +++ b/packages/safetensors/default.nix @@ -0,0 +1,83 @@ +{ lib +, fetchPypi +, runCommand +, rustPlatform +, buildPythonPackage +, setuptools +, setuptools-rust +, wheel +, black +, click +, huggingface-hub +, isort +, jax +, numpy +, pytest +, pytest-benchmark +, tensorflow +, torch +}: + +let + + pname = "safetensors"; + version = "0.2.8"; + + patchedSrc = runCommand "patched-src" { + src = fetchPypi { + inherit pname version; + hash = "sha256-JyCyCmo4x5ncp5vXbK7qwvffWFqdT31Z+n4o7/nMsn8="; + }; + } '' + unpackPhase + cp ${./Cargo.lock} $sourceRoot/Cargo.lock + cp -r $sourceRoot $out + ''; +in + +buildPythonPackage { + inherit pname version; + format = "pyproject"; + src = patchedSrc; + + cargoDeps = rustPlatform.fetchCargoTarball { + src = patchedSrc; + name = "${pname}-${version}"; + hash = "sha256-IZKaw4NquK/BbIv1xkMFgNR20vve4H6Re76mvxtcNUA="; + }; + + nativeBuildInputs = [ + setuptools + setuptools-rust + wheel + rustPlatform.cargoSetupHook + rustPlatform.rust.cargo + rustPlatform.rust.rustc + ]; + + propagatedBuildInputs = [ + black + click +# flake +# flax +# h + huggingface-hub + isort + jax + numpy + pytest + pytest-benchmark + setuptools-rust + tensorflow + torch + ]; + + pythonImportsCheck = [ "safetensors" ]; + + meta = with lib; { + description = "Fast and Safe Tensor serialization"; + homepage = "https://pypi.org/project/safetensors/"; + license = with licenses; [ ]; + maintainers = with maintainers; [ ]; + }; +} diff --git a/projects/invokeai/package.nix b/projects/invokeai/package.nix index 98cdf97..6473a11 100644 --- a/projects/invokeai/package.nix +++ b/projects/invokeai/package.nix @@ -8,14 +8,15 @@ let getVersion = lib.flip lib.pipe [ - (src: builtins.readFile "${src}/setup.py") - (builtins.match ".+VERSION = '([^']+)'.+") + (src: builtins.readFile "${src}/ldm/invoke/_version.py") + (builtins.match ".*__version__='([^']+)'.*") builtins.head ]; in aipython3.buildPythonPackage { pname = "InvokeAI"; + format = "pyproject"; version = getVersion src; inherit src; propagatedBuildInputs = with aipython3; [ @@ -25,6 +26,8 @@ aipython3.buildPythonPackage { pudb imageio imageio-ffmpeg + compel + npyscreen pytorch-lightning protobuf omegaconf @@ -54,10 +57,12 @@ aipython3.buildPythonPackage { eventlet clipseg getpass-asterisk + safetensors + datasets ]; nativeBuildInputs = [ aipython3.pythonRelaxDepsHook ]; - pythonRemoveDeps = [ "clip" "pyreadline3" "flaskwebgui" ]; - pythonRelaxDeps = [ "protobuf" ]; + pythonRemoveDeps = [ "clip" "pyreadline3" "flaskwebgui" "opencv-python" ]; + pythonRelaxDeps = [ "protobuf" "flask" "flask-socketio" "pytorch-lightning" ]; makeWrapperArgs = [ '' --run ' if [ -d "/usr/lib/wsl/lib" ] @@ -70,6 +75,24 @@ aipython3.buildPythonPackage { ' '' ]; + patchPhase = '' + runHook prePatch + + # Add subprocess to the imports + substituteInPlace ./ldm/invoke/config/invokeai_configure.py --replace \ + 'import shutil' \ +' +import shutil +import subprocess +' + # shutil.copytree will inherit the permissions of files in the /nix/store + # which are read only, so we subprocess.call cp instead and tell it not to + # preserve the mode + substituteInPlace ./ldm/invoke/config/invokeai_configure.py --replace \ + "shutil.copytree(configs_src, configs_dest, dirs_exist_ok=True)" \ + "subprocess.call('cp -r --no-preserve=mode {configs_src} {configs_dest}'.format(configs_src=configs_src, configs_dest=configs_dest), shell=True)" + runHook postPatch + ''; postFixup = '' chmod +x $out/bin/* wrapPythonPrograms