From 11ea9ebc5d0a49a999d4ead880a05dac7ca5c8be Mon Sep 17 00:00:00 2001 From: arcnmx Date: Mon, 6 Mar 2023 07:25:53 -0800 Subject: [PATCH 1/3] flake: update --- flake.lock | 12 ++-- modules/aipython3/overlays.nix | 30 ++++++--- packages/apispec-webframeworks/default.nix | 1 + packages/filterpy/default.nix | 40 ------------ packages/hatch-requirements-txt/default.nix | 37 ----------- packages/jsonmerge/default.nix | 34 ---------- packages/markdown-it-py/default.nix | 70 --------------------- packages/shap/default.nix | 63 ------------------- packages/timm/default.nix | 26 -------- projects/invokeai/package.nix | 3 +- 10 files changed, 31 insertions(+), 285 deletions(-) delete mode 100644 packages/filterpy/default.nix delete mode 100644 packages/hatch-requirements-txt/default.nix delete mode 100644 packages/jsonmerge/default.nix delete mode 100644 packages/markdown-it-py/default.nix delete mode 100644 packages/shap/default.nix delete mode 100644 packages/timm/default.nix diff --git a/flake.lock b/flake.lock index 3952d25..299f98e 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1675933616, - "narHash": "sha256-/rczJkJHtx16IFxMmAWu5nNYcSXNg1YYXTHoGjLrLUA=", + "lastModified": 1677714448, + "narHash": "sha256-Hq8qLs8xFu28aDjytfxjdC96bZ6pds21Yy09mSC156I=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "47478a4a003e745402acf63be7f9a092d51b83d7", + "rev": "dc531e3a9ce757041e1afaff8ee932725ca60002", "type": "github" }, "original": { @@ -208,11 +208,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1675763311, - "narHash": "sha256-bz0Q2H3mxsF1CUfk26Sl9Uzi8/HFjGFD/moZHz1HebU=", + "lastModified": 1677932085, + "narHash": "sha256-+AB4dYllWig8iO6vAiGGYl0NEgmMgGHpy9gzWJ3322g=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fab09085df1b60d6a0870c8a89ce26d5a4a708c2", + "rev": "3c5319ad3aa51551182ac82ea17ab1c6b0f0df89", "type": "github" }, "original": { diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 329d8ef..07c0a5e 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -8,9 +8,29 @@ pkgs: { nativeBuildInputs = old.nativeBuildInputs ++ [ final.pythonRelaxDepsHook ]; pythonRelaxDeps = [ "protobuf" ]; }); + markdown-it-py = prev.markdown-it-py.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ final.pythonRelaxDepsHook ]; + pythonRelaxDeps = [ "linkify-it-py" ]; + passthru = old.passthru // { + optional-dependencies = with final; { + linkify = [ linkify-it-py ]; + plugins = [ mdit-py-plugins ]; + }; + }; + }); + filterpy = prev.filterpy.overrideAttrs (old: { + doInstallCheck = false; + }); + shap = prev.shap.overrideAttrs (old: { + doInstallCheck = false; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ final.packaging ]; + pythonImportsCheck = [ "shap" ]; + + meta = old.meta // { + broken = false; + }; + }); scikit-image = final.scikitimage; - #overriding because of https://github.com/NixOS/nixpkgs/issues/196653 - opencv4 = prev.opencv4.override { openblas = pkgs.blas; }; }; extraDeps = final: prev: let @@ -35,21 +55,16 @@ pkgs: { realesrgan = rmCallPackage ../../packages/realesrgan { opencv-python = final.opencv4; }; codeformer = callPackage ../../packages/codeformer { opencv-python = final.opencv4; }; clipseg = rmCallPackage ../../packages/clipseg { opencv-python = final.opencv4; }; - filterpy = callPackage ../../packages/filterpy { }; kornia = callPackage ../../packages/kornia { }; lpips = callPackage ../../packages/lpips { }; ffmpy = callPackage ../../packages/ffmpy { }; - shap = callPackage ../../packages/shap { }; picklescan = callPackage ../../packages/picklescan { }; diffusers = callPackage ../../packages/diffusers { }; pypatchmatch = callPackage ../../packages/pypatchmatch { }; fonts = callPackage ../../packages/fonts { }; font-roboto = callPackage ../../packages/font-roboto { }; analytics-python = callPackage ../../packages/analytics-python { }; - markdown-it-py = callPackage ../../packages/markdown-it-py { }; gradio = callPackage ../../packages/gradio { }; - hatch-requirements-txt = callPackage ../../packages/hatch-requirements-txt { }; - timm = callPackage ../../packages/timm { }; blip = callPackage ../../packages/blip { }; fairscale = callPackage ../../packages/fairscale { }; torch-fidelity = callPackage ../../packages/torch-fidelity { }; @@ -58,7 +73,6 @@ pkgs: { k-diffusion = callPackage ../../packages/k-diffusion { clean-fid = final.clean-fid; }; accelerate = callPackage ../../packages/accelerate { }; clip-anytorch = callPackage ../../packages/clip-anytorch { }; - jsonmerge = callPackage ../../packages/jsonmerge { }; clean-fid = callPackage ../../packages/clean-fid { }; getpass-asterisk = callPackage ../../packages/getpass-asterisk { }; }; diff --git a/packages/apispec-webframeworks/default.nix b/packages/apispec-webframeworks/default.nix index 1ce4320..c7d9361 100644 --- a/packages/apispec-webframeworks/default.nix +++ b/packages/apispec-webframeworks/default.nix @@ -13,6 +13,7 @@ python3Packages.buildPythonPackage rec { propagatedBuildInputs = with python3Packages; [ apispec packaging + pyyaml ]; nativeCheckInputs = with python3Packages; [ diff --git a/packages/filterpy/default.nix b/packages/filterpy/default.nix deleted file mode 100644 index 52d6b0c..0000000 --- a/packages/filterpy/default.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, numpy -, scipy -, matplotlib -, pytest -, isPy3k -}: - -buildPythonPackage rec { - version = "1.4.5"; - pname = "filterpy"; - disabled = !isPy3k; - - src = fetchPypi { - inherit pname version; - extension = "zip"; - sha256 = "4f2a4d39e4ea601b9ab42b2db08b5918a9538c168cff1c6895ae26646f3d73b1"; - }; - - checkInputs = [ pytest ]; - propagatedBuildInputs = [ numpy scipy matplotlib ]; - - # single test fails (even on master branch of repository) - # project does not use CI - checkPhase = '' - pytest --ignore=filterpy/common/tests/test_discretization.py - ''; - - doCheck = false; - dontUsePythonImportsCheck = true; - - meta = with lib; { - homepage = "https://github.com/rlabbe/filterpy"; - description = "Kalman filtering and optimal estimation library"; - license = licenses.mit; - maintainers = [ maintainers.costrouc ]; - }; -} diff --git a/packages/hatch-requirements-txt/default.nix b/packages/hatch-requirements-txt/default.nix deleted file mode 100644 index 070d723..0000000 --- a/packages/hatch-requirements-txt/default.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, pythonOlder -, hatchling -, packaging -}: - -buildPythonPackage rec { - pname = "hatch-requirements-txt"; - version = "0.2.0"; - disabled = pythonOlder "3.6.1"; - format = "pyproject"; - - # We use the Pypi release, as it provides prebuild webui assets, - # and its releases are also more frequent than github tags - src = fetchPypi { - pname = "hatch_requirements_txt"; - inherit version; - sha256 = "sha256-I8z34vvPK7Mg9+Xg2nMgNcgeh5QFB0LV2j0iwzA1QGc="; - }; - - nativeBuildInputs = [ - ]; - propagatedBuildInputs = [ - hatchling - packaging - ]; - - # TODO FIXME - doCheck = false; - - meta = with lib; { - homepage = "https://github.com/repo-helper/hatch-requirements-txt"; - description = "Hatchling plugin to read project dependencies from requirements.txt"; - }; -} diff --git a/packages/jsonmerge/default.nix b/packages/jsonmerge/default.nix deleted file mode 100644 index 06af8f2..0000000 --- a/packages/jsonmerge/default.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, jsonschema -, pytestCheckHook -}: - -buildPythonPackage rec { - pname = "jsonmerge"; - version = "1.8.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "a86bfc44f32f6a28b749743df8960a4ce1930666b3b73882513825f845cb9558"; - }; - - propagatedBuildInputs = [ jsonschema ]; - - checkInputs = [ pytestCheckHook ]; - - disabledTests = [ - # Fails with "Unresolvable JSON pointer" - "test_local_reference_in_meta" - "test_reference_in_meta" - ]; - - meta = with lib; { - description = "Merge a series of JSON documents"; - homepage = "https://github.com/avian2/jsonmerge"; - changelog = "https://github.com/avian2/jsonmerge/blob/jsonmerge-${version}/ChangeLog"; - license = licenses.mit; - maintainers = with maintainers; [ emily ]; - }; -} diff --git a/packages/markdown-it-py/default.nix b/packages/markdown-it-py/default.nix deleted file mode 100644 index 61b7a19..0000000 --- a/packages/markdown-it-py/default.nix +++ /dev/null @@ -1,70 +0,0 @@ -{ lib -, attrs -, buildPythonPackage -, fetchFromGitHub -, linkify-it-py -, mdit-py-plugins -, mdurl -, psutil -, pytest-benchmark -, pytest-regressions -, pytestCheckHook -, pythonOlder -, typing-extensions -}: - -buildPythonPackage rec { - pname = "markdown-it-py"; - version = "2.1.0"; - format = "flit"; - - disabled = pythonOlder "3.6"; - - src = fetchFromGitHub { - owner = "executablebooks"; - repo = pname; - rev = "refs/tags/v${version}"; - sha256 = "sha256-6UATJho3SuIbLktZtFcDrCTWIAh52E+n5adcgl49un0="; - }; - - postPatch = '' - # Allow linkify-it-py v2. - # This only affects projects that depend on 'markdown-it-py[linkify]' - # https://github.com/executablebooks/markdown-it-py/pull/218 - substituteInPlace pyproject.toml \ - --replace "linkify-it-py~=1.0" "linkify-it-py>=1,<3" - ''; - - - propagatedBuildInputs = [ - attrs - mdurl - ] ++ lib.optional (pythonOlder "3.8") [ - typing-extensions - ]; - - passthru.optional-dependencies = { - linkify = [ linkify-it-py ]; - plugins = [ mdit-py-plugins ]; - }; - - checkInputs = [ - linkify-it-py - psutil - pytest-benchmark - pytest-regressions - pytestCheckHook - ]; - - pythonImportsCheck = [ - "markdown_it" - ]; - - meta = with lib; { - description = "Markdown parser in Python"; - homepage = "https://markdown-it-py.readthedocs.io/"; - changelog = "https://github.com/executablebooks/markdown-it-py/blob/${src.rev}/CHANGELOG.md"; - license = licenses.mit; - maintainers = with maintainers; [ bhipple ]; - }; -} diff --git a/packages/shap/default.nix b/packages/shap/default.nix deleted file mode 100644 index f132d67..0000000 --- a/packages/shap/default.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, writeText -, isPy27 -, pytestCheckHook -, pytest-mpl -, numpy -, scipy -, scikit-learn -, pandas -, transformers -, opencv4 -, lightgbm -, catboost -, pyspark -, sentencepiece -, tqdm -, slicer -, numba -, matplotlib -, nose -, lime -, cloudpickle -, ipython -, packaging -}: - -buildPythonPackage rec { - pname = "shap"; - version = "0.41.0"; - disabled = isPy27; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256-pJ6k1lqtvIRaaV+j1+oL38jJKLjiE7D+7fWGit5LPKU="; - }; - - propagatedBuildInputs = [ - numpy - scipy - scikit-learn - pandas - tqdm - slicer - numba - cloudpickle - packaging - ]; - - passthru.optional-dependencies = { - plots = [ matplotlib ipython ]; - others = [ lime ]; - }; - - # TODO FIXME - doCheck = false; - - meta = with lib; { - description = "A unified approach to explain the output of any machine learning model"; - homepage = "https://github.com/slundberg/shap"; - }; -} diff --git a/packages/timm/default.nix b/packages/timm/default.nix deleted file mode 100644 index d168066..0000000 --- a/packages/timm/default.nix +++ /dev/null @@ -1,26 +0,0 @@ -# WARNING: This file was automatically generated. You should avoid editing it. -# If you run pynixify again, the file will be either overwritten or -# deleted, and you will lose the changes you made to it. - -{ buildPythonPackage, fetchPypi, huggingface-hub, lib, pyyaml, torch -, torchvision }: - -buildPythonPackage rec { - pname = "timm"; - version = "0.6.11"; - - src = fetchPypi { - inherit pname version; - sha256 = "199zgg574gijw52jn4fhfzaqlbwhq0z8kav4k34xifssnr18hmh9"; - }; - - propagatedBuildInputs = [ torch torchvision pyyaml huggingface-hub ]; - - # TODO FIXME - doCheck = false; - - meta = with lib; { - description = "(Unofficial) PyTorch Image Models"; - homepage = "https://github.com/rwightman/pytorch-image-models"; - }; -} diff --git a/projects/invokeai/package.nix b/projects/invokeai/package.nix index 6473a11..6350d9a 100644 --- a/projects/invokeai/package.nix +++ b/projects/invokeai/package.nix @@ -21,6 +21,7 @@ aipython3.buildPythonPackage { inherit src; propagatedBuildInputs = with aipython3; [ numpy + dnspython albumentations opencv4 pudb @@ -62,7 +63,7 @@ aipython3.buildPythonPackage { ]; nativeBuildInputs = [ aipython3.pythonRelaxDepsHook ]; pythonRemoveDeps = [ "clip" "pyreadline3" "flaskwebgui" "opencv-python" ]; - pythonRelaxDeps = [ "protobuf" "flask" "flask-socketio" "pytorch-lightning" ]; + pythonRelaxDeps = [ "dnspython" "protobuf" "flask" "flask-socketio" "pytorch-lightning" ]; makeWrapperArgs = [ '' --run ' if [ -d "/usr/lib/wsl/lib" ] From 2f69fda039baf8aeba7e6dcb684e22b07ba02b4d Mon Sep 17 00:00:00 2001 From: arcnmx Date: Fri, 3 Mar 2023 15:25:48 -0800 Subject: [PATCH 2/3] flake: callPackage streamlit --- modules/aipython3/overlays.nix | 39 ++++++++++++++++++++-------------- projects/invokeai/package.nix | 7 +----- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 07c0a5e..ac3a0a2 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -1,13 +1,12 @@ pkgs: { - fixPackages = final: prev: { - pytorch-lightning = prev.pytorch-lightning.overrideAttrs (old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ final.pythonRelaxDepsHook ]; - pythonRelaxDeps = [ "protobuf" ]; - }); - wandb = prev.wandb.overrideAttrs (old: { + fixPackages = final: prev: let + relaxProtobuf = pkg: pkg.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ final.pythonRelaxDepsHook ]; pythonRelaxDeps = [ "protobuf" ]; }); + in { + pytorch-lightning = relaxProtobuf prev.pytorch-lightning; + wandb = relaxProtobuf prev.wandb; markdown-it-py = prev.markdown-it-py.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ final.pythonRelaxDepsHook ]; pythonRelaxDeps = [ "linkify-it-py" ]; @@ -30,7 +29,11 @@ pkgs: { broken = false; }; }); - scikit-image = final.scikitimage; + streamlit = let + streamlit = final.callPackage (pkgs.path + "/pkgs/applications/science/machine-learning/streamlit") { + protobuf3 = final.protobuf; + }; + in final.toPythonModule (relaxProtobuf streamlit); }; extraDeps = final: prev: let @@ -41,20 +44,24 @@ pkgs: { callPackage = final.callPackage; rmCallPackage = path: args: rm (callPackage path args); in { + scikit-image = final.scikitimage; + opencv-python-headless = final.opencv-python; + opencv-python = final.opencv4; + safetensors = callPackage ../../packages/safetensors { }; compel = callPackage ../../packages/compel { }; apispec-webframeworks = callPackage ../../packages/apispec-webframeworks { }; pydeprecate = callPackage ../../packages/pydeprecate { }; taming-transformers-rom1504 = callPackage ../../packages/taming-transformers-rom1504 { }; - albumentations = rmCallPackage ../../packages/albumentations { opencv-python-headless = final.opencv4; }; - qudida = rmCallPackage ../../packages/qudida { opencv-python-headless = final.opencv4; }; - gfpgan = rmCallPackage ../../packages/gfpgan { opencv-python = final.opencv4; }; - basicsr = rmCallPackage ../../packages/basicsr { opencv-python = final.opencv4; }; - facexlib = rmCallPackage ../../packages/facexlib { opencv-python = final.opencv4; }; - realesrgan = rmCallPackage ../../packages/realesrgan { opencv-python = final.opencv4; }; - codeformer = callPackage ../../packages/codeformer { opencv-python = final.opencv4; }; - clipseg = rmCallPackage ../../packages/clipseg { opencv-python = final.opencv4; }; + albumentations = rmCallPackage ../../packages/albumentations { }; + qudida = rmCallPackage ../../packages/qudida { }; + gfpgan = rmCallPackage ../../packages/gfpgan { }; + basicsr = rmCallPackage ../../packages/basicsr { }; + facexlib = rmCallPackage ../../packages/facexlib { }; + realesrgan = rmCallPackage ../../packages/realesrgan { }; + codeformer = callPackage ../../packages/codeformer { }; + clipseg = rmCallPackage ../../packages/clipseg { }; kornia = callPackage ../../packages/kornia { }; lpips = callPackage ../../packages/lpips { }; ffmpy = callPackage ../../packages/ffmpy { }; @@ -70,7 +77,7 @@ pkgs: { torch-fidelity = callPackage ../../packages/torch-fidelity { }; resize-right = callPackage ../../packages/resize-right { }; torchdiffeq = callPackage ../../packages/torchdiffeq { }; - k-diffusion = callPackage ../../packages/k-diffusion { clean-fid = final.clean-fid; }; + k-diffusion = callPackage ../../packages/k-diffusion { }; accelerate = callPackage ../../packages/accelerate { }; clip-anytorch = callPackage ../../packages/clip-anytorch { }; clean-fid = callPackage ../../packages/clean-fid { }; diff --git a/projects/invokeai/package.nix b/projects/invokeai/package.nix index 6350d9a..0fcd5cc 100644 --- a/projects/invokeai/package.nix +++ b/projects/invokeai/package.nix @@ -1,6 +1,4 @@ { aipython3 -# dependencies -, streamlit # misc , lib , src @@ -33,10 +31,7 @@ aipython3.buildPythonPackage { protobuf omegaconf test-tube - ((streamlit.overrideAttrs (old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pythonRelaxDepsHook ]; - pythonRelaxDeps = [ "protobuf" ]; - })).override { protobuf3 = protobuf; }) + streamlit einops taming-transformers-rom1504 torch-fidelity From 193981b84d5b5f5fa42f17c871b81aec25cd3ff8 Mon Sep 17 00:00:00 2001 From: arcnmx Date: Mon, 6 Mar 2023 08:14:26 -0800 Subject: [PATCH 3/3] packages/diffusers: 0.13.1 -> 0.14.0 --- packages/diffusers/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/diffusers/default.nix b/packages/diffusers/default.nix index 6b7e005..e28e30f 100644 --- a/packages/diffusers/default.nix +++ b/packages/diffusers/default.nix @@ -34,14 +34,14 @@ buildPythonPackage rec { pname = "diffusers"; - version = "0.13.1"; + version = "0.14.0"; disabled = isPy27; format = "pyproject"; src = fetchPypi { inherit pname version; - sha256 = "sha256-KbZ6HvNNn1sFw+qwpJZpfaskllLSmNp0JVuKpPxPjkw="; + sha256 = "sha256-sqQqEtq1OMtFo7DGVQMFO6RG5fLfSDbeOFtSON+DCkY="; }; propagatedBuildInputs = [