uboot: (firmwareOdroidC2/C4) don't invoke patch tool, use patches = [] instead

https://github.com/NixOS/nixpkgs/blob/master/pkgs/stdenv/generic/setup.sh#L948
this can do it nicely.

Signed-off-by: Anton Arapov <anton@deadbeef.mx>
This commit is contained in:
Anton Arapov 2021-04-03 12:58:10 +02:00 committed by Alan Daniels
commit 56de2bcd43
30691 changed files with 3076956 additions and 0 deletions

View file

@ -0,0 +1,6 @@
Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix
The update procedure is as-follows:
1. Send your change to the upstream poetry2nix repository
2. Get it approved with tests passing
3. Run the update script in pkgs/development/tools/poetry2nix

View file

@ -0,0 +1,157 @@
#!/usr/bin/env python
from concurrent.futures import ThreadPoolExecutor
import subprocess
import textwrap
import argparse
import toml
import json
import sys
from typing import Dict, Any, Tuple, List
class Package:
def __init__(self, attrs: Dict[str, Any]) -> None:
self.attrs = attrs
self.name = attrs["name"]
self.source = self.attrs["source"]
def fetch(self) -> Tuple["Package", subprocess.CompletedProcess]:
raise NotImplementedError()
def expression(self, output: str) -> str:
raise NotImplementedError()
class UrlPackage(Package):
def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
return (
self,
subprocess.run(
[
"nix-prefetch-url",
"--unpack",
self.source["url"],
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
),
)
def expression(self, output: str) -> str:
sha256 = output.rstrip()
return textwrap.dedent("""
%s = super.%s.overridePythonAttrs (
_: {
src = pkgs.fetchzip {
url = "%s";
sha256 = "%s";
};
}
);""" % (self.name, self.name, self.source["url"], sha256))
class GitPackage(Package):
def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
reference = self.source.get("resolved_reference", self.source["reference"])
return (
self,
subprocess.run(
[
"nix-prefetch-git",
"--fetch-submodules",
"--url",
self.source["url"],
"--rev",
reference,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
),
)
def expression(self, output: str) -> str:
meta = json.loads(output)
return textwrap.dedent("""
%s = super.%s.overridePythonAttrs (
_: {
src = pkgs.fetchgit {
url = "%s";
rev = "%s";
sha256 = "%s";
};
}
);""" % (self.name, self.name, meta["url"], meta["rev"], meta["sha256"]))
def parse_args() -> argparse.Namespace:
argparser = argparse.ArgumentParser(description="Poetry2nix CLI")
subparsers = argparser.add_subparsers(dest="subcommand")
subparsers.required = True
parser_lock = subparsers.add_parser("lock", help="Generate overrides for git hashes",)
parser_lock.add_argument(
"--lock", default="poetry.lock", help="Path to input poetry.lock",
)
parser_lock.add_argument(
"--out", default="poetry-git-overlay.nix", help="Output file",
)
return argparser.parse_args()
def indent(expr: str, spaces: int = 2) -> str:
i = " " * spaces
return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")])
def main() -> None:
args = parse_args()
with open(args.lock) as lockf:
lock = toml.load(lockf)
pkgs: List[Package] = []
for pkg in lock["package"]:
if "source" in pkg:
source_type = pkg["source"]["type"]
if source_type == "git":
pkgs.append(GitPackage(pkg))
elif source_type == "url":
pkgs.append(UrlPackage(pkg))
with ThreadPoolExecutor() as e:
futures = []
for pkg in pkgs:
futures.append(e.submit(pkg.fetch))
lines = [
"{ pkgs }:",
"self: super: {",
]
for f in futures:
package, p = f.result()
if p.returncode != 0:
sys.stderr.write(p.stderr)
sys.stderr.flush()
exit(p.returncode)
expr = package.expression(p.stdout)
lines.append(indent(expr))
lines.extend(["", "}", ""])
expr = "\n".join(lines)
with open(args.out, "w") as fout:
fout.write(expr)
print(f"Wrote {args.out}")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,49 @@
{ pkgs ? import <nixpkgs> { }
, lib ? pkgs.lib
, version
}:
let
inherit (pkgs) python3;
in
pkgs.stdenv.mkDerivation {
pname = "poetry2nix";
inherit version;
buildInputs = [
(python3.withPackages (ps: [ ps.toml ]))
];
nativeBuildInputs = [
pkgs.makeWrapper
];
src = ./bin;
dontConfigure = true;
buildPhase = ''
runHook preBuild
patchShebangs poetry2nix
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p $out/bin
mv poetry2nix $out/bin
wrapProgram $out/bin/poetry2nix --prefix PATH ":" ${lib.makeBinPath [
pkgs.nix-prefetch-git
]}
runHook postInstall
'';
meta = {
homepage = "https://github.com/nix-community/poetry2nix";
description = "CLI to supplement sha256 hashes for git dependencies";
license = lib.licenses.mit;
maintainers = [ lib.maintainers.adisbladis ];
};
}

View file

@ -0,0 +1,470 @@
{ pkgs ? import <nixpkgs> { }
, lib ? pkgs.lib
, poetry ? null
, poetryLib ? import ./lib.nix { inherit lib pkgs; stdenv = pkgs.stdenv; }
}:
let
# Poetry2nix version
version = "1.30.0";
inherit (poetryLib) isCompatible readTOML moduleName;
# Map SPDX identifiers to license names
spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
# Get license by id falling back to input string
getLicenseBySpdxId = spdxId: spdxLicenses.${spdxId} or spdxId;
# Experimental withPlugins functionality
toPluginAble = (import ./plugins.nix { inherit pkgs lib; }).toPluginAble;
mkInputAttrs =
{ py
, pyProject
, attrs
, includeBuildSystem ? true
}:
let
getInputs = attr: attrs.${attr} or [ ];
# Get dependencies and filter out depending on interpreter version
getDeps = depAttr:
let
compat = isCompatible (poetryLib.getPythonVersion py);
deps = pyProject.tool.poetry.${depAttr} or { };
depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps);
in
(
builtins.map
(
dep:
let
pkg = py.pkgs."${moduleName dep}";
constraints = deps.${dep}.python or "";
isCompat = compat constraints;
in
if isCompat then pkg else null
)
depAttrs
);
buildSystemPkgs = poetryLib.getBuildSystemPkgs {
inherit pyProject;
pythonPackages = py.pkgs;
};
mkInput = attr: extraInputs: getInputs attr ++ extraInputs;
in
{
buildInputs = mkInput "buildInputs" (if includeBuildSystem then buildSystemPkgs else [ ]);
propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]);
nativeBuildInputs = mkInput "nativeBuildInputs" [ ];
checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies");
};
in
lib.makeScope pkgs.newScope (self: {
inherit version;
/* Returns a package of editable sources whose changes will be available without needing to restart the
nix-shell.
In editablePackageSources you can pass a mapping from package name to source directory to have
those packages available in the resulting environment, whose source changes are immediately available.
*/
mkPoetryEditablePackage =
{ projectDir ? null
, pyproject ? projectDir + "/pyproject.toml"
, python ? pkgs.python3
, pyProject ? readTOML pyproject
# Example: { my-app = ./src; }
, editablePackageSources
}:
assert editablePackageSources != { };
import ./editable.nix {
inherit pyProject python pkgs lib poetryLib editablePackageSources;
};
/* Returns a package containing scripts defined in tool.poetry.scripts.
*/
mkPoetryScriptsPackage =
{ projectDir ? null
, pyproject ? projectDir + "/pyproject.toml"
, python ? pkgs.python3
, pyProject ? readTOML pyproject
, scripts ? pyProject.tool.poetry.scripts
}:
assert scripts != { };
import ./shell-scripts.nix {
inherit lib python scripts;
};
/*
Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.
*/
mkPoetryPackages =
{ projectDir ? null
, pyproject ? projectDir + "/pyproject.toml"
, poetrylock ? projectDir + "/poetry.lock"
, overrides ? self.defaultPoetryOverrides
, python ? pkgs.python3
, pwd ? projectDir
, preferWheels ? false
# Example: { my-app = ./src; }
, editablePackageSources ? { }
, pyProject ? readTOML pyproject
}@attrs:
let
/* The default list of poetry2nix override overlays */
mkEvalPep508 = import ./pep508.nix {
inherit lib poetryLib;
inherit (python) stdenv;
};
getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn;
poetryPkg = poetry.override { inherit python; };
scripts = pyProject.tool.poetry.scripts or { };
hasScripts = scripts != { };
scriptsPackage = self.mkPoetryScriptsPackage {
inherit python scripts;
};
editablePackageSources' = lib.filterAttrs (name: path: path != null) editablePackageSources;
hasEditable = editablePackageSources' != { };
editablePackage = self.mkPoetryEditablePackage {
inherit pyProject python;
editablePackageSources = editablePackageSources';
};
poetryLock = readTOML poetrylock;
lockFiles =
let
lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
in
lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles);
specialAttrs = [
"overrides"
"poetrylock"
"projectDir"
"pwd"
"preferWheels"
];
passedAttrs = builtins.removeAttrs attrs specialAttrs;
evalPep508 = mkEvalPep508 python;
# Filter packages by their PEP508 markers & pyproject interpreter version
partitions =
let
supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions;
in
lib.partition supportsPythonVersion poetryLock.package;
compatible = partitions.right;
incompatible = partitions.wrong;
# Create an overridden version of pythonPackages
#
# We need to avoid mixing multiple versions of pythonPackages in the same
# closure as python can only ever have one version of a dependency
baseOverlay = self: super:
let
getDep = depName: self.${depName};
lockPkgs = builtins.listToAttrs (
builtins.map
(
pkgMeta: rec {
name = moduleName pkgMeta.name;
value = self.mkPoetryDep (
pkgMeta // {
inherit pwd preferWheels;
source = pkgMeta.source or null;
files = lockFiles.${name};
pythonPackages = self;
sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name} or { };
}
);
}
)
(lib.reverseList compatible)
);
in
lockPkgs // {
# Create a dummy null package for the current project in case any dependencies depend on the root project (issue #307)
${pyProject.tool.poetry.name} = null;
};
overlays = builtins.map
getFunctorFn
(
[
(
self: super:
let
hooks = self.callPackage ./hooks { };
in
{
mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
inherit lib python poetryLib evalPep508;
};
# # Use poetry-core from the poetry build (pep517/518 build-system)
poetry-core = poetryPkg.passthru.python.pkgs.poetry-core;
poetry = poetryPkg;
__toPluginAble = toPluginAble self;
inherit (hooks) pipBuildHook removePathDependenciesHook removeGitDependenciesHook poetry2nixFixupHook wheelUnpackHook;
} // lib.optionalAttrs (! super ? setuptools-scm) {
# The canonical name is setuptools-scm
setuptools-scm = super.setuptools_scm;
}
)
# Fix infinite recursion in a lot of packages because of checkInputs
(self: super: lib.mapAttrs
(name: value: (
if lib.isDerivation value && lib.hasAttr "overridePythonAttrs" value
then value.overridePythonAttrs (_: { doCheck = false; })
else value
))
super)
# Null out any filtered packages, we don't want python.pkgs from nixpkgs
(self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible))
# Create poetry2nix layer
baseOverlay
] ++ # User provided overrides
(if builtins.typeOf overrides == "list" then overrides else [ overrides ])
);
packageOverrides = lib.foldr lib.composeExtensions (self: super: { }) overlays;
py = python.override { inherit packageOverrides; self = py; };
inputAttrs = mkInputAttrs { inherit py pyProject; attrs = { }; includeBuildSystem = false; };
requiredPythonModules = python.pkgs.requiredPythonModules;
/* Include all the nested dependencies which are required for each package.
This guarantees that using the "poetryPackages" attribute will return
complete list of dependencies for the poetry project to be portable.
*/
storePackages = requiredPythonModules (builtins.foldl' (acc: v: acc ++ v) [ ] (lib.attrValues inputAttrs));
in
{
python = py;
poetryPackages = storePackages
++ lib.optional hasScripts scriptsPackage
++ lib.optional hasEditable editablePackage;
poetryLock = poetryLock;
inherit pyProject;
};
/* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file.
In editablePackageSources you can pass a mapping from package name to source directory to have
those packages available in the resulting environment, whose source changes are immediately available.
Example:
poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; }
*/
mkPoetryEnv =
{ projectDir ? null
, pyproject ? projectDir + "/pyproject.toml"
, poetrylock ? projectDir + "/poetry.lock"
, overrides ? self.defaultPoetryOverrides
, pwd ? projectDir
, python ? pkgs.python3
, preferWheels ? false
, editablePackageSources ? { }
, extraPackages ? ps: [ ]
}:
let
inherit (lib) elem hasAttr;
pyProject = readTOML pyproject;
# Automatically add dependencies with develop = true as editable packages, but only if path dependencies
getEditableDeps = set: lib.mapAttrs
(name: value: projectDir + "/${value.path}")
(lib.filterAttrs (name: dep: dep.develop or false && hasAttr "path" dep) set);
excludedEditablePackageNames = builtins.filter
(pkg: editablePackageSources."${pkg}" == null)
(builtins.attrNames editablePackageSources);
allEditablePackageSources = (
(getEditableDeps (pyProject.tool.poetry."dependencies" or { }))
// (getEditableDeps (pyProject.tool.poetry."dev-dependencies" or { }))
// editablePackageSources
);
editablePackageSources' = builtins.removeAttrs
allEditablePackageSources
excludedEditablePackageNames;
poetryPython = self.mkPoetryPackages {
inherit pyproject poetrylock overrides python pwd preferWheels pyProject;
editablePackageSources = editablePackageSources';
};
inherit (poetryPython) poetryPackages;
# Don't add editable sources to the environment since they will sometimes fail to build and are not useful in the development env
editableAttrs = lib.attrNames editablePackageSources';
envPkgs = builtins.filter (drv: ! lib.elem (drv.pname or drv.name or "") editableAttrs) poetryPackages;
in
poetryPython.python.withPackages (ps: envPkgs ++ (extraPackages ps));
/* Creates a Python application from pyproject.toml and poetry.lock
The result also contains a .dependencyEnv attribute which is a python
environment of all dependencies and this apps modules. This is useful if
you rely on dependencies to invoke your modules for deployment: e.g. this
allows `gunicorn my-module:app`.
*/
mkPoetryApplication =
{ projectDir ? null
, src ? (
# Assume that a project which is the result of a derivation is already adequately filtered
if lib.isDerivation projectDir then projectDir else self.cleanPythonSources { src = projectDir; }
)
, pyproject ? projectDir + "/pyproject.toml"
, poetrylock ? projectDir + "/poetry.lock"
, overrides ? self.defaultPoetryOverrides
, meta ? { }
, python ? pkgs.python3
, pwd ? projectDir
, preferWheels ? false
, ...
}@attrs:
let
poetryPython = self.mkPoetryPackages {
inherit pyproject poetrylock overrides python pwd preferWheels;
};
py = poetryPython.python;
inherit (poetryPython) pyProject;
specialAttrs = [
"overrides"
"poetrylock"
"projectDir"
"pwd"
"pyproject"
"preferWheels"
];
passedAttrs = builtins.removeAttrs attrs specialAttrs;
inputAttrs = mkInputAttrs { inherit py pyProject attrs; };
app = py.pkgs.buildPythonPackage (
passedAttrs // inputAttrs // {
nativeBuildInputs = inputAttrs.nativeBuildInputs ++ [
py.pkgs.removePathDependenciesHook
py.pkgs.removeGitDependenciesHook
];
} // {
pname = moduleName pyProject.tool.poetry.name;
version = pyProject.tool.poetry.version;
inherit src;
format = "pyproject";
# Like buildPythonApplication, but without the toPythonModule part
# Meaning this ends up looking like an application but it also
# provides python modules
namePrefix = "";
passthru = {
python = py;
dependencyEnv = (
lib.makeOverridable ({ app, ... }@attrs:
let
args = builtins.removeAttrs attrs [ "app" ] // {
extraLibs = [ app ];
};
in
py.buildEnv.override args)
) { inherit app; };
};
# Extract position from explicitly passed attrs so meta.position won't point to poetry2nix internals
pos = builtins.unsafeGetAttrPos (lib.elemAt (lib.attrNames attrs) 0) attrs;
meta = lib.optionalAttrs (lib.hasAttr "description" pyProject.tool.poetry)
{
inherit (pyProject.tool.poetry) description;
} // lib.optionalAttrs (lib.hasAttr "homepage" pyProject.tool.poetry) {
inherit (pyProject.tool.poetry) homepage;
} // {
inherit (py.meta) platforms;
license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown");
} // meta;
}
);
in
app;
/* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies */
cli = import ./cli.nix {
inherit pkgs lib;
inherit (self) version;
};
# inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages;
inherit (poetryLib) cleanPythonSources;
/*
Create a new default set of overrides with the same structure as the built-in ones
*/
mkDefaultPoetryOverrides = defaults: {
__functor = defaults;
extend = overlay:
let
composed = lib.foldr lib.composeExtensions overlay [ defaults ];
in
self.mkDefaultPoetryOverrides composed;
overrideOverlay = fn:
let
overlay = self: super:
let
defaultSet = defaults self super;
customSet = fn self super;
in
defaultSet // customSet;
in
self.mkDefaultPoetryOverrides overlay;
};
/*
The default list of poetry2nix override overlays
Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
*/
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
/*
Convenience functions for specifying overlays with or without the poerty2nix default overrides
*/
overrides = {
/*
Returns the specified overlay in a list
*/
withoutDefaults = overlay: [
overlay
];
/*
Returns the specified overlay and returns a list
combining it with poetry2nix default overrides
*/
withDefaults = overlay: [
self.defaultPoetryOverrides
overlay
];
};
})

View file

@ -0,0 +1,55 @@
{ pkgs
, lib
, poetryLib
, pyProject
, python
, editablePackageSources
}:
let
name = poetryLib.moduleName pyProject.tool.poetry.name;
# Just enough standard PKG-INFO fields for an editable installation
pkgInfoFields = {
Metadata-Version = "2.1";
Name = name;
# While the pyproject.toml could contain arbitrary version strings, for
# simplicity we just use the same one for PKG-INFO, even though that
# should follow follow PEP 440: https://www.python.org/dev/peps/pep-0345/#version
# This is how poetry transforms it: https://github.com/python-poetry/poetry/blob/6cd3645d889f47c10425961661b8193b23f0ed79/poetry/version/version.py
Version = pyProject.tool.poetry.version;
Summary = pyProject.tool.poetry.description;
};
pkgInfoFile = builtins.toFile "${name}-PKG-INFO"
(lib.concatStringsSep "\n" (lib.mapAttrsToList (key: value: "${key}: ${value}") pkgInfoFields));
entryPointsFile = builtins.toFile "${name}-entry_points.txt"
(lib.generators.toINI { } pyProject.tool.poetry.plugins);
# A python package that contains simple .egg-info and .pth files for an editable installation
editablePackage = python.pkgs.toPythonModule (pkgs.runCommand "${name}-editable"
{ } ''
mkdir -p "$out/${python.sitePackages}"
cd "$out/${python.sitePackages}"
# See https://docs.python.org/3.8/library/site.html for info on such .pth files
# These add another site package path for each line
touch poetry2nix-editable.pth
${lib.concatMapStringsSep "\n"
(src: ''
echo "${toString src}" >> poetry2nix-editable.pth
'')
(lib.attrValues editablePackageSources)}
# Create a very simple egg so pkg_resources can find this package
# See https://setuptools.readthedocs.io/en/latest/formats.html for more info on the egg format
mkdir "${name}.egg-info"
cd "${name}.egg-info"
ln -s ${pkgInfoFile} PKG-INFO
${lib.optionalString (pyProject.tool.poetry ? plugins) ''
ln -s ${entryPointsFile} entry_points.txt
''}
''
);
in
editablePackage

View file

@ -0,0 +1,15 @@
[
"egg",
"tar",
"tar.bz2",
"tar.gz",
"tar.lz",
"tar.lzma",
"tar.xz",
"tbz",
"tgz",
"tlz",
"txz",
"whl",
"zip"
]

View file

@ -0,0 +1,24 @@
source $stdenv/setup
set -euo pipefail
curl="curl \
--location \
--max-redirs 20 \
--retry 2 \
--disable-epsv \
--cookie-jar cookies \
--insecure \
--speed-time 5 \
--progress-bar \
--fail \
$curlOpts \
$NIX_CURL_FLAGS"
echo "Trying to fetch with predicted URL: $predictedURL"
$curl $predictedURL --output $out && exit 0
echo "Predicted URL '$predictedURL' failed, querying pypi.org"
$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url
url=$(cat url)
$curl -k $url --output $out

View file

@ -0,0 +1,119 @@
# Some repositories (such as Devpi) expose the Pypi legacy API
# (https://warehouse.pypa.io/api-reference/legacy.html).
#
# Note it is not possible to use pip
# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12
import os
import sys
import netrc
from urllib.parse import urlparse, urlunparse
from html.parser import HTMLParser
import urllib.request
import shutil
import ssl
from os.path import normpath
# Parse the legacy index page to extract the href and package names
class Pep503(HTMLParser):
def __init__(self):
super().__init__()
self.sources = {}
self.url = None
self.name = None
def handle_data(self, data):
if self.url is not None:
self.name = data
def handle_starttag(self, tag, attrs):
if tag == "a":
for name, value in attrs:
if name == "href":
self.url = value
def handle_endtag(self, tag):
if self.url is not None:
self.sources[self.name] = self.url
self.url = None
url = sys.argv[1]
package_name = sys.argv[2]
index_url = url + "/" + package_name + "/"
package_filename = sys.argv[3]
# Parse username and password for this host from the netrc file if given.
username, password = None, None
if os.environ["NETRC"]:
netrc_obj = netrc.netrc(os.environ["NETRC"])
host = urlparse(index_url).netloc
# Strip port number if present
if ":" in host:
host = host.split(":")[0]
username, _, password = netrc_obj.authenticators(host)
print("Reading index %s" % index_url)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
req = urllib.request.Request(index_url)
if username and password:
import base64
password_b64 = base64.b64encode(":".join((username, password)).encode()).decode(
"utf-8"
)
req.add_header("Authorization", "Basic {}".format(password_b64))
response = urllib.request.urlopen(req, context=context)
index = response.read()
parser = Pep503()
parser.feed(str(index))
if package_filename not in parser.sources:
print(
"The file %s has not be found in the index %s" % (package_filename, index_url)
)
exit(1)
package_file = open(package_filename, "wb")
# Sometimes the href is a relative path
if urlparse(parser.sources[package_filename]).netloc == "":
parsed_url = urlparse(index_url)
package_url = urlunparse(
(
parsed_url.scheme,
parsed_url.netloc,
parsed_url.path + "/" + parser.sources[package_filename],
None,
None,
None,
)
)
else:
package_url = parser.sources[package_filename]
# Handle urls containing "../"
parsed_url = urlparse(package_url)
real_package_url = urlunparse(
(
parsed_url.scheme,
parsed_url.netloc,
normpath(parsed_url.path),
parsed_url.params,
parsed_url.query,
parsed_url.fragment,
)
)
print("Downloading %s" % real_package_url)
req = urllib.request.Request(real_package_url)
if username and password:
req.add_unredirected_header("Authorization", "Basic {}".format(password_b64))
response = urllib.request.urlopen(req, context=context)
with response as r:
shutil.copyfileobj(r, package_file)

View file

@ -0,0 +1,90 @@
{ python
, buildPackages
, makeSetupHook
, wheel
, pip
, pkgs
}:
let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
nonOverlayedPython = pkgs.python3.pythonForBuild.withPackages (ps: [ ps.tomlkit ]);
makeRemoveSpecialDependenciesHook = { fields, kind }:
nonOverlayedPython.pkgs.callPackage
(
{}:
makeSetupHook
{
name = "remove-path-dependencies.sh";
deps = [ ];
substitutions = {
# NOTE: We have to use a non-overlayed Python here because otherwise we run into an infinite recursion
# because building of tomlkit and its dependencies also use these hooks.
pythonPath = nonOverlayedPython.pkgs.makePythonPath [ nonOverlayedPython ];
pythonInterpreter = nonOverlayedPython.interpreter;
pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
fields = fields;
kind = kind;
};
} ./remove-special-dependencies.sh
)
{ };
in
{
removePathDependenciesHook = makeRemoveSpecialDependenciesHook {
fields = [ "path" ];
kind = "path";
};
removeGitDependenciesHook = makeRemoveSpecialDependenciesHook {
fields = [ "git" "branch" "rev" "tag" ];
kind = "git";
};
pipBuildHook = callPackage
(
{ pip, wheel }:
makeSetupHook
{
name = "pip-build-hook.sh";
deps = [ pip wheel ];
substitutions = {
inherit pythonInterpreter pythonSitePackages;
};
} ./pip-build-hook.sh
)
{ };
poetry2nixFixupHook = callPackage
(
{}:
makeSetupHook
{
name = "fixup-hook.sh";
deps = [ ];
substitutions = {
inherit pythonSitePackages;
filenames = builtins.concatStringsSep " " [
"pyproject.toml"
"README.md"
];
};
} ./fixup-hook.sh
)
{ };
# When the "wheel" package itself is a wheel the nixpkgs hook (which pulls in "wheel") leads to infinite recursion
# It doesn't _really_ depend on wheel though, it just copies the wheel.
wheelUnpackHook = callPackage
({}:
makeSetupHook
{
name = "wheel-unpack-hook.sh";
deps = [ ];
} ./wheel-unpack-hook.sh
)
{ };
}

View file

@ -0,0 +1,20 @@
poetry2nix-fixup-hook() {
# Including tests in the output is a common mistake
if [ -z "${dontFixupTests-}" ]; then
rm -rf $out/@pythonSitePackages@/tests
fi
# Including files in site-packages is a common packaging mistake
#
# While we cannot remove all normal files dumped in site-packages
# we can clean up some common mistakes
if [ -z "${dontFixupSitePackages-}" ]; then
for f in @filenames@; do
rm -f $out/@pythonSitePackages@/$f
done
fi
}
postFixupHooks+=(poetry2nix-fixup-hook)

View file

@ -0,0 +1,42 @@
# Setup hook to use for pip projects
echo "Sourcing pip-build-hook"
pipBuildPhase() {
echo "Executing pipBuildPhase"
runHook preBuild
mkdir -p dist
echo "Creating a wheel..."
@pythonInterpreter@ -m pip wheel --verbose --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist .
echo "Finished creating a wheel..."
runHook postBuild
echo "Finished executing pipBuildPhase"
}
pipShellHook() {
echo "Executing pipShellHook"
runHook preShellHook
# Long-term setup.py should be dropped.
if [ -e pyproject.toml ]; then
tmp_path=$(mktemp -d)
export PATH="$tmp_path/bin:$PATH"
export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH"
mkdir -p "$tmp_path/@pythonSitePackages@"
@pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2
fi
runHook postShellHook
echo "Finished executing pipShellHook"
}
if [ -z "${dontUsePipBuild-}" ] && [ -z "${buildPhase-}" ]; then
echo "Using pipBuildPhase"
buildPhase=pipBuildPhase
fi
if [ -z "${shellHook-}" ]; then
echo "Using pipShellHook"
shellHook=pipShellHook
fi

View file

@ -0,0 +1,53 @@
#!/usr/bin/env python
# Patch out special dependencies (git and path) from a pyproject.toml file
import argparse
import sys
import tomlkit
def main(input, output, fields_to_remove):
data = tomlkit.loads(input.read())
try:
deps = data["tool"]["poetry"]["dependencies"]
except KeyError:
pass
else:
for dep in deps.values():
if isinstance(dep, dict):
any_removed = False
for field in fields_to_remove:
any_removed |= dep.pop(field, None) is not None
if any_removed:
dep["version"] = "*"
output.write(tomlkit.dumps(data))
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument(
"-i",
"--input",
type=argparse.FileType("r"),
default=sys.stdin,
help="Location from which to read input TOML",
)
p.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
default=sys.stdout,
help="Location to write output TOML",
)
p.add_argument(
"-f",
"--fields-to-remove",
nargs="+",
help="The fields to remove from the dependency's TOML",
)
args = p.parse_args()
main(args.input, args.output, args.fields_to_remove)

View file

@ -0,0 +1,23 @@
remove-@kind@-dependencies-hook() {
# Tell poetry not to resolve special dependencies. Any version is fine!
if ! test -f pyproject.toml; then
return
fi
echo "Removing @kind@ dependencies"
# NOTE: We have to reset PYTHONPATH to avoid having propagatedBuildInputs
# from the currently building derivation leaking into our unrelated Python
# environment.
PYTHONPATH=@pythonPath@ \
@pythonInterpreter@ \
@pyprojectPatchScript@ \
--fields-to-remove @fields@ < pyproject.toml > pyproject.formatted.toml
mv pyproject.formatted.toml pyproject.toml
echo "Finished removing @kind@ dependencies"
}
postPatchHooks+=(remove-@kind@-dependencies-hook)

View file

@ -0,0 +1,18 @@
# Setup hook to use in case a wheel is fetched
echo "Sourcing wheel setup hook"
wheelUnpackPhase(){
echo "Executing wheelUnpackPhase"
runHook preUnpack
mkdir -p dist
cp "$src" "dist/$(stripHash "$src")"
# runHook postUnpack # Calls find...?
echo "Finished executing wheelUnpackPhase"
}
if [ -z "${dontUseWheelUnpack-}" ] && [ -z "${unpackPhase-}" ]; then
echo "Using wheelUnpackPhase"
unpackPhase=wheelUnpackPhase
fi

View file

@ -0,0 +1,240 @@
{ lib, pkgs, stdenv }:
let
inherit (import ./semver.nix { inherit lib ireplace; }) satisfiesSemver;
inherit (builtins) genList length;
# Replace a list entry at defined index with set value
ireplace = idx: value: list: (
genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
);
# Do some canonicalisation of module names
moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name);
# Get a full semver pythonVersion from a python derivation
getPythonVersion = python:
let
pyVer = lib.splitVersion python.pythonVersion ++ [ "0" ];
ver = lib.splitVersion python.version;
major = l: lib.elemAt l 0;
minor = l: lib.elemAt l 1;
joinVersion = v: lib.concatStringsSep "." v;
in
joinVersion (if major pyVer == major ver && minor pyVer == minor ver then ver else pyVer);
# Compare a semver expression with a version
isCompatible = version:
let
operators = {
"||" = cond1: cond2: cond1 || cond2;
"," = cond1: cond2: cond1 && cond2; # , means &&
"&&" = cond1: cond2: cond1 && cond2;
};
splitRe = "(" + (builtins.concatStringsSep "|" (builtins.map (x: lib.replaceStrings [ "|" ] [ "\\|" ] x) (lib.attrNames operators))) + ")";
in
expr:
let
tokens = builtins.filter (x: x != "") (builtins.split splitRe expr);
combine = acc: v:
let
isOperator = builtins.typeOf v == "list";
operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
in
if isOperator then (acc // { inherit operator; }) else {
inherit operator;
state = operators."${operator}" acc.state (satisfiesSemver version v);
};
initial = { operator = "&&"; state = true; };
in
if expr == "" then true else (builtins.foldl' combine initial tokens).state;
fromTOML = builtins.fromTOML or
(
toml: builtins.fromJSON (
builtins.readFile (
pkgs.runCommand "from-toml"
{
inherit toml;
allowSubstitutes = false;
preferLocalBuild = true;
}
''
${pkgs.remarshal}/bin/remarshal \
-if toml \
-i <(echo "$toml") \
-of json \
-o $out
''
)
)
);
readTOML = path: fromTOML (builtins.readFile path);
#
# Returns the appropriate manylinux dependencies and string representation for the file specified
#
getManyLinuxDeps = f:
let
ml = pkgs.pythonManylinuxPackages;
in
if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; }
else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; }
else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
else if lib.strings.hasInfix "manylinux_" f then { pkg = [ ml.manylinux2014 ]; str = "pep600"; }
else { pkg = [ ]; str = null; };
# Predict URL from the PyPI index.
# Args:
# pname: package name
# file: filename including extension
# hash: SRI hash
# kind: Language implementation and version tag
predictURLFromPypi = lib.makeOverridable (
{ pname, file, hash, kind }:
"https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"
);
# Fetch from the PyPI index.
# At first we try to fetch the predicated URL but if that fails we
# will use the Pypi API to determine the correct URL.
# Args:
# pname: package name
# file: filename including extension
# version: the version string of the dependency
# hash: SRI hash
# kind: Language implementation and version tag
fetchFromPypi = lib.makeOverridable (
{ pname, file, version, hash, kind, curlOpts ? "" }:
let
predictedURL = predictURLFromPypi { inherit pname file hash kind; };
in
(pkgs.stdenvNoCC.mkDerivation {
name = file;
nativeBuildInputs = [
pkgs.buildPackages.curl
pkgs.buildPackages.jq
];
isWheel = lib.strings.hasSuffix "whl" file;
system = "builtin";
preferLocalBuild = true;
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
"NIX_CURL_FLAGS"
];
inherit pname file version curlOpts predictedURL;
builder = ./fetch-from-pypi.sh;
outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;
passthru = {
urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl
};
})
);
fetchFromLegacy = lib.makeOverridable (
{ python, pname, url, file, hash }:
let
pathParts =
(builtins.filter
({ prefix, path }: "NETRC" == prefix)
builtins.nixPath);
netrc_file = if (pathParts != [ ]) then (builtins.head pathParts).path else "";
in
pkgs.runCommand file
{
nativeBuildInputs = [ python ];
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;
NETRC = netrc_file;
} ''
python ${./fetch_from_legacy.py} ${url} ${pname} ${file}
mv ${file} $out
''
);
getBuildSystemPkgs =
{ pythonPackages
, pyProject
}:
let
missingBuildBackendError = "No build-system.build-backend section in pyproject.toml. "
+ "Add such a section as described in https://python-poetry.org/docs/pyproject/#poetry-and-pep-517";
requires = lib.attrByPath [ "build-system" "requires" ] (throw missingBuildBackendError) pyProject;
requiredPkgs = builtins.map (n: lib.elemAt (builtins.match "([^!=<>~[]+).*" n) 0) requires;
in
builtins.map (drvAttr: pythonPackages.${drvAttr} or (throw "unsupported build system requirement ${drvAttr}")) requiredPkgs;
# Find gitignore files recursively in parent directory stopping with .git
findGitIgnores = path:
let
parent = path + "/..";
gitIgnore = path + "/.gitignore";
isGitRoot = builtins.pathExists (path + "/.git");
hasGitIgnore = builtins.pathExists gitIgnore;
gitIgnores = if hasGitIgnore then [ gitIgnore ] else [ ];
in
lib.optionals (builtins.pathExists path && builtins.toString path != "/" && ! isGitRoot) (findGitIgnores parent) ++ gitIgnores;
/*
Provides a source filtering mechanism that:
- Filters gitignore's
- Filters pycache/pyc files
- Uses cleanSourceFilter to filter out .git/.hg, .o/.so, editor backup files & nix result symlinks
*/
cleanPythonSources = { src }:
let
gitIgnores = findGitIgnores src;
pycacheFilter = name: type:
(type == "directory" && ! lib.strings.hasInfix "__pycache__" name)
|| (type == "regular" && ! lib.strings.hasSuffix ".pyc" name)
;
in
lib.cleanSourceWith {
filter = lib.cleanSourceFilter;
src = lib.cleanSourceWith {
filter = pkgs.nix-gitignore.gitignoreFilterPure pycacheFilter gitIgnores src;
inherit src;
};
};
# Maps Nixpkgs CPU values to target machines known to be supported for manylinux* wheels.
# (a.k.a. `uname -m` output from CentOS 7)
#
# This is current as of manylinux2014 (PEP-0599), and is a superset of manylinux2010 / manylinux1.
# s390x is not supported in Nixpkgs, so we don't map it.
manyLinuxTargetMachines = {
x86_64 = "x86_64";
i686 = "i686";
aarch64 = "aarch64";
armv7l = "armv7l";
powerpc64 = "ppc64";
powerpc64le = "ppc64le";
};
# Machine tag for our target platform (if available)
getTargetMachine = stdenv: manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name} or null;
in
{
inherit
fetchFromPypi
fetchFromLegacy
getManyLinuxDeps
isCompatible
readTOML
getBuildSystemPkgs
satisfiesSemver
cleanPythonSources
moduleName
getPythonVersion
getTargetMachine
;
}

View file

@ -0,0 +1,207 @@
{ autoPatchelfHook
, lib
, python
, buildPythonPackage
, poetryLib
, evalPep508
}:
{ name
, version
, files
, source
, dependencies ? { }
, pythonPackages
, python-versions
, pwd
, sourceSpec
, supportedExtensions ? lib.importJSON ./extensions.json
, preferWheels ? false
, ...
}:
pythonPackages.callPackage
(
{ preferWheel ? preferWheels
, ...
}@args:
let
inherit (python) stdenv;
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi moduleName;
inherit (import ./pep425.nix {
inherit lib poetryLib python stdenv;
}) selectWheel
;
fileCandidates =
let
supportedRegex = ("^.*(" + builtins.concatStringsSep "|" supportedExtensions + ")");
matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." "+" ] [ "\\." "\\+" ] version + ".*$") fname != null;
hasSupportedExtension = fname: builtins.match supportedRegex fname != null;
isCompatibleEgg = fname: ! lib.strings.hasSuffix ".egg" fname || lib.strings.hasSuffix "py${python.pythonVersion}.egg" fname;
in
builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file && isCompatibleEgg f.file) files;
toPath = s: pwd + "/${s}";
isSource = source != null;
isGit = isSource && source.type == "git";
isUrl = isSource && source.type == "url";
isDirectory = isSource && source.type == "directory";
isFile = isSource && source.type == "file";
isLegacy = isSource && source.type == "legacy";
localDepPath = toPath source.url;
buildSystemPkgs =
let
pyProjectPath = localDepPath + "/pyproject.toml";
pyProject = poetryLib.readTOML pyProjectPath;
in
if builtins.pathExists pyProjectPath then
poetryLib.getBuildSystemPkgs
{
inherit pythonPackages pyProject;
} else [ ];
fileInfo =
let
isBdist = f: lib.strings.hasSuffix "whl" f.file;
isSdist = f: ! isBdist f && ! isEgg f;
isEgg = f: lib.strings.hasSuffix ".egg" f.file;
binaryDist = selectWheel fileCandidates;
sourceDist = builtins.filter isSdist fileCandidates;
eggs = builtins.filter isEgg fileCandidates;
entries = (if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs;
lockFileEntry = (
if lib.length entries > 0 then builtins.head entries
else throw "Missing suitable source/wheel file entry for ${name}"
);
_isEgg = isEgg lockFileEntry;
in
rec {
inherit (lockFileEntry) file hash;
name = file;
format =
if _isEgg then "egg"
else if lib.strings.hasSuffix ".whl" name then "wheel"
else "pyproject";
kind =
if _isEgg then python.pythonVersion
else if format == "pyproject" then "source"
else (builtins.elemAt (lib.strings.splitString "-" name) 2);
};
# Prevent infinite recursion
skipSetupToolsSCM = [
"setuptools_scm"
"setuptools-scm"
"toml" # Toml is an extra for setuptools-scm
"tomli" # tomli is an extra for later versions of setuptools-scm
"flit-core"
"packaging"
"six"
"pyparsing"
];
baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
format = if isDirectory || isGit || isUrl then "pyproject" else fileInfo.format;
in
buildPythonPackage {
pname = moduleName name;
version = version;
inherit format;
doCheck = false; # We never get development deps
# Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned`
dontStrip = format == "wheel";
nativeBuildInputs = [
pythonPackages.poetry2nixFixupHook
]
++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook
++ lib.optionals (format == "pyproject") [
pythonPackages.removePathDependenciesHook
pythonPackages.removeGitDependenciesHook
];
buildInputs = (
baseBuildInputs
++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
++ lib.optional isDirectory buildSystemPkgs
);
propagatedBuildInputs =
let
compat = isCompatible (poetryLib.getPythonVersion python);
deps = lib.filterAttrs
(n: v: v)
(
lib.mapAttrs
(
n: v:
let
constraints = v.python or "";
pep508Markers = v.markers or "";
in
compat constraints && evalPep508 pep508Markers
)
dependencies
);
depAttrs = lib.attrNames deps;
in
builtins.map (n: pythonPackages.${moduleName n}) depAttrs;
meta = {
broken = ! isCompatible (poetryLib.getPythonVersion python) python-versions;
license = [ ];
inherit (python.meta) platforms;
};
passthru = {
inherit args;
};
# We need to retrieve kind from the interpreter and the filename of the package
# Interpreters should declare what wheel types they're compatible with (python type + ABI)
# Here we can then choose a file based on that info.
src =
if isGit then
(
builtins.fetchGit ({
inherit (source) url;
rev = source.resolved_reference or source.reference;
ref = sourceSpec.branch or (if sourceSpec ? tag then "refs/tags/${sourceSpec.tag}" else "HEAD");
} // (
let
nixVersion = builtins.substring 0 3 builtins.nixVersion;
in
lib.optionalAttrs ((sourceSpec ? rev) && (lib.versionAtLeast nixVersion "2.4")) {
allRefs = true;
}
))
)
else if isUrl then
builtins.fetchTarball
{
inherit (source) url;
}
else if isDirectory then
(poetryLib.cleanPythonSources { src = localDepPath; })
else if isFile then
localDepPath
else if isLegacy then
fetchFromLegacy
{
pname = name;
inherit python;
inherit (fileInfo) file hash;
inherit (source) url;
}
else
fetchFromPypi {
pname = name;
inherit (fileInfo) file hash kind;
inherit version;
};
}
)
{ }

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,46 @@
"""
Rewrite libc/library path references to Nix store paths
Nixpkgs uses a normal patch for this but we need to be less
sensitive to changes between versions.
"""
from textwrap import dedent
import sys
import ast
import os
with open(sys.argv[1]) as f:
mod = ast.parse(f.read(), "geos.py")
class LibTransformer(ast.NodeTransformer):
_lgeos_replaced = False
def visit_If(self, node):
if ast.unparse(node).startswith("if sys.platform.startswith('linux')"):
return ast.parse(
dedent(
"""
free = CDLL(%s).free
free.argtypes = [c_void_p]
free.restype = None
"""
)
% (lambda x: "'" + x + "'" if x else None)(os.environ.get("GEOS_LIBC"))
)
return node
def visit_Assign(self, node):
_target = node.targets[0]
if (
not self._lgeos_replaced
and isinstance(_target, ast.Name)
and _target.id == "_lgeos"
):
self._lgeos_replaced = True
return ast.parse("_lgeos = CDLL('%s')" % os.environ["GEOS_LIBRARY_PATH"])
return node
with open(sys.argv[1], "w") as f:
f.write(ast.unparse(LibTransformer().visit(mod)))

View file

@ -0,0 +1,117 @@
{ lib, stdenv, poetryLib, python, isLinux ? stdenv.isLinux }:
let
inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removePrefix removeSuffix;
targetMachine = poetryLib.getTargetMachine stdenv;
# The 'cpxy" as determined by `python.version`
#
# e.g "2.7.17" -> "cp27"
# "3.5.9" -> "cp35"
pythonTag =
let
ver = builtins.splitVersion python.version;
major = builtins.elemAt ver 0;
minor = builtins.elemAt ver 1;
in
"cp${major}${minor}";
abiTag = "${pythonTag}m";
#
# Parses wheel file returning an attribute set
#
toWheelAttrs = str:
let
entries' = splitString "-" str;
# Hack: Remove version "suffixes" like 2.11.4-1
# Some wheels have build tag with more than one digit
# like openvino-2022.1.0-7019-cp36-cp36m-manylinux_2_27_x86_64.whl
entries = builtins.filter (x: builtins.match "[0-9]*" x == null) entries';
p = removeSuffix ".whl" (builtins.elemAt entries 4);
in
{
pkgName = builtins.elemAt entries 0;
pkgVer = builtins.elemAt entries 1;
pyVer = builtins.elemAt entries 2;
abi = builtins.elemAt entries 3;
platform = p;
};
#
# Builds list of acceptable osx wheel files
#
# <versions> accepted versions in descending order of preference
# <candidates> list of wheel files to select from
findBestMatches = versions: candidates:
let
v = lib.lists.head versions;
vs = lib.lists.tail versions;
in
if (builtins.length versions == 0)
then [ ]
else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates);
# pyver = "cpXX"
# x = "cpXX" | "py2" | "py3" | "py2.py3"
isPyVersionCompatible = pyver: x:
let
normalize = y: ''cp${removePrefix "cp" (removePrefix "py" y)}'';
isCompat = p: x: hasPrefix (normalize x) p;
in
lib.lists.any (isCompat pyver) (splitString "." x);
#
# Selects the best matching wheel file from a list of files
#
selectWheel = files:
let
filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files);
isPyAbiCompatible = pyabi: x: x == "none" || hasPrefix pyabi x || hasPrefix x pyabi || (
# The CPython stable ABI is abi3 as in the shared library suffix.
python.passthru.implementation == "cpython" &&
builtins.elemAt (lib.splitString "." python.version) 0 == "3" &&
x == "abi3"
);
withPython = ver: abi: x: (isPyVersionCompatible ver x.pyVer) && (isPyAbiCompatible abi x.abi);
withPlatform =
if isLinux
then
if targetMachine != null
then
# See PEP 600 for details.
(p:
builtins.match "any|manylinux(1|2010|2014)_${escapeRegex targetMachine}|manylinux_[0-9]+_[0-9]+_${escapeRegex targetMachine}" p != null
)
else
(p: p == "any")
else
if stdenv.isDarwin
then
if stdenv.targetPlatform.isAarch64
then (p: p == "any" || (hasInfix "macosx" p && lib.lists.any (e: hasSuffix e p) [ "arm64" "aarch64" ]))
else (p: p == "any" || (hasInfix "macosx" p && hasSuffix "x86_64" p))
else (p: p == "any");
withPlatforms = x: lib.lists.any withPlatform (splitString "." x.platform);
filterWheel = x:
let
f = toWheelAttrs x.file;
in
(withPython pythonTag abiTag f) && (withPlatforms f);
filtered = builtins.filter filterWheel filesWithoutSources;
choose = files:
let
osxMatches = [ "12_0" "11_0" "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "manylinux_" "any" ];
chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x);
chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);
in
if isLinux
then chooseLinux files
else chooseOSX files;
in
if (builtins.length filtered == 0)
then [ ]
else choose (filtered);
in
{
inherit selectWheel toWheelAttrs isPyVersionCompatible;
}

View file

@ -0,0 +1,258 @@
{ lib, stdenv, poetryLib }: python:
let
inherit (poetryLib) ireplace;
targetMachine = poetryLib.getTargetMachine stdenv;
# Like builtins.substring but with stop being offset instead of length
substr = start: stop: s: builtins.substring start (stop - start) s;
# Strip leading/trailing whitespace from string
stripStr = s: lib.elemAt (builtins.split "^ *" (lib.elemAt (builtins.split " *$" s) 0)) 2;
findSubExpressionsFun = acc: c: (
if c == "(" then
(
let
posNew = acc.pos + 1;
isOpen = acc.openP == 0;
startPos = if isOpen then posNew else acc.startPos;
in
acc // {
inherit startPos;
exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ];
pos = posNew;
openP = acc.openP + 1;
}
) else if c == ")" then
(
let
openP = acc.openP - 1;
exprs = findSubExpressions (substr acc.startPos acc.pos acc.expr);
in
acc // {
inherit openP;
pos = acc.pos + 1;
exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs;
exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos;
}
) else acc // { pos = acc.pos + 1; }
);
# Make a tree out of expression groups (parens)
findSubExpressions = expr':
let
expr = " " + expr';
acc = builtins.foldl'
findSubExpressionsFun
{
exprs = [ ];
expr = expr;
pos = 0;
openP = 0;
exprPos = 0;
startPos = 0;
}
(lib.stringToCharacters expr);
tailExpr = (substr acc.exprPos acc.pos expr);
tailExprs = if tailExpr != "" then [ tailExpr ] else [ ];
in
acc.exprs ++ tailExprs;
parseExpressions = exprs:
let
splitCond = (
s: builtins.map
(x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x))
(builtins.split " (and|or) " (s + " "))
);
mapfn = expr: (
if (builtins.match "^ ?$" expr != null) then null # Filter empty
else if (builtins.elem expr [ "and" "or" ]) then {
type = "bool";
value = expr;
}
else {
type = "expr";
value = expr;
}
);
parse = expr: builtins.filter (x: x != null) (builtins.map mapfn (splitCond expr));
in
builtins.foldl'
(
acc: v: acc ++ (if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ])
) [ ]
exprs;
# Transform individual expressions to structured expressions
# This function also performs variable substitution, replacing environment markers with their explicit values
transformExpressions = exprs:
let
variables = {
os_name = (
if python.pname == "jython" then "java"
else "posix"
);
sys_platform = (
if stdenv.isLinux then "linux"
else if stdenv.isDarwin then "darwin"
else throw "Unsupported platform"
);
platform_machine = targetMachine;
platform_python_implementation =
let
impl = python.passthru.implementation;
in
(
if impl == "cpython" then "CPython"
else if impl == "pypy" then "PyPy"
else throw "Unsupported implementation ${impl}"
);
platform_release = ""; # Field not reproducible
platform_system = (
if stdenv.isLinux then "Linux"
else if stdenv.isDarwin then "Darwin"
else throw "Unsupported platform"
);
platform_version = ""; # Field not reproducible
python_version = python.passthru.pythonVersion;
python_full_version = python.version;
implementation_name = python.implementation;
implementation_version = python.version;
# extra = "";
};
substituteVar = value: if builtins.hasAttr value variables then (builtins.toJSON variables."${value}") else value;
processVar = value: builtins.foldl' (acc: v: v acc) value [
stripStr
substituteVar
];
in
if builtins.typeOf exprs == "set" then
(
if exprs.type == "expr" then
(
let
mVal = ''[a-zA-Z0-9\'"_\. \-]+'';
mOp = "in|[!=<>]+";
e = stripStr exprs.value;
m' = builtins.match ''^(${mVal}) +(${mOp}) *(${mVal})$'' e;
m = builtins.map stripStr (if m' != null then m' else builtins.match ''^(${mVal}) +(${mOp}) *(${mVal})$'' e);
m0 = processVar (builtins.elemAt m 0);
m2 = processVar (builtins.elemAt m 2);
in
{
type = "expr";
value = {
# HACK: We don't know extra at eval time, so we assume the expression is always true
op = if m0 == "extra" then "true" else builtins.elemAt m 1;
values = [ m0 m2 ];
};
}
) else exprs
) else builtins.map transformExpressions exprs;
# Recursively eval all expressions
evalExpressions = exprs:
let
unmarshal = v: (
# TODO: Handle single quoted values
if v == "True" then true
else if v == "False" then false
else builtins.fromJSON v
);
hasElem = needle: haystack: builtins.elem needle (builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " haystack));
op = {
"true" = x: y: true;
"<=" = x: y: op.">=" y x;
"<" = x: y: lib.versionOlder (unmarshal x) (unmarshal y);
"!=" = x: y: x != y;
"==" = x: y: x == y;
">=" = x: y: lib.versionAtLeast (unmarshal x) (unmarshal y);
">" = x: y: op."<" y x;
"~=" = v: c:
let
parts = builtins.splitVersion c;
pruned = lib.take ((builtins.length parts) - 1) parts;
upper = builtins.toString (
(lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
);
upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned);
in
op.">=" v c && op."<" v upperConstraint;
"===" = x: y: x == y;
"in" = x: y:
let
values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y));
in
builtins.elem (unmarshal x) values;
};
in
if builtins.typeOf exprs == "set" then
(
if exprs.type == "expr" then
(
let
expr = exprs;
result = (op."${expr.value.op}") (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1);
in
{
type = "value";
value = result;
}
) else exprs
) else builtins.map evalExpressions exprs;
# Now that we have performed an eval all that's left to do is to concat the graph into a single bool
reduceExpressions = exprs:
let
cond = {
"and" = x: y: x && y;
"or" = x: y: x || y;
};
reduceExpressionsFun = acc: v: (
if builtins.typeOf v == "set" then
(
if v.type == "value" then
(
acc // {
value = cond."${acc.cond}" acc.value v.value;
}
) else if v.type == "bool" then
(
acc // {
cond = v.value;
}
) else throw "Unsupported type"
) else if builtins.typeOf v == "list" then
(
let
ret = builtins.foldl'
reduceExpressionsFun
{
value = true;
cond = "and";
}
v;
in
acc // {
value = cond."${acc.cond}" acc.value ret.value;
}
) else throw "Unsupported type"
);
in
(
builtins.foldl'
reduceExpressionsFun
{
value = true;
cond = "and";
}
exprs
).value;
in
e: builtins.foldl' (acc: v: v acc) e [
findSubExpressions
parseExpressions
transformExpressions
evalExpressions
reduceExpressions
]

View file

@ -0,0 +1,54 @@
{ lib
, poetry2nix
, python
, fetchFromGitHub
, projectDir ? ./.
, pyproject ? projectDir + "/pyproject.toml"
, poetrylock ? projectDir + "/poetry.lock"
}:
poetry2nix.mkPoetryApplication {
inherit python;
inherit projectDir pyproject poetrylock;
src = fetchFromGitHub (lib.importJSON ./src.json);
# "Vendor" dependencies (for build-system support)
postPatch = ''
echo "import sys" >> poetry/__init__.py
for path in $propagatedBuildInputs; do
echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
done
'';
postInstall = ''
# Figure out the location of poetry.core
# As poetry.core is using the same root import name as the poetry package and the python module system wont look for the root
# in the separate second location we need to link poetry.core to poetry
ln -s $(python -c 'import poetry.core; import os.path; print(os.path.dirname(poetry.core.__file__))') $out/${python.sitePackages}/poetry/core
mkdir -p "$out/share/bash-completion/completions"
"$out/bin/poetry" completions bash > "$out/share/bash-completion/completions/poetry"
mkdir -p "$out/share/zsh/vendor-completions"
"$out/bin/poetry" completions zsh > "$out/share/zsh/vendor-completions/_poetry"
mkdir -p "$out/share/fish/vendor_completions.d"
"$out/bin/poetry" completions fish > "$out/share/fish/vendor_completions.d/poetry.fish"
'';
# Propagating dependencies leads to issues downstream
# We've already patched poetry to prefer "vendored" dependencies
postFixup = ''
rm $out/nix-support/propagated-build-inputs
'';
# Fails because of impurities (network, git etc etc)
doCheck = false;
meta = with lib; {
inherit (python.meta) platforms;
maintainers = with maintainers; [ adisbladis jakewaksbaum ];
};
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,123 @@
[tool.poetry]
name = "poetry"
version = "1.1.13"
description = "Python dependency management and packaging made easy."
authors = [
"Sébastien Eustace <sebastien@eustace.io>"
]
license = "MIT"
readme = "README.md"
homepage = "https://python-poetry.org/"
repository = "https://github.com/python-poetry/poetry"
documentation = "https://python-poetry.org/docs"
keywords = ["packaging", "dependency", "poetry"]
classifiers = [
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules"
]
# Requirements
[tool.poetry.dependencies]
python = "^3.5"
poetry-core = "~1.0.7"
cleo = "^0.8.1"
clikit = "^0.6.2"
crashtest = { version = "^0.3.0", python = "^3.6" }
requests = "^2.18"
cachy = "^0.3.0"
requests-toolbelt = "^0.9.1"
cachecontrol = [
{ version = "^0.12.4", extras = ["filecache"], python = "<3.6" },
{ version = "^0.12.9", extras = ["filecache"], python = "^3.6" }
]
pkginfo = "^1.4"
html5lib = "^1.0"
shellingham = "^1.1"
tomlkit = ">=0.7.0,<1.0.0"
pexpect = "^4.7.0"
packaging = "^20.4"
virtualenv = { version = "^20.0.26" }
# The typing module is not in the stdlib in Python 2.7
typing = { version = "^3.6", python = "~2.7" }
# Use pathlib2 for Python 2.7
pathlib2 = { version = "^2.3", python = "~2.7" }
# Use futures on Python 2.7
futures = { version = "^3.3.0", python = "~2.7" }
# Use glob2 for Python 2.7 and 3.4
glob2 = { version = "^0.6", python = "~2.7" }
# functools32 is needed for Python 2.7
functools32 = { version = "^3.2.3", python = "~2.7" }
keyring = [
{ version = "^18.0.1", python = "~2.7" },
{ version = "^20.0.1", python = "~3.5" },
{ version = ">=21.2.0", python = "^3.6" }
]
# Use subprocess32 for Python 2.7
subprocess32 = { version = "^3.5", python = "~2.7" }
importlib-metadata = {version = "^1.6.0", python = "<3.8"}
[tool.poetry.dev-dependencies]
pytest = [
{version = "^4.1", python = "<3.5"},
{version = "^5.4.3", python = "~3.5"},
{version = "^6.2.5", python = ">=3.6"}
]
pytest-cov = "^2.5"
pytest-mock = "^1.9"
pre-commit = { version = "^2.6", python = "^3.6.1" }
tox = "^3.0"
pytest-sugar = "^0.9.2"
httpretty = "^1.0.3"
# We need to restrict the version of urllib3 to avoid
# httpretty breaking. This is fixed in httpretty >= 1.0.3
# but it's not compatible with Python 2.7 and 3.5.
urllib3 = "~1.26.9"
[tool.poetry.scripts]
poetry = "poetry.console:main"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.isort]
profile = "black"
force_single_line = true
atomic = true
include_trailing_comma = true
lines_after_imports = 2
lines_between_types = 1
use_parentheses = true
src_paths = ["poetry", "tests"]
skip_glob = ["*/setup.py"]
filter_files = true
known_first_party = "poetry"
[tool.black]
line-length = 88
include = '\.pyi?$'
exclude = '''
/(
\.eggs
| \.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
| tests/.*/setup.py
)/
'''

View file

@ -0,0 +1,7 @@
{
"owner": "python-poetry",
"repo": "poetry",
"rev": "1.1.13",
"sha256": "0zm73wcxjfad3841wl1xpx20drhynrqlxsjh1d1gsh618b7ldpib",
"fetchSubmodules": true
}

View file

@ -0,0 +1,70 @@
{ pkgs, lib }:
let
inherit (pkgs) stdenv;
mkPluginDrv =
{ self
, plugins
, drv
, postInstall ? ""
, nativeBuildInputs ? [ ]
, buildInputs ? [ ]
}:
let
env = self.python.withPackages (ps: plugins);
in
stdenv.mkDerivation {
pname = drv.pname + "-with-plugins";
inherit (drv) src version meta;
buildInputs = drv.buildInputs ++ drv.propagatedBuildInputs ++ buildInputs;
nativeBuildInputs = drv.nativeBuildInputs ++ nativeBuildInputs;
dontConfigure = true;
dontBuild = true;
dontUsePythonRecompileBytecode = true;
passthru = {
inherit (drv.passthru) withPlugins;
inherit plugins;
};
# Link bin/ from environment, but only if it's in a plugin
installPhase = ''
runHook preInstall
mkdir -p $out/bin
for bindir in ${lib.concatStringsSep " " (map (d: "${lib.getBin d}/bin") plugins)}; do
for bin in $bindir/*; do
ln -s ${env}/bin/$(basename $bin) $out/bin/
done
done
runHook postInstall
'';
inherit postInstall;
};
in
{
# Provide the `withPlugins` function
toPluginAble = self: { drv
, finalDrv
, postInstall ? ""
, nativeBuildInputs ? [ ]
, buildInputs ? [ ]
}: drv.overridePythonAttrs (old: {
passthru = old.passthru // {
withPlugins = pluginFn: mkPluginDrv {
plugins = [ finalDrv ] ++ pluginFn self;
inherit self postInstall nativeBuildInputs buildInputs;
drv = finalDrv;
};
};
});
}

View file

@ -0,0 +1,87 @@
{ lib, ireplace }:
let
inherit (builtins) elemAt match;
operators =
let
matchWildCard = s: match "([^*])(\\.[*])" s;
mkComparison = ret: version: v: builtins.compareVersions version v == ret;
mkIdxComparison = idx: version: v:
let
ver = builtins.splitVersion v;
minor = builtins.toString (lib.toInt (elemAt ver idx) + 1);
upper = builtins.concatStringsSep "." (ireplace idx minor ver);
in
operators.">=" version v && operators."<" version upper;
dropWildcardPrecision = f: version: constraint:
let
m = matchWildCard constraint;
hasWildcard = m != null;
c = if hasWildcard then (elemAt m 0) else constraint;
v =
if hasWildcard then (builtins.substring 0 (builtins.stringLength c) version)
else version;
in
f v c;
in
{
# Prefix operators
"==" = dropWildcardPrecision (mkComparison 0);
">" = dropWildcardPrecision (mkComparison 1);
"<" = dropWildcardPrecision (mkComparison (-1));
"!=" = v: c: ! operators."==" v c;
">=" = v: c: operators."==" v c || operators.">" v c;
"<=" = v: c: operators."==" v c || operators."<" v c;
# Semver specific operators
"~" = mkIdxComparison 1;
"^" = mkIdxComparison 0;
"~=" = v: c:
let
# Prune constraint
parts = builtins.splitVersion c;
pruned = lib.take ((builtins.length parts) - 1) parts;
upper = builtins.toString (
(lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
);
upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned);
in
operators.">=" v c && operators."<" v upperConstraint;
# Infix operators
"-" = version: v: operators.">=" version v.vl && operators."<=" version v.vu;
# Arbitrary equality clause, just run simple comparison
"===" = v: c: v == c;
#
};
re = {
operators = "([=><!~^]+)";
version = "([0-9.*x]+)";
};
parseConstraint = constraint:
let
constraintStr = builtins.replaceStrings [ " " ] [ "" ] constraint;
# The common prefix operators
mPre = match "${re.operators} *${re.version}" constraintStr;
# There is also an infix operator to match ranges
mIn = match "${re.version} *(-) *${re.version}" constraintStr;
in
(
if mPre != null then {
op = elemAt mPre 0;
v = elemAt mPre 1;
}
# Infix operators are range matches
else if mIn != null then {
op = elemAt mIn 1;
v = {
vl = (elemAt mIn 0);
vu = (elemAt mIn 2);
};
}
else throw "Constraint \"${constraintStr}\" could not be parsed"
);
satisfiesSemver = version: constraint:
let
inherit (parseConstraint constraint) op v;
in
if constraint == "*" then true else operators."${op}" version v;
in
{ inherit satisfiesSemver; }

View file

@ -0,0 +1,41 @@
{ lib
, scripts
, python
}:
let
mkScript = bin: entrypoint:
let
elem = builtins.elemAt (builtins.split ":" entrypoint);
module = elem 0;
fn = elem 2;
in
''
cat << EOF >> $out/bin/${bin}
#!${python.interpreter}
import sys
import re
# Insert "" to add CWD to import path
sys.path.insert(0, "")
from ${module} import ${fn}
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', "", sys.argv[0])
sys.exit(${fn}())
EOF
chmod +x $out/bin/${bin}
'';
in
python.pkgs.buildPythonPackage {
name = "poetry2nix-env-scripts";
dontUnpack = true;
dontUseSetuptoolsBuild = true;
dontConfigure = true;
dontUseSetuptoolsCheck = true;
installPhase = ''
mkdir -p $out/bin
${lib.concatStringsSep "\n" (lib.mapAttrsToList mkScript scripts)}
'';
}

View file

@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
pwd=$(pwd)
workdir=$(mktemp -d)
function cleanup {
cd "$pwd"
rm -rf $workdir
}
trap cleanup EXIT
cd "$workdir"
curl -L -s https://github.com/nix-community/poetry2nix/archive/master.tar.gz | tar -xz
mv poetry2nix-master/* .
mkdir build
cp *.* build/
cp -r overrides pkgs hooks bin build/
rm build/shell.nix build/generate.py build/overlay.nix build/flake.* build/check-fmt.nix build/pkgs/poetry/update*
cat > build/README.md << EOF
Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix
The update procedure is as-follows:
1. Send your change to the upstream poetry2nix repository
2. Get it approved with tests passing
3. Run the update script in pkgs/development/tools/poetry2nix
EOF
rm -rf "$pwd/poetry2nix"
mv build "$pwd/poetry2nix"