639 lines
22 KiB
Python
639 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import functools
|
|
import glob
|
|
import logging
|
|
import os
|
|
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
from typing import Any
|
|
from typing import Mapping
|
|
from typing import Sequence
|
|
|
|
import pkginfo
|
|
|
|
from poetry.core.factory import Factory
|
|
from poetry.core.packages.dependency import Dependency
|
|
from poetry.core.packages.package import Package
|
|
from poetry.core.pyproject.toml import PyProjectTOML
|
|
from poetry.core.utils.helpers import parse_requires
|
|
from poetry.core.utils.helpers import temporary_directory
|
|
from poetry.core.version.markers import InvalidMarker
|
|
from poetry.core.version.requirements import InvalidRequirement
|
|
|
|
from poetry.utils.env import EnvCommandError
|
|
from poetry.utils.env import ephemeral_environment
|
|
from poetry.utils.helpers import extractall
|
|
from poetry.utils.setup_reader import SetupReader
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Iterator
|
|
|
|
from packaging.metadata import RawMetadata
|
|
from packaging.utils import NormalizedName
|
|
from poetry.core.packages.project_package import ProjectPackage
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
PEP517_META_BUILD = """\
|
|
import build
|
|
import build.env
|
|
import pyproject_hooks
|
|
|
|
source = '{source}'
|
|
dest = '{dest}'
|
|
|
|
with build.env.DefaultIsolatedEnv() as env:
|
|
builder = build.ProjectBuilder.from_isolated_env(
|
|
env, source, runner=pyproject_hooks.quiet_subprocess_runner
|
|
)
|
|
env.install(builder.build_system_requires)
|
|
env.install(builder.get_requires_for_build('wheel'))
|
|
builder.metadata_path(dest)
|
|
"""
|
|
|
|
PEP517_META_BUILD_DEPS = ["build==1.0.3", "pyproject_hooks==1.0.0"]
|
|
|
|
|
|
class PackageInfoError(ValueError):
|
|
def __init__(self, path: Path, *reasons: BaseException | str) -> None:
|
|
reasons = (f"Unable to determine package info for path: {path!s}", *reasons)
|
|
super().__init__("\n\n".join(str(msg).strip() for msg in reasons if msg))
|
|
|
|
|
|
class PackageInfo:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
name: str | None = None,
|
|
version: str | None = None,
|
|
summary: str | None = None,
|
|
requires_dist: list[str] | None = None,
|
|
requires_python: str | None = None,
|
|
files: Sequence[Mapping[str, str]] | None = None,
|
|
yanked: str | bool = False,
|
|
cache_version: str | None = None,
|
|
) -> None:
|
|
self.name = name
|
|
self.version = version
|
|
self.summary = summary
|
|
self.requires_dist = requires_dist
|
|
self.requires_python = requires_python
|
|
self.files = files or []
|
|
self.yanked = yanked
|
|
self._cache_version = cache_version
|
|
self._source_type: str | None = None
|
|
self._source_url: str | None = None
|
|
self._source_reference: str | None = None
|
|
|
|
@property
|
|
def cache_version(self) -> str | None:
|
|
return self._cache_version
|
|
|
|
def update(self, other: PackageInfo) -> PackageInfo:
|
|
self.name = other.name or self.name
|
|
self.version = other.version or self.version
|
|
self.summary = other.summary or self.summary
|
|
self.requires_dist = other.requires_dist or self.requires_dist
|
|
self.requires_python = other.requires_python or self.requires_python
|
|
self.files = other.files or self.files
|
|
self._cache_version = other.cache_version or self._cache_version
|
|
return self
|
|
|
|
def asdict(self) -> dict[str, Any]:
|
|
"""
|
|
Helper method to convert package info into a dictionary used for caching.
|
|
"""
|
|
return {
|
|
"name": self.name,
|
|
"version": self.version,
|
|
"summary": self.summary,
|
|
"requires_dist": self.requires_dist,
|
|
"requires_python": self.requires_python,
|
|
"files": self.files,
|
|
"yanked": self.yanked,
|
|
"_cache_version": self._cache_version,
|
|
}
|
|
|
|
@classmethod
|
|
def load(cls, data: dict[str, Any]) -> PackageInfo:
|
|
"""
|
|
Helper method to load data from a dictionary produced by `PackageInfo.asdict()`.
|
|
|
|
:param data: Data to load. This is expected to be a `dict` object output by
|
|
`asdict()`.
|
|
"""
|
|
cache_version = data.pop("_cache_version", None)
|
|
return cls(cache_version=cache_version, **data)
|
|
|
|
def to_package(
|
|
self,
|
|
name: str | None = None,
|
|
extras: list[str] | None = None,
|
|
root_dir: Path | None = None,
|
|
) -> Package:
|
|
"""
|
|
Create a new `poetry.core.packages.package.Package` instance using metadata from
|
|
this instance.
|
|
|
|
:param name: Name to use for the package, if not specified name from this
|
|
instance is used.
|
|
:param extras: Extras to activate for this package.
|
|
:param root_dir: Optional root directory to use for the package. If set,
|
|
dependency strings will be parsed relative to this directory.
|
|
"""
|
|
name = name or self.name
|
|
|
|
if not name:
|
|
raise RuntimeError("Unable to create package with no name")
|
|
|
|
if not self.version:
|
|
# The version could not be determined, so we raise an error since it is
|
|
# mandatory.
|
|
raise RuntimeError(f"Unable to retrieve the package version for {name}")
|
|
|
|
package = Package(
|
|
name=name,
|
|
version=self.version,
|
|
source_type=self._source_type,
|
|
source_url=self._source_url,
|
|
source_reference=self._source_reference,
|
|
yanked=self.yanked,
|
|
)
|
|
if self.summary is not None:
|
|
package.description = self.summary
|
|
package.root_dir = root_dir
|
|
package.python_versions = self.requires_python or "*"
|
|
package.files = self.files
|
|
|
|
# If this is a local poetry project, we can extract "richer" requirement
|
|
# information, eg: development requirements etc.
|
|
if root_dir is not None:
|
|
path = root_dir
|
|
elif self._source_type == "directory" and self._source_url is not None:
|
|
path = Path(self._source_url)
|
|
else:
|
|
path = None
|
|
|
|
if path is not None:
|
|
poetry_package = self._get_poetry_package(path=path)
|
|
if poetry_package:
|
|
package.extras = poetry_package.extras
|
|
for dependency in poetry_package.requires:
|
|
package.add_dependency(dependency)
|
|
|
|
return package
|
|
|
|
seen_requirements = set()
|
|
|
|
package_extras: dict[NormalizedName, list[Dependency]] = {}
|
|
for req in self.requires_dist or []:
|
|
try:
|
|
# Attempt to parse the PEP-508 requirement string
|
|
dependency = Dependency.create_from_pep_508(req, relative_to=root_dir)
|
|
except InvalidMarker:
|
|
# Invalid marker, We strip the markers hoping for the best
|
|
logger.warning(
|
|
"Stripping invalid marker (%s) found in %s-%s dependencies",
|
|
req,
|
|
package.name,
|
|
package.version,
|
|
)
|
|
req = req.split(";")[0]
|
|
dependency = Dependency.create_from_pep_508(req, relative_to=root_dir)
|
|
except InvalidRequirement:
|
|
# Unable to parse requirement so we skip it
|
|
logger.warning(
|
|
"Invalid requirement (%s) found in %s-%s dependencies, skipping",
|
|
req,
|
|
package.name,
|
|
package.version,
|
|
)
|
|
continue
|
|
|
|
if dependency.in_extras:
|
|
# this dependency is required by an extra package
|
|
for extra in dependency.in_extras:
|
|
if extra not in package_extras:
|
|
# this is the first time we encounter this extra for this
|
|
# package
|
|
package_extras[extra] = []
|
|
|
|
package_extras[extra].append(dependency)
|
|
|
|
req = dependency.to_pep_508(with_extras=True)
|
|
|
|
if req not in seen_requirements:
|
|
package.add_dependency(dependency)
|
|
seen_requirements.add(req)
|
|
|
|
package.extras = package_extras
|
|
|
|
return package
|
|
|
|
@classmethod
|
|
def _from_distribution(
|
|
cls, dist: pkginfo.BDist | pkginfo.SDist | pkginfo.Wheel
|
|
) -> PackageInfo:
|
|
"""
|
|
Helper method to parse package information from a `pkginfo.Distribution`
|
|
instance.
|
|
|
|
:param dist: The distribution instance to parse information from.
|
|
"""
|
|
requirements = None
|
|
|
|
if dist.requires_dist:
|
|
requirements = list(dist.requires_dist)
|
|
else:
|
|
requires = Path(dist.filename) / "requires.txt"
|
|
if requires.exists():
|
|
text = requires.read_text(encoding="utf-8")
|
|
requirements = parse_requires(text)
|
|
|
|
info = cls(
|
|
name=dist.name,
|
|
version=dist.version,
|
|
summary=dist.summary,
|
|
requires_dist=requirements,
|
|
requires_python=dist.requires_python,
|
|
)
|
|
|
|
info._source_type = "file"
|
|
info._source_url = Path(dist.filename).resolve().as_posix()
|
|
|
|
return info
|
|
|
|
@classmethod
|
|
def _from_sdist_file(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Helper method to parse package information from an sdist file. We attempt to
|
|
first inspect the file using `pkginfo.SDist`. If this does not provide us with
|
|
package requirements, we extract the source and handle it as a directory.
|
|
|
|
:param path: The sdist file to parse information from.
|
|
"""
|
|
info = None
|
|
|
|
with contextlib.suppress(ValueError):
|
|
sdist = pkginfo.SDist(str(path))
|
|
info = cls._from_distribution(sdist)
|
|
|
|
if info is not None and info.requires_dist is not None:
|
|
# we successfully retrieved dependencies from sdist metadata
|
|
return info
|
|
|
|
# Still not dependencies found
|
|
# So, we unpack and introspect
|
|
suffix = path.suffix
|
|
zip = suffix == ".zip"
|
|
|
|
if suffix == ".bz2":
|
|
suffixes = path.suffixes
|
|
if len(suffixes) > 1 and suffixes[-2] == ".tar":
|
|
suffix = ".tar.bz2"
|
|
elif not zip:
|
|
suffix = ".tar.gz"
|
|
|
|
with temporary_directory() as tmp_str:
|
|
tmp = Path(tmp_str)
|
|
extractall(source=path, dest=tmp, zip=zip)
|
|
|
|
# a little bit of guess work to determine the directory we care about
|
|
elements = list(tmp.glob("*"))
|
|
|
|
if len(elements) == 1 and elements[0].is_dir():
|
|
sdist_dir = elements[0]
|
|
else:
|
|
sdist_dir = tmp / path.name.rstrip(suffix)
|
|
if not sdist_dir.is_dir():
|
|
sdist_dir = tmp
|
|
|
|
# now this is an unpacked directory we know how to deal with
|
|
new_info = cls.from_directory(path=sdist_dir)
|
|
new_info._source_type = "file"
|
|
new_info._source_url = path.resolve().as_posix()
|
|
|
|
if not info:
|
|
return new_info
|
|
|
|
return info.update(new_info)
|
|
|
|
@staticmethod
|
|
def has_setup_files(path: Path) -> bool:
|
|
return any((path / f).exists() for f in SetupReader.FILES)
|
|
|
|
@classmethod
|
|
def from_setup_files(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Mechanism to parse package information from a `setup.[py|cfg]` file. This uses
|
|
the implementation at `poetry.utils.setup_reader.SetupReader` in order to parse
|
|
the file. This is not reliable for complex setup files and should only attempted
|
|
as a fallback.
|
|
|
|
:param path: Path to `setup.py` file
|
|
"""
|
|
if not cls.has_setup_files(path):
|
|
raise PackageInfoError(
|
|
path, "No setup files (setup.py, setup.cfg) was found."
|
|
)
|
|
|
|
try:
|
|
result = SetupReader.read_from_directory(path)
|
|
except Exception as e:
|
|
raise PackageInfoError(path, e)
|
|
|
|
python_requires = result["python_requires"]
|
|
if python_requires is None:
|
|
python_requires = "*"
|
|
|
|
requires = "".join(dep + "\n" for dep in result["install_requires"])
|
|
if result["extras_require"]:
|
|
requires += "\n"
|
|
|
|
for extra_name, deps in result["extras_require"].items():
|
|
requires += f"[{extra_name}]\n"
|
|
|
|
for dep in deps:
|
|
requires += dep + "\n"
|
|
|
|
requires += "\n"
|
|
|
|
requirements = parse_requires(requires)
|
|
|
|
info = cls(
|
|
name=result.get("name"),
|
|
version=result.get("version"),
|
|
summary=result.get("description", ""),
|
|
requires_dist=requirements or None,
|
|
requires_python=python_requires,
|
|
)
|
|
|
|
if not (info.name and info.version) and not info.requires_dist:
|
|
# there is nothing useful here
|
|
raise PackageInfoError(
|
|
path,
|
|
"No core metadata (name, version, requires-dist) could be retrieved.",
|
|
)
|
|
|
|
return info
|
|
|
|
@staticmethod
|
|
def _find_dist_info(path: Path) -> Iterator[Path]:
|
|
"""
|
|
Discover all `*.*-info` directories in a given path.
|
|
|
|
:param path: Path to search.
|
|
"""
|
|
pattern = "**/*.*-info"
|
|
# Sometimes pathlib will fail on recursive symbolic links, so we need to work
|
|
# around it and use the glob module instead. Note that this does not happen with
|
|
# pathlib2 so it's safe to use it for Python < 3.4.
|
|
directories = glob.iglob(path.joinpath(pattern).as_posix(), recursive=True)
|
|
|
|
for d in directories:
|
|
yield Path(d)
|
|
|
|
@classmethod
|
|
def from_metadata(cls, metadata: RawMetadata) -> PackageInfo:
|
|
"""
|
|
Create package information from core metadata.
|
|
|
|
:param metadata: raw metadata
|
|
"""
|
|
return cls(
|
|
name=metadata.get("name"),
|
|
version=metadata.get("version"),
|
|
summary=metadata.get("summary"),
|
|
requires_dist=metadata.get("requires_dist"),
|
|
requires_python=metadata.get("requires_python"),
|
|
)
|
|
|
|
@classmethod
|
|
def from_metadata_directory(cls, path: Path) -> PackageInfo | None:
|
|
"""
|
|
Helper method to parse package information from an unpacked metadata directory.
|
|
|
|
:param path: The metadata directory to parse information from.
|
|
"""
|
|
if path.suffix in {".dist-info", ".egg-info"}:
|
|
directories = [path]
|
|
else:
|
|
directories = list(cls._find_dist_info(path=path))
|
|
|
|
dist: pkginfo.BDist | pkginfo.SDist | pkginfo.Wheel
|
|
for directory in directories:
|
|
try:
|
|
if directory.suffix == ".egg-info":
|
|
dist = pkginfo.UnpackedSDist(directory.as_posix())
|
|
elif directory.suffix == ".dist-info":
|
|
dist = pkginfo.Wheel(directory.as_posix())
|
|
else:
|
|
continue
|
|
break
|
|
except ValueError:
|
|
continue
|
|
else:
|
|
try:
|
|
# handle PKG-INFO in unpacked sdist root
|
|
dist = pkginfo.UnpackedSDist(path.as_posix())
|
|
except ValueError:
|
|
return None
|
|
|
|
return cls._from_distribution(dist=dist)
|
|
|
|
@classmethod
|
|
def from_package(cls, package: Package) -> PackageInfo:
|
|
"""
|
|
Helper method to inspect a `Package` object, in order to generate package info.
|
|
|
|
:param package: This must be a poetry package instance.
|
|
"""
|
|
requires = {dependency.to_pep_508() for dependency in package.requires}
|
|
|
|
for extra_requires in package.extras.values():
|
|
for dependency in extra_requires:
|
|
requires.add(dependency.to_pep_508())
|
|
|
|
return cls(
|
|
name=package.name,
|
|
version=str(package.version),
|
|
summary=package.description,
|
|
requires_dist=list(requires),
|
|
requires_python=package.python_versions,
|
|
files=package.files,
|
|
yanked=package.yanked_reason if package.yanked else False,
|
|
)
|
|
|
|
@staticmethod
|
|
def _get_poetry_package(path: Path) -> ProjectPackage | None:
|
|
# Note: we ignore any setup.py file at this step
|
|
# TODO: add support for handling non-poetry PEP-517 builds
|
|
if PyProjectTOML(path.joinpath("pyproject.toml")).is_poetry_project():
|
|
with contextlib.suppress(RuntimeError):
|
|
return Factory().create_poetry(path).package
|
|
|
|
return None
|
|
|
|
@classmethod
|
|
def from_directory(cls, path: Path, disable_build: bool = False) -> PackageInfo:
|
|
"""
|
|
Generate package information from a package source directory. If `disable_build`
|
|
is not `True` and introspection of all available metadata fails, the package is
|
|
attempted to be built in an isolated environment so as to generate required
|
|
metadata.
|
|
|
|
:param path: Path to generate package information from.
|
|
:param disable_build: If not `True` and setup reader fails, PEP 517 isolated
|
|
build is attempted in order to gather metadata.
|
|
"""
|
|
project_package = cls._get_poetry_package(path)
|
|
info: PackageInfo | None
|
|
if project_package:
|
|
info = cls.from_package(project_package)
|
|
else:
|
|
info = cls.from_metadata_directory(path)
|
|
|
|
if not info or info.requires_dist is None:
|
|
try:
|
|
if disable_build:
|
|
info = cls.from_setup_files(path)
|
|
else:
|
|
info = get_pep517_metadata(path)
|
|
except PackageInfoError:
|
|
if not info:
|
|
raise
|
|
|
|
# we discovered PkgInfo but no requirements were listed
|
|
|
|
info._source_type = "directory"
|
|
info._source_url = path.as_posix()
|
|
|
|
return info
|
|
|
|
@classmethod
|
|
def from_sdist(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Gather package information from an sdist file, packed or unpacked.
|
|
|
|
:param path: Path to an sdist file or unpacked directory.
|
|
"""
|
|
if path.is_file():
|
|
return cls._from_sdist_file(path=path)
|
|
|
|
# if we get here then it is neither an sdist instance nor a file
|
|
# so, we assume this is an directory
|
|
return cls.from_directory(path=path)
|
|
|
|
@classmethod
|
|
def from_wheel(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Gather package information from a wheel.
|
|
|
|
:param path: Path to wheel.
|
|
"""
|
|
try:
|
|
wheel = pkginfo.Wheel(str(path))
|
|
return cls._from_distribution(wheel)
|
|
except ValueError:
|
|
return PackageInfo()
|
|
|
|
@classmethod
|
|
def from_bdist(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Gather package information from a bdist (wheel etc.).
|
|
|
|
:param path: Path to bdist.
|
|
"""
|
|
if path.suffix == ".whl":
|
|
return cls.from_wheel(path=path)
|
|
|
|
try:
|
|
bdist = pkginfo.BDist(str(path))
|
|
return cls._from_distribution(bdist)
|
|
except ValueError as e:
|
|
raise PackageInfoError(path, e)
|
|
|
|
@classmethod
|
|
def from_path(cls, path: Path) -> PackageInfo:
|
|
"""
|
|
Gather package information from a given path (bdist, sdist, directory).
|
|
|
|
:param path: Path to inspect.
|
|
"""
|
|
try:
|
|
return cls.from_bdist(path=path)
|
|
except PackageInfoError:
|
|
return cls.from_sdist(path=path)
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def get_pep517_metadata(path: Path) -> PackageInfo:
|
|
"""
|
|
Helper method to use PEP-517 library to build and read package metadata.
|
|
|
|
:param path: Path to package source to build and read metadata for.
|
|
"""
|
|
info = None
|
|
|
|
with contextlib.suppress(PackageInfoError):
|
|
info = PackageInfo.from_setup_files(path)
|
|
if all([info.version, info.name, info.requires_dist]):
|
|
return info
|
|
|
|
with ephemeral_environment(
|
|
flags={"no-pip": False, "setuptools": "bundle", "wheel": "bundle"}
|
|
) as venv:
|
|
# TODO: cache PEP 517 build environment corresponding to each project venv
|
|
dest_dir = venv.path.parent / "dist"
|
|
dest_dir.mkdir()
|
|
|
|
pep517_meta_build_script = PEP517_META_BUILD.format(
|
|
source=path.as_posix(), dest=dest_dir.as_posix()
|
|
)
|
|
|
|
try:
|
|
venv.run_pip(
|
|
"install",
|
|
"--disable-pip-version-check",
|
|
"--ignore-installed",
|
|
"--no-input",
|
|
*PEP517_META_BUILD_DEPS,
|
|
)
|
|
venv.run_python_script(pep517_meta_build_script)
|
|
info = PackageInfo.from_metadata_directory(dest_dir)
|
|
except EnvCommandError as e:
|
|
# something went wrong while attempting pep517 metadata build
|
|
# fallback to egg_info if setup.py available
|
|
logger.debug("PEP517 build failed: %s", e)
|
|
setup_py = path / "setup.py"
|
|
if not setup_py.exists():
|
|
raise PackageInfoError(
|
|
path,
|
|
e,
|
|
"No fallback setup.py file was found to generate egg_info.",
|
|
)
|
|
|
|
cwd = Path.cwd()
|
|
os.chdir(path)
|
|
try:
|
|
venv.run("python", "setup.py", "egg_info")
|
|
info = PackageInfo.from_metadata_directory(path)
|
|
except EnvCommandError as fbe:
|
|
raise PackageInfoError(
|
|
path, e, "Fallback egg_info generation failed.", fbe
|
|
)
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
if info:
|
|
logger.debug("Falling back to parsed setup.py file for %s", path)
|
|
return info
|
|
|
|
# if we reach here, everything has failed and all hope is lost
|
|
raise PackageInfoError(path, "Exhausted all core metadata sources.")
|