__init__.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191
  1. """
  2. APIs exposing metadata from third-party Python packages.
  3. This codebase is shared between importlib.metadata in the stdlib
  4. and importlib_metadata in PyPI. See
  5. https://github.com/python/importlib_metadata/wiki/Development-Methodology
  6. for more detail.
  7. """
  8. from __future__ import annotations
  9. import abc
  10. import collections
  11. import email
  12. import functools
  13. import itertools
  14. import operator
  15. import os
  16. import pathlib
  17. import posixpath
  18. import re
  19. import sys
  20. import textwrap
  21. import types
  22. from collections.abc import Iterable, Mapping
  23. from contextlib import suppress
  24. from importlib import import_module
  25. from importlib.abc import MetaPathFinder
  26. from itertools import starmap
  27. from typing import Any
  28. from . import _meta
  29. from ._collections import FreezableDefaultDict, Pair
  30. from ._compat import (
  31. NullFinder,
  32. install,
  33. )
  34. from ._functools import method_cache, noop, pass_none, passthrough
  35. from ._itertools import always_iterable, bucket, unique_everseen
  36. from ._meta import PackageMetadata, SimplePath
  37. from ._typing import md_none
  38. from .compat import py39, py311
  39. __all__ = [
  40. 'Distribution',
  41. 'DistributionFinder',
  42. 'PackageMetadata',
  43. 'PackageNotFoundError',
  44. 'SimplePath',
  45. 'distribution',
  46. 'distributions',
  47. 'entry_points',
  48. 'files',
  49. 'metadata',
  50. 'packages_distributions',
  51. 'requires',
  52. 'version',
  53. ]
  54. class PackageNotFoundError(ModuleNotFoundError):
  55. """The package was not found."""
  56. def __str__(self) -> str:
  57. return f"No package metadata was found for {self.name}"
  58. @property
  59. def name(self) -> str: # type: ignore[override] # make readonly
  60. (name,) = self.args
  61. return name
  62. class Sectioned:
  63. """
  64. A simple entry point config parser for performance
  65. >>> for item in Sectioned.read(Sectioned._sample):
  66. ... print(item)
  67. Pair(name='sec1', value='# comments ignored')
  68. Pair(name='sec1', value='a = 1')
  69. Pair(name='sec1', value='b = 2')
  70. Pair(name='sec2', value='a = 2')
  71. >>> res = Sectioned.section_pairs(Sectioned._sample)
  72. >>> item = next(res)
  73. >>> item.name
  74. 'sec1'
  75. >>> item.value
  76. Pair(name='a', value='1')
  77. >>> item = next(res)
  78. >>> item.value
  79. Pair(name='b', value='2')
  80. >>> item = next(res)
  81. >>> item.name
  82. 'sec2'
  83. >>> item.value
  84. Pair(name='a', value='2')
  85. >>> list(res)
  86. []
  87. """
  88. _sample = textwrap.dedent(
  89. """
  90. [sec1]
  91. # comments ignored
  92. a = 1
  93. b = 2
  94. [sec2]
  95. a = 2
  96. """
  97. ).lstrip()
  98. @classmethod
  99. def section_pairs(cls, text):
  100. return (
  101. section._replace(value=Pair.parse(section.value))
  102. for section in cls.read(text, filter_=cls.valid)
  103. if section.name is not None
  104. )
  105. @staticmethod
  106. def read(text, filter_=None):
  107. lines = filter(filter_, map(str.strip, text.splitlines()))
  108. name = None
  109. for value in lines:
  110. section_match = value.startswith('[') and value.endswith(']')
  111. if section_match:
  112. name = value.strip('[]')
  113. continue
  114. yield Pair(name, value)
  115. @staticmethod
  116. def valid(line: str):
  117. return line and not line.startswith('#')
  118. class _EntryPointMatch(types.SimpleNamespace):
  119. module: str
  120. attr: str
  121. extras: str
  122. class EntryPoint:
  123. """An entry point as defined by Python packaging conventions.
  124. See `the packaging docs on entry points
  125. <https://packaging.python.org/specifications/entry-points/>`_
  126. for more information.
  127. >>> ep = EntryPoint(
  128. ... name=None, group=None, value='package.module:attr [extra1, extra2]')
  129. >>> ep.module
  130. 'package.module'
  131. >>> ep.attr
  132. 'attr'
  133. >>> ep.extras
  134. ['extra1', 'extra2']
  135. If the value package or module are not valid identifiers, a
  136. ValueError is raised on access.
  137. >>> EntryPoint(name=None, group=None, value='invalid-name').module
  138. Traceback (most recent call last):
  139. ...
  140. ValueError: ('Invalid object reference...invalid-name...
  141. >>> EntryPoint(name=None, group=None, value='invalid-name').attr
  142. Traceback (most recent call last):
  143. ...
  144. ValueError: ('Invalid object reference...invalid-name...
  145. >>> EntryPoint(name=None, group=None, value='invalid-name').extras
  146. Traceback (most recent call last):
  147. ...
  148. ValueError: ('Invalid object reference...invalid-name...
  149. The same thing happens on construction.
  150. >>> EntryPoint(name=None, group=None, value='invalid-name')
  151. Traceback (most recent call last):
  152. ...
  153. ValueError: ('Invalid object reference...invalid-name...
  154. """
  155. pattern = re.compile(
  156. r'(?P<module>[\w.]+)\s*'
  157. r'(:\s*(?P<attr>[\w.]+)\s*)?'
  158. r'((?P<extras>\[.*\])\s*)?$'
  159. )
  160. """
  161. A regular expression describing the syntax for an entry point,
  162. which might look like:
  163. - module
  164. - package.module
  165. - package.module:attribute
  166. - package.module:object.attribute
  167. - package.module:attr [extra1, extra2]
  168. Other combinations are possible as well.
  169. The expression is lenient about whitespace around the ':',
  170. following the attr, and following any extras.
  171. """
  172. name: str
  173. value: str
  174. group: str
  175. dist: Distribution | None = None
  176. def __init__(self, name: str, value: str, group: str) -> None:
  177. vars(self).update(name=name, value=value, group=group)
  178. self.module
  179. def load(self) -> Any:
  180. """Load the entry point from its definition. If only a module
  181. is indicated by the value, return that module. Otherwise,
  182. return the named object.
  183. """
  184. module = import_module(self.module)
  185. attrs = filter(None, (self.attr or '').split('.'))
  186. return functools.reduce(getattr, attrs, module)
  187. @property
  188. def module(self) -> str:
  189. return self._match.module
  190. @property
  191. def attr(self) -> str:
  192. return self._match.attr
  193. @property
  194. def extras(self) -> list[str]:
  195. return re.findall(r'\w+', self._match.extras or '')
  196. @functools.cached_property
  197. def _match(self) -> _EntryPointMatch:
  198. match = self.pattern.match(self.value)
  199. if not match:
  200. raise ValueError(
  201. 'Invalid object reference. '
  202. 'See https://packaging.python.org'
  203. '/en/latest/specifications/entry-points/#data-model',
  204. self.value,
  205. )
  206. return _EntryPointMatch(**match.groupdict())
  207. def _for(self, dist):
  208. vars(self).update(dist=dist)
  209. return self
  210. def matches(self, **params):
  211. """
  212. EntryPoint matches the given parameters.
  213. >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]')
  214. >>> ep.matches(group='foo')
  215. True
  216. >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]')
  217. True
  218. >>> ep.matches(group='foo', name='other')
  219. False
  220. >>> ep.matches()
  221. True
  222. >>> ep.matches(extras=['extra1', 'extra2'])
  223. True
  224. >>> ep.matches(module='bing')
  225. True
  226. >>> ep.matches(attr='bong')
  227. True
  228. """
  229. self._disallow_dist(params)
  230. attrs = (getattr(self, param) for param in params)
  231. return all(map(operator.eq, params.values(), attrs))
  232. @staticmethod
  233. def _disallow_dist(params):
  234. """
  235. Querying by dist is not allowed (dist objects are not comparable).
  236. >>> EntryPoint(name='fan', value='fav', group='fag').matches(dist='foo')
  237. Traceback (most recent call last):
  238. ...
  239. ValueError: "dist" is not suitable for matching...
  240. """
  241. if "dist" in params:
  242. raise ValueError(
  243. '"dist" is not suitable for matching. '
  244. "Instead, use Distribution.entry_points.select() on a "
  245. "located distribution."
  246. )
  247. def _key(self):
  248. return self.name, self.value, self.group
  249. def __lt__(self, other):
  250. return self._key() < other._key()
  251. def __eq__(self, other):
  252. return self._key() == other._key()
  253. def __setattr__(self, name, value):
  254. raise AttributeError("EntryPoint objects are immutable.")
  255. def __repr__(self):
  256. return (
  257. f'EntryPoint(name={self.name!r}, value={self.value!r}, '
  258. f'group={self.group!r})'
  259. )
  260. def __hash__(self) -> int:
  261. return hash(self._key())
  262. class EntryPoints(tuple):
  263. """
  264. An immutable collection of selectable EntryPoint objects.
  265. """
  266. __slots__ = ()
  267. def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] # Work with str instead of int
  268. """
  269. Get the EntryPoint in self matching name.
  270. """
  271. try:
  272. return next(iter(self.select(name=name)))
  273. except StopIteration:
  274. raise KeyError(name)
  275. def __repr__(self):
  276. """
  277. Repr with classname and tuple constructor to
  278. signal that we deviate from regular tuple behavior.
  279. """
  280. return '%s(%r)' % (self.__class__.__name__, tuple(self))
  281. def select(self, **params) -> EntryPoints:
  282. """
  283. Select entry points from self that match the
  284. given parameters (typically group and/or name).
  285. """
  286. return EntryPoints(ep for ep in self if py39.ep_matches(ep, **params))
  287. @property
  288. def names(self) -> set[str]:
  289. """
  290. Return the set of all names of all entry points.
  291. """
  292. return {ep.name for ep in self}
  293. @property
  294. def groups(self) -> set[str]:
  295. """
  296. Return the set of all groups of all entry points.
  297. """
  298. return {ep.group for ep in self}
  299. @classmethod
  300. def _from_text_for(cls, text, dist):
  301. return cls(ep._for(dist) for ep in cls._from_text(text))
  302. @staticmethod
  303. def _from_text(text):
  304. return (
  305. EntryPoint(name=item.value.name, value=item.value.value, group=item.name)
  306. for item in Sectioned.section_pairs(text or '')
  307. )
  308. class PackagePath(pathlib.PurePosixPath):
  309. """A reference to a path in a package"""
  310. hash: FileHash | None
  311. size: int
  312. dist: Distribution
  313. def read_text(self, encoding: str = 'utf-8') -> str:
  314. return self.locate().read_text(encoding=encoding)
  315. def read_binary(self) -> bytes:
  316. return self.locate().read_bytes()
  317. def locate(self) -> SimplePath:
  318. """Return a path-like object for this path"""
  319. return self.dist.locate_file(self)
  320. class FileHash:
  321. def __init__(self, spec: str) -> None:
  322. self.mode, _, self.value = spec.partition('=')
  323. def __repr__(self) -> str:
  324. return f'<FileHash mode: {self.mode} value: {self.value}>'
  325. class Distribution(metaclass=abc.ABCMeta):
  326. """
  327. An abstract Python distribution package.
  328. Custom providers may derive from this class and define
  329. the abstract methods to provide a concrete implementation
  330. for their environment. Some providers may opt to override
  331. the default implementation of some properties to bypass
  332. the file-reading mechanism.
  333. """
  334. @abc.abstractmethod
  335. def read_text(self, filename) -> str | None:
  336. """Attempt to load metadata file given by the name.
  337. Python distribution metadata is organized by blobs of text
  338. typically represented as "files" in the metadata directory
  339. (e.g. package-1.0.dist-info). These files include things
  340. like:
  341. - METADATA: The distribution metadata including fields
  342. like Name and Version and Description.
  343. - entry_points.txt: A series of entry points as defined in
  344. `the entry points spec <https://packaging.python.org/en/latest/specifications/entry-points/#file-format>`_.
  345. - RECORD: A record of files according to
  346. `this recording spec <https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-record-file>`_.
  347. A package may provide any set of files, including those
  348. not listed here or none at all.
  349. :param filename: The name of the file in the distribution info.
  350. :return: The text if found, otherwise None.
  351. """
  352. @abc.abstractmethod
  353. def locate_file(self, path: str | os.PathLike[str]) -> SimplePath:
  354. """
  355. Given a path to a file in this distribution, return a SimplePath
  356. to it.
  357. This method is used by callers of ``Distribution.files()`` to
  358. locate files within the distribution. If it's possible for a
  359. Distribution to represent files in the distribution as
  360. ``SimplePath`` objects, it should implement this method
  361. to resolve such objects.
  362. Some Distribution providers may elect not to resolve SimplePath
  363. objects within the distribution by raising a
  364. NotImplementedError, but consumers of such a Distribution would
  365. be unable to invoke ``Distribution.files()``.
  366. """
  367. @classmethod
  368. def from_name(cls, name: str) -> Distribution:
  369. """Return the Distribution for the given package name.
  370. :param name: The name of the distribution package to search for.
  371. :return: The Distribution instance (or subclass thereof) for the named
  372. package, if found.
  373. :raises PackageNotFoundError: When the named package's distribution
  374. metadata cannot be found.
  375. :raises ValueError: When an invalid value is supplied for name.
  376. """
  377. if not name:
  378. raise ValueError("A distribution name is required.")
  379. try:
  380. return next(iter(cls._prefer_valid(cls.discover(name=name))))
  381. except StopIteration:
  382. raise PackageNotFoundError(name)
  383. @classmethod
  384. def discover(
  385. cls, *, context: DistributionFinder.Context | None = None, **kwargs
  386. ) -> Iterable[Distribution]:
  387. """Return an iterable of Distribution objects for all packages.
  388. Pass a ``context`` or pass keyword arguments for constructing
  389. a context.
  390. :context: A ``DistributionFinder.Context`` object.
  391. :return: Iterable of Distribution objects for packages matching
  392. the context.
  393. """
  394. if context and kwargs:
  395. raise ValueError("cannot accept context and kwargs")
  396. context = context or DistributionFinder.Context(**kwargs)
  397. return itertools.chain.from_iterable(
  398. resolver(context) for resolver in cls._discover_resolvers()
  399. )
  400. @staticmethod
  401. def _prefer_valid(dists: Iterable[Distribution]) -> Iterable[Distribution]:
  402. """
  403. Prefer (move to the front) distributions that have metadata.
  404. Ref python/importlib_resources#489.
  405. """
  406. buckets = bucket(dists, lambda dist: bool(dist.metadata))
  407. return itertools.chain(buckets[True], buckets[False])
  408. @staticmethod
  409. def at(path: str | os.PathLike[str]) -> Distribution:
  410. """Return a Distribution for the indicated metadata path.
  411. :param path: a string or path-like object
  412. :return: a concrete Distribution instance for the path
  413. """
  414. return PathDistribution(pathlib.Path(path))
  415. @staticmethod
  416. def _discover_resolvers():
  417. """Search the meta_path for resolvers (MetadataPathFinders)."""
  418. declared = (
  419. getattr(finder, 'find_distributions', None) for finder in sys.meta_path
  420. )
  421. return filter(None, declared)
  422. @property
  423. def metadata(self) -> _meta.PackageMetadata | None:
  424. """Return the parsed metadata for this Distribution.
  425. The returned object will have keys that name the various bits of
  426. metadata per the
  427. `Core metadata specifications <https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata>`_.
  428. Custom providers may provide the METADATA file or override this
  429. property.
  430. """
  431. text = (
  432. self.read_text('METADATA')
  433. or self.read_text('PKG-INFO')
  434. # This last clause is here to support old egg-info files. Its
  435. # effect is to just end up using the PathDistribution's self._path
  436. # (which points to the egg-info file) attribute unchanged.
  437. or self.read_text('')
  438. )
  439. return self._assemble_message(text)
  440. @staticmethod
  441. @pass_none
  442. def _assemble_message(text: str) -> _meta.PackageMetadata:
  443. # deferred for performance (python/cpython#109829)
  444. from . import _adapters
  445. return _adapters.Message(email.message_from_string(text))
  446. @property
  447. def name(self) -> str:
  448. """Return the 'Name' metadata for the distribution package."""
  449. return md_none(self.metadata)['Name']
  450. @property
  451. def _normalized_name(self):
  452. """Return a normalized version of the name."""
  453. return Prepared.normalize(self.name)
  454. @property
  455. def version(self) -> str:
  456. """Return the 'Version' metadata for the distribution package."""
  457. return md_none(self.metadata)['Version']
  458. @property
  459. def entry_points(self) -> EntryPoints:
  460. """
  461. Return EntryPoints for this distribution.
  462. Custom providers may provide the ``entry_points.txt`` file
  463. or override this property.
  464. """
  465. return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self)
  466. @property
  467. def files(self) -> list[PackagePath] | None:
  468. """Files in this distribution.
  469. :return: List of PackagePath for this distribution or None
  470. Result is `None` if the metadata file that enumerates files
  471. (i.e. RECORD for dist-info, or installed-files.txt or
  472. SOURCES.txt for egg-info) is missing.
  473. Result may be empty if the metadata exists but is empty.
  474. Custom providers are recommended to provide a "RECORD" file (in
  475. ``read_text``) or override this property to allow for callers to be
  476. able to resolve filenames provided by the package.
  477. """
  478. def make_file(name, hash=None, size_str=None):
  479. result = PackagePath(name)
  480. result.hash = FileHash(hash) if hash else None
  481. result.size = int(size_str) if size_str else None
  482. result.dist = self
  483. return result
  484. @pass_none
  485. def make_files(lines):
  486. # Delay csv import, since Distribution.files is not as widely used
  487. # as other parts of importlib.metadata
  488. import csv
  489. return starmap(make_file, csv.reader(lines))
  490. @pass_none
  491. def skip_missing_files(package_paths):
  492. return list(filter(lambda path: path.locate().exists(), package_paths))
  493. return skip_missing_files(
  494. make_files(
  495. self._read_files_distinfo()
  496. or self._read_files_egginfo_installed()
  497. or self._read_files_egginfo_sources()
  498. )
  499. )
  500. def _read_files_distinfo(self):
  501. """
  502. Read the lines of RECORD.
  503. """
  504. text = self.read_text('RECORD')
  505. return text and text.splitlines()
  506. def _read_files_egginfo_installed(self):
  507. """
  508. Read installed-files.txt and return lines in a similar
  509. CSV-parsable format as RECORD: each file must be placed
  510. relative to the site-packages directory and must also be
  511. quoted (since file names can contain literal commas).
  512. This file is written when the package is installed by pip,
  513. but it might not be written for other installation methods.
  514. Assume the file is accurate if it exists.
  515. """
  516. text = self.read_text('installed-files.txt')
  517. # Prepend the .egg-info/ subdir to the lines in this file.
  518. # But this subdir is only available from PathDistribution's
  519. # self._path.
  520. subdir = getattr(self, '_path', None)
  521. if not text or not subdir:
  522. return
  523. paths = (
  524. py311
  525. .relative_fix((subdir / name).resolve())
  526. .relative_to(self.locate_file('').resolve(), walk_up=True)
  527. .as_posix()
  528. for name in text.splitlines()
  529. )
  530. return map('"{}"'.format, paths)
  531. def _read_files_egginfo_sources(self):
  532. """
  533. Read SOURCES.txt and return lines in a similar CSV-parsable
  534. format as RECORD: each file name must be quoted (since it
  535. might contain literal commas).
  536. Note that SOURCES.txt is not a reliable source for what
  537. files are installed by a package. This file is generated
  538. for a source archive, and the files that are present
  539. there (e.g. setup.py) may not correctly reflect the files
  540. that are present after the package has been installed.
  541. """
  542. text = self.read_text('SOURCES.txt')
  543. return text and map('"{}"'.format, text.splitlines())
  544. @property
  545. def requires(self) -> list[str] | None:
  546. """Generated requirements specified for this Distribution"""
  547. reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
  548. return reqs and list(reqs)
  549. def _read_dist_info_reqs(self):
  550. return self.metadata.get_all('Requires-Dist')
  551. def _read_egg_info_reqs(self):
  552. source = self.read_text('requires.txt')
  553. return pass_none(self._deps_from_requires_text)(source)
  554. @classmethod
  555. def _deps_from_requires_text(cls, source):
  556. return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source))
  557. @staticmethod
  558. def _convert_egg_info_reqs_to_simple_reqs(sections):
  559. """
  560. Historically, setuptools would solicit and store 'extra'
  561. requirements, including those with environment markers,
  562. in separate sections. More modern tools expect each
  563. dependency to be defined separately, with any relevant
  564. extras and environment markers attached directly to that
  565. requirement. This method converts the former to the
  566. latter. See _test_deps_from_requires_text for an example.
  567. """
  568. def make_condition(name):
  569. return name and f'extra == "{name}"'
  570. def quoted_marker(section):
  571. section = section or ''
  572. extra, sep, markers = section.partition(':')
  573. if extra and markers:
  574. markers = f'({markers})'
  575. conditions = list(filter(None, [markers, make_condition(extra)]))
  576. return '; ' + ' and '.join(conditions) if conditions else ''
  577. def url_req_space(req):
  578. """
  579. PEP 508 requires a space between the url_spec and the quoted_marker.
  580. Ref python/importlib_metadata#357.
  581. """
  582. # '@' is uniquely indicative of a url_req.
  583. return ' ' * ('@' in req)
  584. for section in sections:
  585. space = url_req_space(section.value)
  586. yield section.value + space + quoted_marker(section.name)
  587. @property
  588. def origin(self):
  589. return self._load_json('direct_url.json')
  590. def _load_json(self, filename):
  591. # Deferred for performance (python/importlib_metadata#503)
  592. import json
  593. return pass_none(json.loads)(
  594. self.read_text(filename),
  595. object_hook=lambda data: types.SimpleNamespace(**data),
  596. )
  597. class DistributionFinder(MetaPathFinder):
  598. """
  599. A MetaPathFinder capable of discovering installed distributions.
  600. Custom providers should implement this interface in order to
  601. supply metadata.
  602. """
  603. class Context:
  604. """
  605. Keyword arguments presented by the caller to
  606. ``distributions()`` or ``Distribution.discover()``
  607. to narrow the scope of a search for distributions
  608. in all DistributionFinders.
  609. Each DistributionFinder may expect any parameters
  610. and should attempt to honor the canonical
  611. parameters defined below when appropriate.
  612. This mechanism gives a custom provider a means to
  613. solicit additional details from the caller beyond
  614. "name" and "path" when searching distributions.
  615. For example, imagine a provider that exposes suites
  616. of packages in either a "public" or "private" ``realm``.
  617. A caller may wish to query only for distributions in
  618. a particular realm and could call
  619. ``distributions(realm="private")`` to signal to the
  620. custom provider to only include distributions from that
  621. realm.
  622. """
  623. name = None
  624. """
  625. Specific name for which a distribution finder should match.
  626. A name of ``None`` matches all distributions.
  627. """
  628. def __init__(self, **kwargs):
  629. vars(self).update(kwargs)
  630. @property
  631. def path(self) -> list[str]:
  632. """
  633. The sequence of directory path that a distribution finder
  634. should search.
  635. Typically refers to Python installed package paths such as
  636. "site-packages" directories and defaults to ``sys.path``.
  637. """
  638. return vars(self).get('path', sys.path)
  639. @abc.abstractmethod
  640. def find_distributions(self, context=Context()) -> Iterable[Distribution]:
  641. """
  642. Find distributions.
  643. Return an iterable of all Distribution instances capable of
  644. loading the metadata for packages matching the ``context``,
  645. a DistributionFinder.Context instance.
  646. """
  647. @passthrough
  648. def _clear_after_fork(cached):
  649. """Ensure ``func`` clears cached state after ``fork`` when supported.
  650. ``FastPath`` caches zip-backed ``pathlib.Path`` objects that retain a
  651. reference to the parent's open ``ZipFile`` handle. Re-using a cached
  652. instance in a forked child can therefore resurrect invalid file pointers
  653. and trigger ``BadZipFile``/``OSError`` failures (python/importlib_metadata#520).
  654. Registering ``cache_clear`` with ``os.register_at_fork`` keeps each process
  655. on its own cache.
  656. """
  657. getattr(os, 'register_at_fork', noop)(after_in_child=cached.cache_clear)
  658. class FastPath:
  659. """
  660. Micro-optimized class for searching a root for children.
  661. Root is a path on the file system that may contain metadata
  662. directories either as natural directories or within a zip file.
  663. >>> FastPath('').children()
  664. ['...']
  665. FastPath objects are cached and recycled for any given root.
  666. >>> FastPath('foobar') is FastPath('foobar')
  667. True
  668. """
  669. @_clear_after_fork # type: ignore[misc]
  670. @functools.lru_cache()
  671. def __new__(cls, root):
  672. return super().__new__(cls)
  673. def __init__(self, root):
  674. self.root = root
  675. def joinpath(self, child):
  676. return pathlib.Path(self.root, child)
  677. def children(self):
  678. with suppress(Exception):
  679. return os.listdir(self.root or '.')
  680. with suppress(Exception):
  681. return self.zip_children()
  682. return []
  683. def zip_children(self):
  684. # deferred for performance (python/importlib_metadata#502)
  685. from zipp.compat.overlay import zipfile
  686. zip_path = zipfile.Path(self.root)
  687. names = zip_path.root.namelist()
  688. self.joinpath = zip_path.joinpath
  689. return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names)
  690. def search(self, name):
  691. return self.lookup(self.mtime).search(name)
  692. @property
  693. def mtime(self):
  694. with suppress(OSError):
  695. return os.stat(self.root).st_mtime
  696. self.lookup.cache_clear()
  697. @method_cache
  698. def lookup(self, mtime):
  699. return Lookup(self)
  700. class Lookup:
  701. """
  702. A micro-optimized class for searching a (fast) path for metadata.
  703. """
  704. def __init__(self, path: FastPath):
  705. """
  706. Calculate all of the children representing metadata.
  707. From the children in the path, calculate early all of the
  708. children that appear to represent metadata (infos) or legacy
  709. metadata (eggs).
  710. """
  711. base = os.path.basename(path.root).lower()
  712. base_is_egg = base.endswith(".egg")
  713. self.infos = FreezableDefaultDict(list)
  714. self.eggs = FreezableDefaultDict(list)
  715. for child in path.children():
  716. low = child.lower()
  717. if low.endswith((".dist-info", ".egg-info")):
  718. # rpartition is faster than splitext and suitable for this purpose.
  719. name = low.rpartition(".")[0].partition("-")[0]
  720. normalized = Prepared.normalize(name)
  721. self.infos[normalized].append(path.joinpath(child))
  722. elif base_is_egg and low == "egg-info":
  723. name = base.rpartition(".")[0].partition("-")[0]
  724. legacy_normalized = Prepared.legacy_normalize(name)
  725. self.eggs[legacy_normalized].append(path.joinpath(child))
  726. self.infos.freeze()
  727. self.eggs.freeze()
  728. def search(self, prepared: Prepared):
  729. """
  730. Yield all infos and eggs matching the Prepared query.
  731. """
  732. infos = (
  733. self.infos[prepared.normalized]
  734. if prepared
  735. else itertools.chain.from_iterable(self.infos.values())
  736. )
  737. eggs = (
  738. self.eggs[prepared.legacy_normalized]
  739. if prepared
  740. else itertools.chain.from_iterable(self.eggs.values())
  741. )
  742. return itertools.chain(infos, eggs)
  743. class Prepared:
  744. """
  745. A prepared search query for metadata on a possibly-named package.
  746. Pre-calculates the normalization to prevent repeated operations.
  747. >>> none = Prepared(None)
  748. >>> none.normalized
  749. >>> none.legacy_normalized
  750. >>> bool(none)
  751. False
  752. >>> sample = Prepared('Sample__Pkg-name.foo')
  753. >>> sample.normalized
  754. 'sample_pkg_name_foo'
  755. >>> sample.legacy_normalized
  756. 'sample__pkg_name.foo'
  757. >>> bool(sample)
  758. True
  759. """
  760. normalized = None
  761. legacy_normalized = None
  762. def __init__(self, name: str | None):
  763. self.name = name
  764. if name is None:
  765. return
  766. self.normalized = self.normalize(name)
  767. self.legacy_normalized = self.legacy_normalize(name)
  768. @staticmethod
  769. def normalize(name):
  770. """
  771. PEP 503 normalization plus dashes as underscores.
  772. """
  773. return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_')
  774. @staticmethod
  775. def legacy_normalize(name):
  776. """
  777. Normalize the package name as found in the convention in
  778. older packaging tools versions and specs.
  779. """
  780. return name.lower().replace('-', '_')
  781. def __bool__(self):
  782. return bool(self.name)
  783. @install
  784. class MetadataPathFinder(NullFinder, DistributionFinder):
  785. """A degenerate finder for distribution packages on the file system.
  786. This finder supplies only a find_distributions() method for versions
  787. of Python that do not have a PathFinder find_distributions().
  788. """
  789. @classmethod
  790. def find_distributions(
  791. cls, context=DistributionFinder.Context()
  792. ) -> Iterable[PathDistribution]:
  793. """
  794. Find distributions.
  795. Return an iterable of all Distribution instances capable of
  796. loading the metadata for packages matching ``context.name``
  797. (or all names if ``None`` indicated) along the paths in the list
  798. of directories ``context.path``.
  799. """
  800. found = cls._search_paths(context.name, context.path)
  801. return map(PathDistribution, found)
  802. @classmethod
  803. def _search_paths(cls, name, paths):
  804. """Find metadata directories in paths heuristically."""
  805. prepared = Prepared(name)
  806. return itertools.chain.from_iterable(
  807. path.search(prepared) for path in map(FastPath, paths)
  808. )
  809. @classmethod
  810. def invalidate_caches(cls) -> None:
  811. FastPath.__new__.cache_clear()
  812. class PathDistribution(Distribution):
  813. def __init__(self, path: SimplePath) -> None:
  814. """Construct a distribution.
  815. :param path: SimplePath indicating the metadata directory.
  816. """
  817. self._path = path
  818. def read_text(self, filename: str | os.PathLike[str]) -> str | None:
  819. with suppress(
  820. FileNotFoundError,
  821. IsADirectoryError,
  822. KeyError,
  823. NotADirectoryError,
  824. PermissionError,
  825. ):
  826. return self._path.joinpath(filename).read_text(encoding='utf-8')
  827. return None
  828. read_text.__doc__ = Distribution.read_text.__doc__
  829. def locate_file(self, path: str | os.PathLike[str]) -> SimplePath:
  830. return self._path.parent / path
  831. @property
  832. def _normalized_name(self):
  833. """
  834. Performance optimization: where possible, resolve the
  835. normalized name from the file system path.
  836. """
  837. stem = os.path.basename(str(self._path))
  838. return (
  839. pass_none(Prepared.normalize)(self._name_from_stem(stem))
  840. or super()._normalized_name
  841. )
  842. @staticmethod
  843. def _name_from_stem(stem):
  844. """
  845. >>> PathDistribution._name_from_stem('foo-3.0.egg-info')
  846. 'foo'
  847. >>> PathDistribution._name_from_stem('CherryPy-3.0.dist-info')
  848. 'CherryPy'
  849. >>> PathDistribution._name_from_stem('face.egg-info')
  850. 'face'
  851. >>> PathDistribution._name_from_stem('foo.bar')
  852. """
  853. filename, ext = os.path.splitext(stem)
  854. if ext not in ('.dist-info', '.egg-info'):
  855. return
  856. name, sep, rest = filename.partition('-')
  857. return name
  858. def distribution(distribution_name: str) -> Distribution:
  859. """Get the ``Distribution`` instance for the named package.
  860. :param distribution_name: The name of the distribution package as a string.
  861. :return: A ``Distribution`` instance (or subclass thereof).
  862. """
  863. return Distribution.from_name(distribution_name)
  864. def distributions(**kwargs) -> Iterable[Distribution]:
  865. """Get all ``Distribution`` instances in the current environment.
  866. :return: An iterable of ``Distribution`` instances.
  867. """
  868. return Distribution.discover(**kwargs)
  869. def metadata(distribution_name: str) -> _meta.PackageMetadata | None:
  870. """Get the metadata for the named package.
  871. :param distribution_name: The name of the distribution package to query.
  872. :return: A PackageMetadata containing the parsed metadata.
  873. """
  874. return Distribution.from_name(distribution_name).metadata
  875. def version(distribution_name: str) -> str:
  876. """Get the version string for the named package.
  877. :param distribution_name: The name of the distribution package to query.
  878. :return: The version string for the package as defined in the package's
  879. "Version" metadata key.
  880. """
  881. return distribution(distribution_name).version
  882. _unique = functools.partial(
  883. unique_everseen,
  884. key=py39.normalized_name,
  885. )
  886. """
  887. Wrapper for ``distributions`` to return unique distributions by name.
  888. """
  889. def entry_points(**params) -> EntryPoints:
  890. """Return EntryPoint objects for all installed packages.
  891. Pass selection parameters (group or name) to filter the
  892. result to entry points matching those properties (see
  893. EntryPoints.select()).
  894. :return: EntryPoints for all installed packages.
  895. """
  896. eps = itertools.chain.from_iterable(
  897. dist.entry_points for dist in _unique(distributions())
  898. )
  899. return EntryPoints(eps).select(**params)
  900. def files(distribution_name: str) -> list[PackagePath] | None:
  901. """Return a list of files for the named package.
  902. :param distribution_name: The name of the distribution package to query.
  903. :return: List of files composing the distribution.
  904. """
  905. return distribution(distribution_name).files
  906. def requires(distribution_name: str) -> list[str] | None:
  907. """
  908. Return a list of requirements for the named package.
  909. :return: An iterable of requirements, suitable for
  910. packaging.requirement.Requirement.
  911. """
  912. return distribution(distribution_name).requires
  913. def packages_distributions() -> Mapping[str, list[str]]:
  914. """
  915. Return a mapping of top-level packages to their
  916. distributions.
  917. >>> import collections.abc
  918. >>> pkgs = packages_distributions()
  919. >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values())
  920. True
  921. """
  922. pkg_to_dist = collections.defaultdict(list)
  923. for dist in distributions():
  924. for pkg in _top_level_declared(dist) or _top_level_inferred(dist):
  925. pkg_to_dist[pkg].append(md_none(dist.metadata)['Name'])
  926. return dict(pkg_to_dist)
  927. def _top_level_declared(dist):
  928. return (dist.read_text('top_level.txt') or '').split()
  929. def _topmost(name: PackagePath) -> str | None:
  930. """
  931. Return the top-most parent as long as there is a parent.
  932. """
  933. top, *rest = name.parts
  934. return top if rest else None
  935. def _get_toplevel_name(name: PackagePath) -> str:
  936. """
  937. Infer a possibly importable module name from a name presumed on
  938. sys.path.
  939. >>> _get_toplevel_name(PackagePath('foo.py'))
  940. 'foo'
  941. >>> _get_toplevel_name(PackagePath('foo'))
  942. 'foo'
  943. >>> _get_toplevel_name(PackagePath('foo.pyc'))
  944. 'foo'
  945. >>> _get_toplevel_name(PackagePath('foo/__init__.py'))
  946. 'foo'
  947. >>> _get_toplevel_name(PackagePath('foo.pth'))
  948. 'foo.pth'
  949. >>> _get_toplevel_name(PackagePath('foo.dist-info'))
  950. 'foo.dist-info'
  951. """
  952. # Defer import of inspect for performance (python/cpython#118761)
  953. import inspect
  954. return _topmost(name) or inspect.getmodulename(name) or str(name)
  955. def _top_level_inferred(dist):
  956. opt_names = set(map(_get_toplevel_name, always_iterable(dist.files)))
  957. def importable_name(name):
  958. return '.' not in name
  959. return filter(importable_name, opt_names)