requirements/openstack_requirements/project.py

# Copyright 2012 OpenStack Foundation
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""The project abstraction."""

import configparser
import errno
import io
import os
import sys
from typing import Any
from typing import TypedDict

try:
    # Python 3.11+
    import tomllib
except ImportError:
    # Python 3.10 and lower
    import tomli as tomllib  # type: ignore


def _read_raw(root: str, filename: str) -> str | None:
    try:
        path = os.path.join(root, filename)
        with open(path, encoding="utf-8") as f:
            data = f.read()
            if not data.endswith('\n'):
                print(
                    f"Requirements file {filename} does not end with a "
                    f"newline.",
                    file=sys.stderr,
                )
            return data
    except OSError as e:
        if e.errno == errno.ENOENT:
            return None

        raise


def _read_pyproject_toml(root: str) -> dict[str, Any] | None:
    data = _read_raw(root, 'pyproject.toml')
    if data is None:
        return None

    return tomllib.loads(data)


def _read_requirements_txt(root: str, filename: str) -> list[str] | None:
    data = _read_raw(root, filename)
    if data is None:
        return None

    result = []
    for line in data.splitlines():
        # we only ignore comments and empty lines: everything else is
        # handled later
        line = line.strip()

        if line.startswith('#') or not line:
            continue

        result.append(line)

    return result


def _read_pyproject_toml_requirements(root: str) -> list[str] | None:
    data = _read_pyproject_toml(root) or {}

    # projects may not have PEP-621 project metadata
    if 'project' not in data:
        return None

    return data['project'].get('dependencies', [])


def _read_pyproject_toml_extras(root: str) -> dict[str, list[str]] | None:
    data = _read_pyproject_toml(root) or {}

    # projects may not have PEP-621 project metadata
    if 'project' not in data:
        return None

    return data['project'].get('optional-dependencies', {})


def _read_setup_cfg_extras(root: str) -> dict[str, list[str]] | None:
    data = _read_raw(root, 'setup.cfg')
    if data is None:
        return None

    c = configparser.ConfigParser()
    c.read_file(io.StringIO(data))
    if not c.has_section('extras'):
        return None

    result: dict[str, list[str]] = {}
    for extra, deps in c.items('extras'):
        result[extra] = []
        for line in deps.splitlines():
            # we only ignore comments and empty lines: everything else is
            # handled later
            line = line.strip()

            if line.startswith('#') or not line:
                continue

            result[extra].append(line)

    return result


class Project(TypedDict):
    # The root directory path
    root: str
    # A mapping of filename to the contents of that file
    requirements: dict[str, list[str]]
    # A mapping of filename to extras from that file
    extras: dict[str, dict[str, list[str]]]


def read(root: str) -> Project:
    """Read into memory the packaging data for the project at root.

    :param root: A directory path.
    :return: A dict representing the project with the following keys:
        - root: The root dir.
        - requirements: Dict of requirement file name
        - extras: Dict of extras file name to a dict of extra names and
          requirements
    """
    # Store root directory and installer-related files for later processing
    result: Project = {
        'root': root,
        'requirements': {},
        'extras': {},
    }

    # Store requirements
    if (data := _read_pyproject_toml_requirements(root)) is not None:
        result['requirements']['pyproject.toml'] = data

    for filename in [
        'requirements.txt',
        'test-requirements.txt',
        'doc/requirements.txt',
        # deprecated aliases (warnings are handled elsewhere)
        'tools/pip-requires',
        'tools/test-requires',
        'requirements-py2.txt',
        'requirements-py3.txt',
        'test-requirements-py2.txt',
        'test-requirements-py3.txt',
    ]:
        if (data := _read_requirements_txt(root, filename)) is not None:
            result['requirements'][filename] = data

    # Store extras
    if (data := _read_setup_cfg_extras(root)) is not None:
        result['extras']['setup.cfg'] = data

    if (data := _read_pyproject_toml_extras(root)) is not None:
        result['extras']['pyproject.toml'] = data

    return result