diff --git a/global-requirements.txt b/global-requirements.txt index 56af287e6d..735e586ce4 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -1,14 +1,14 @@ ## section:general -alembic # MIT -amqp # BSD -ansible-runner # Apache 2.0 +alembic!=1.2.0,!=1.6.3 # MIT +amqp!=2.1.4,!=5.0.4 # BSD +ansible-runner!=1.3.5 # Apache 2.0 appdirs # MIT License apscheduler # MIT License autobahn # MIT License automaton # Apache-2.0 autopage # Apache-2.0 -Babel # BSD +Babel!=2.4.0 # BSD bcrypt # Apache-2.0 beautifulsoup4 # MIT betamax # Apache-2.0 @@ -16,15 +16,15 @@ boto # MIT boto3 # Apache-2.0 botocore # Apache-2.0 cachetools # MIT License -cassandra-driver # Apache-2.0 +cassandra-driver!=3.6.0 # Apache-2.0 cffi # MIT -cmd2 # MIT -confluent-kafka # Apache-2.0 +cmd2!=0.8.3 # MIT +confluent-kafka!=1.4.0 # Apache-2.0 confspirator # Apache-2.0 construct # MIT cotyledon # Apache-2.0 croniter # MIT License -cryptography # BSD/Apache-2.0 +cryptography!=2.0 # BSD/Apache-2.0 cursive # Apache-2.0 datetimerange # MIT decorator # BSD @@ -34,25 +34,27 @@ django-debreach # BSD django-formtools # BSD django-pyscss # BSD License (2 clause) Django<4.3 # BSD -dnspython # http://www.dnspython.org/LICENSE -dogpile.cache!=1.1.7 # BSD +# eventlet is not compatibile with 2.0.0: https://github.com/eventlet/eventlet/issues/619 +dnspython!=2.0.0,!=2.2.0 # http://www.dnspython.org/LICENSE +dogpile.cache!=0.9.1,!=1.1.7 # BSD dogtag-pki # LGPLv3+ -dulwich # Apache-2.0 +dulwich!=0.19.3,!=0.19.7 # Apache-2.0 edgegrid-python # Apache-2.0 elasticsearch<3.0.0 # Apache-2.0 enmerkar # BSD # NOTE: New versions of eventlet should not be accepted lightly # as they have earned a reputation of frequently breaking things. -eventlet!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT -exabgp # BSD +eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,!=0.32.0,!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT +exabgp!=4.0.6 # BSD falcon # Apache-2.0 -fasteners # Apache-2.0 -Flask # BSD +# https://github.com/harlowja/fasteners/issues/36 +fasteners!=0.15,!=0.16 # Apache-2.0 +Flask!=0.11 # BSD Flask-RESTful # BSD GitPython # BSD License (3 clause) google-api-python-client # Apache-2.0 -graphviz # MIT License -greenlet # MIT +graphviz!=0.5.0 # MIT License +greenlet!=0.4.14 # MIT gunicorn # MIT httplib2 # MIT httpx # BSD @@ -62,7 +64,7 @@ icalendar # BSD # newer code than in [most] releases of the Python std library. importlib-metadata # Apache-2.0 infinisdk # BSD-3 -influxdb # MIT +influxdb!=5.3.0 # MIT influxdb-client # MIT infoblox-client # Apache-2.0 iso8601 # MIT @@ -70,33 +72,33 @@ jaeger-client # Apache-2.0 Jinja2 # BSD License (3 clause) jira # BSD License (2 clause) jmespath # MIT -jsonpatch # BSD +jsonpatch!=1.20 # BSD jsonschema # MIT kazoo # Apache-2.0 -kombu # BSD +kombu!=4.0.2 # BSD kubernetes # Apache-2.0 ldap3 # LGPLv3 -libvirt-python # LGPLv2+ -lxml # BSD +libvirt-python!=4.1.0,!=4.2.0 # LGPLv2+ +lxml!=3.7.0 # BSD Mako # MIT msgpack # Apache-2.0 munch # MIT ncclient # Apache-2.0 netaddr # BSD -netifaces # MIT +netifaces!=0.10.0,!=0.10.1 # MIT netmiko # MIT networkx!=2.8.4 # BSD oauthlib # BSD opentelemetry-exporter-otlp # Apache-2.0 opentelemetry-sdk # Apache-2.0 ovs # Apache-2.0 -packaging # Apache-2.0 -paramiko # LGPLv2.1+ +packaging!=20.5,!=20.6,!=20.7 # Apache-2.0 +paramiko!=2.9.0,!=2.9.1 # LGPLv2.1+ passlib # BSD Paste # MIT PasteDeploy # MIT -pecan # BSD -pexpect # ISC License +pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,!=1.4.0 # BSD +pexpect!=3.3 # ISC License pika # BSD Pillow # PIL License Pint # BSD @@ -105,27 +107,27 @@ PrettyTable!=3.4.0 # BSD prometheus-client # Apache-2.0 protobuf # BSD License (3 clause) psutil # BSD -pyasn1 # BSD +pyasn1!=0.2.3 # BSD pyasn1-lextudio # BSD pyasn1-modules # BSD pyasn1-modules-lextudio # BSD -pycadf # Apache-2.0 +pycadf!=2.0.0 # Apache-2.0 pycdlib # LGPLv2+ PyECLib # BSD -pyghmi # Apache-2.0 +pyghmi!=1.4.0,!=1.5.11 # Apache-2.0 PyJWT # MIT pykmip # Apache 2.0 License pylxd # Apache-2.0 -pymemcache # Apache 2.0 License +pymemcache!=1.3.0 # Apache 2.0 License PyMI;sys_platform=='win32' # Apache 2.0 License -pymongo # Apache-2.0 +pymongo!=3.1 # Apache-2.0 PyMySQL # MIT License pyngus # Apache-2.0 pyOpenSSL # Apache-2.0 pyparsing # MIT -pyroute2!=0.7.1;sys_platform != "win32" # Apache-2.0 (+ dual licensed GPL2) -pysaml2 # Apache-2.0 -pyScss # MIT License +pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2) +pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 +pyScss!=1.3.5 # MIT License pysnmp-lextudio # BSD pystache # MIT # Only required for sasl/binary protocol @@ -140,7 +142,7 @@ pywinrm # MIT PyYAML # MIT pyzabbix # LGPL rbd-iscsi-client # Apache-2.0 -requests # Apache-2.0 +requests!=2.20.0,!=2.24.0 # Apache-2.0 requests-aws # BSD License (3 clause) requests-kerberos # ISC requestsexceptions # Apache-2.0 @@ -148,14 +150,18 @@ rfc3986 # Apache-2.0 Routes # MIT rtslib-fb # Apache-2.0 ruamel.yaml # MIT -salt # Apache-2.0 +salt!=2019.2.1,!=2019.2.2 # Apache-2.0 scikit-learn # BSD scipy # BSD # https://github.com/holgern/py-scrypt/issues/16 scrypt!=0.8.21 # BSD semantic-version # BSD setproctitle # BSD -SQLAlchemy # MIT +# NOTE(yamahata): +# bug work around of sqlalchemy +# https://bitbucket.org/zzzeek/sqlalchemy/issues/3952/ +# The fix which is in git master branch is planned for 1.1.9 +SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT sqlalchemy-filters # Apache-2.0 sqlalchemy-migrate # Apache-2.0 SQLAlchemy-Utils # BSD License @@ -215,7 +221,7 @@ XStatic-Spin # MIT License XStatic-term.js # MIT License XStatic-tv4 # MIT # NOTE(anilvenkata): This is required for profiling oslo.service processes -Yappi # MIT +Yappi!=0.98,!=0.99 # MIT zeroconf # LGPL zipp # MIT zstd # BSD License (2 clause) @@ -224,7 +230,7 @@ zVMCloudConnector;sys_platform!='win32' # Apache 2.0 License ## section:testing bashate # Apache-2.0 -coverage # Apache-2.0 +coverage!=4.4 # Apache-2.0 ddt # MIT django-nose # BSD docker # Apache-2.0 @@ -239,7 +245,7 @@ ldappool # MPL # Do not make mock conditional on Python version: we depend on newer code than # in [most] releases of the Python std library. # https://github.com/testing-cabal/mock/issues/487 for 4.0.[0-1] blacklist -mock # BSD +mock!=4.0.0,!=4.0.1 # BSD moto # Apache-2.0 mypy # MIT nodeenv # BSD @@ -261,13 +267,13 @@ pytest-xdist # MIT python-consul # MIT License python-subunit # Apache-2.0/BSD pyzmq # LGPL+BSD -redis # MIT +redis!=4.0.0 # MIT requests-mock # Apache-2.0 -retrying # Apache-2.0 +retrying!=1.3.0 # Apache-2.0 sadisplay # BSD selenium<4.0.0 # Apache-2.0 -stestr # Apache-2.0 -sushy # Apache-2.0 +stestr!=2.3.0,!=3.0.0 # Apache-2.0 +sushy!=1.9.0 # Apache-2.0 tabulate # MIT tenacity # Apache-2.0 testrepository # Apache-2.0/BSD @@ -283,7 +289,7 @@ typing # PSF typing-extensions # PSF tzdata # MIT virtualbmc # Apache-2.0 -virtualenv # MIT +virtualenv!=16.3.0 # MIT WebTest # MIT Werkzeug!=2.2.0 # BSD License whereto # Apache-2.0 @@ -294,13 +300,13 @@ xvfbwrapper #license: MIT ## section:docs -blockdiag # Apache-2.0 +blockdiag!=2.0.0 # Apache-2.0 doc8 # Apache-2.0 pydot # MIT License pydotplus # MIT License Pygments # BSD license rst2txt # BSD -sphinx # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2 # BSD sphinxcontrib-actdiag # BSD sphinxcontrib-apidoc # BSD sphinxcontrib-blockdiag # BSD @@ -325,16 +331,16 @@ ceilometer # Apache-2.0 # ceilometermiddleware might not show up with a search of setup.cfg and # requirements files, but some projects use it via being installed by devstack ceilometermiddleware # Apache-2.0 -cliff # Apache-2.0 +cliff!=2.9.0,!=2.17.0 # Apache-2.0 debtcollector # Apache-2.0 dib-utils # Apache-2.0 -diskimage-builder # Apache-2.0 -etcd3gw # Apache-2.0 +diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1 # Apache-2.0 +etcd3gw!=0.2.2,!=0.2.3,!=0.2.6 # Apache-2.0 futurist # Apache-2.0 -glance-store # Apache-2.0 +glance-store!=0.29.0 # Apache-2.0 heat-translator # Apache-2.0 horizon # Apache-2.0 -ironic-lib # Apache-2.0 +ironic-lib!=4.6.0 # Apache-2.0 keystoneauth1 # Apache-2.0 keystonemiddleware # Apache-2.0 kuryr-lib # Apache-2.0 @@ -355,7 +361,7 @@ neutron-fwaas # Apache-2.0 neutron-lib # Apache-2.0 octavia-lib # Apache-2.0 os-apply-config # Apache-2.0 -os-brick # Apache-2.0 +os-brick!=2.8.0 # Apache-2.0 os-client-config # Apache-2.0 os-collect-config # Apache-2.0 os-ken # Apache-2.0 @@ -363,34 +369,36 @@ os-refresh-config # Apache-2.0 os-resource-classes # Apache-2.0 os-service-types # Apache-2.0 os-traits # Apache-2.0 -os-vif!=3.0.0 # Apache-2.0 +os-vif!=1.8.0,!=1.12.0,!=3.0.0 # Apache-2.0 os-win # Apache-2.0 osc-lib # Apache-2.0 osc-placement # Apache-2.0 -oslo.cache # Apache-2.0 +oslo.cache!=1.31.1,!=2.1.0 # Apache-2.0 oslo.concurrency # Apache-2.0 -oslo.config # Apache-2.0 +oslo.config!=4.3.0,!=4.4.0 # Apache-2.0 oslo.context # Apache-2.0 oslo.db # Apache-2.0 oslo.i18n # Apache-2.0 oslo.limit # Apache-2.0 -oslo.log!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0 -oslo.messaging # Apache-2.0 +oslo.log!=3.44.2,!=4.1.2,!=4.2.0,!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0 +oslo.messaging!=9.0.0 # Apache-2.0 oslo.metrics # Apache-2.0 oslo.middleware # Apache-2.0 -oslo.policy # Apache-2.0 +oslo.policy!=3.0.0,!=3.6.1 # Apache-2.0 oslo.privsep # Apache-2.0 oslo.reports # Apache-2.0 oslo.rootwrap # Apache-2.0 -oslo.serialization # Apache-2.0 -oslo.service # Apache-2.0 +# NOTE(mriedem): oslo.serialization 2.19.1 is blocked for bug 1593641 +oslo.serialization!=2.19.1 # Apache-2.0 +oslo.service!=1.28.1 # Apache-2.0 oslo.upgradecheck # Apache-2.0 -oslo.utils # Apache-2.0 +# NOTE(lajoskatona): oslo.utils version between 3.39.1 and 3.40.1 excluded due to bug 1812922 +oslo.utils!=3.39.1,!=3.40.0,!=3.40.1 # Apache-2.0 oslo.versionedobjects # Apache-2.0 oslo.vmware # Apache-2.0 osprofiler # Apache-2.0 -pbr # Apache-2.0 -stevedore # Apache-2.0 +pbr!=2.1.0 # Apache-2.0 +stevedore!=3.0.0 # Apache-2.0 tap-as-a-service # Apache-2.0 taskflow # Apache-2.0 tempest # Apache-2.0 @@ -409,7 +417,7 @@ gnocchiclient # Apache-2.0 openstacksdk # Apache-2.0 python-barbicanclient # Apache-2.0 python-blazarclient # Apache-2.0 -python-cinderclient # Apache-2.0 +python-cinderclient!=4.0.0 # Apache-2.0 python-cloudkittyclient # Apache-2.0 python-cyborgclient # Apache-2.0 python-designateclient # Apache-2.0 @@ -417,12 +425,12 @@ python-freezerclient # Apache-2.0 python-glanceclient # Apache-2.0 python-heatclient # Apache-2.0 python-ironic-inspector-client # Apache-2.0 -python-ironicclient # Apache-2.0 -python-keystoneclient # Apache-2.0 +python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0 # Apache-2.0 +python-keystoneclient!=2.1.0 # Apache-2.0 python-magnumclient # Apache-2.0 python-manilaclient # Apache-2.0 python-masakariclient # Apache-2.0 -python-mistralclient # Apache-2.0 +python-mistralclient!=3.2.0 # Apache-2.0 python-monascaclient # Apache-2.0 python-muranoclient # Apache-2.0 python-neutronclient # Apache-2.0 @@ -445,7 +453,7 @@ python-zunclient # Apache-2.0 ## ## Docs-related projects under openstack governance -openstackdocstheme # Apache-2.0 +openstackdocstheme!=2.1.0,!=2.1.1 # Apache-2.0 os-api-ref # Apache-2.0 oslosphinx # Apache-2.0 reno # Apache-2.0 @@ -475,7 +483,7 @@ python-linstor # LGPLv3 pywbem # LGPLv2.1+ rsd-lib # Apache-2.0 storops # Apache-2.0 -storpool # Apache-2.0 +storpool!=5.2.0,!=5.3.0 # Apache-2.0 storpool.spopenstack # Apache-2.0 ## section:internal @@ -502,7 +510,7 @@ extras # MIT jsonpath-rw # Apache-2.0 jsonpath-rw-ext # Apache-2.0 kafka-python # Apache-2.0 -oauth2client # Apache-2.0 +oauth2client!=4.0.0 # Apache-2.0 pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5' # MIT # pysnmp library is not maintained since 4 years, it is # not recommended to use it, use its fork pysnmp-lextudio instead diff --git a/tools/lint.py b/tools/lint.py index e30edfde31..7ac319bf56 100755 --- a/tools/lint.py +++ b/tools/lint.py @@ -12,20 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from concurrent import futures -import datetime import os -import sys - -from packaging import requirements -import requests GLOBAL_REQS = os.path.join( os.path.dirname(os.path.realpath(__file__)), '..', 'global-requirements.txt', ) -MAX_EXCLUDE_AGE = datetime.timedelta(365 * 2) def sort() -> None: @@ -82,120 +75,5 @@ def sort() -> None: fh.write(dep) -def validate_excludes( - name: str, specifiers: requirements.SpecifierSet -) -> tuple[str, str]: - data = requests.get(f'https://pypi.org/pypi/{name}/json').json() - latest_release = max(data['releases']) - - result = [] - for specifier in specifiers: - if specifier.operator != '!=': - result.append(specifier) - # non-exclusion specifier - continue - - exclude = specifier.version - - if exclude == latest_release: - print( - f'Release {exclude} is the latest release for package {name}. ' - f'Skipping checks.' - ) - result.append(specifier) - continue - - release = data['releases'].get(exclude) - if not release: - print( - f'Failed to find release {exclude} for package {name}', - file=sys.stderr, - ) - continue - - if all(r['yanked'] for r in release): - print(f'Release {exclude} for package {name} was yanked') - continue - - now = datetime.datetime.now(datetime.timezone.utc) - age = min( - (now - datetime.datetime.fromisoformat(r['upload_time_iso_8601'])) - for r in release - ) - if age >= MAX_EXCLUDE_AGE: - print( - f'Release {exclude} for package {name} is older than the ' - f'upper limit for age ' - f'({age.days} days >= {MAX_EXCLUDE_AGE.days} days)' - ) - continue - - # exclude is recent enough and not yanked so keep it - result.append(specifier) - - return name, ','.join(sorted(str(r) for r in result)) - - -def remove_old_excludes(): - """Remove excludes for old package versions. - - If we exclude e.g. v1.22 of a package but that version was release over 2 - years ago and said package is currently at v1.45, then there's no reason to - keep that exclude around. - """ - deps: dict[str, set[str]] = {} - - with open(GLOBAL_REQS) as fh: - for line in fh.readlines(): - if not line.strip() or line.startswith('#'): - # ignore blank lines and comments - continue - - req = requirements.Requirement(line.split(' #')[0]) - - if req.name in ('setuptools',): - # ignore certain packages where we want to retain all excludes - continue - - # these shouldn't be in our global-requirements file so we don't - # handle them...but make sure - assert not req.extras, f'unexpected extras: {req}' - assert not req.url, f'unexpected url: {req}' - - if any(s.operator == '!=' for s in req.specifier): - deps[req.name] = req.specifier - - with futures.ThreadPoolExecutor() as executor: - res = executor.map(validate_excludes, *zip(*deps.items())) - - deps.update(dict(res)) - - with open(GLOBAL_REQS) as fh: - data = fh.read() - - with open(GLOBAL_REQS, 'w') as fh: - for i, line in enumerate(data.split('\n')): - if i != 0: - fh.write('\n') - - if line.startswith('#') or not line.strip(): - # skipped (empty or comment) - fh.write(line) - continue - - dep, comment, license = line.partition(' #') - req = requirements.Requirement(dep) - if req.name not in deps: - # skipped (no cap) - fh.write(line) - continue - - req.specifier = deps[req.name] - # requirements.Requirement.__str__ adds a space after the semicolon - # which we don't want - fh.write(str(req).replace('; ', ';') + comment + license) - - if __name__ == '__main__': - remove_old_excludes() sort()