openstack/requirements
Code Issues Proposed changes

Revert "Add ability to strip old excludes"

Browse Source 
This reverts commit 314734e938.

Reason for revert:
This change caused failures in requirement-check jobs in multiple repos
because old excludes are still kept in their requirements files. There
are multiple feedbacks about removing old excludes and most of these
are negative because of concern with removing our old knowledge about
know bad versions.

Also global requirements have had to be the central place to find know
bad versions but clean up requires distros to look at every single
repository to get full list of known bad versions.

Given these negative feedbacks, it's better to revert the change now,
and discuss further maintenance policy before we re-do this clean up.

Change-Id: I8e010ad56ddcafdad3a05dce4630d7d158fdc418
This commit is contained in:
Takashi Kajinami 2024-05-02 02:40:33 +09:00
parent fccd0b4f8c
commit cf1e5cae45
2 changed files with 80 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
global-requirements.txt
View File

@ -1,14 +1,14 @@
## section:general ## section:general
alembic # MIT alembic!=1.2.0,!=1.6.3 # MIT
amqp # BSD amqp!=2.1.4,!=5.0.4 # BSD
ansible-runner # Apache 2.0 ansible-runner!=1.3.5 # Apache 2.0
appdirs # MIT License appdirs # MIT License
apscheduler # MIT License apscheduler # MIT License
autobahn # MIT License autobahn # MIT License
automaton # Apache-2.0 automaton # Apache-2.0
autopage # Apache-2.0 autopage # Apache-2.0
Babel # BSD Babel!=2.4.0 # BSD
bcrypt # Apache-2.0 bcrypt # Apache-2.0
beautifulsoup4 # MIT beautifulsoup4 # MIT
betamax # Apache-2.0 betamax # Apache-2.0
@ -16,15 +16,15 @@ boto # MIT
boto3 # Apache-2.0 boto3 # Apache-2.0
botocore # Apache-2.0 botocore # Apache-2.0
cachetools # MIT License cachetools # MIT License
cassandra-driver # Apache-2.0 cassandra-driver!=3.6.0 # Apache-2.0
cffi # MIT cffi # MIT
cmd2 # MIT cmd2!=0.8.3 # MIT
confluent-kafka # Apache-2.0 confluent-kafka!=1.4.0 # Apache-2.0
confspirator # Apache-2.0 confspirator # Apache-2.0
construct # MIT construct # MIT
cotyledon # Apache-2.0 cotyledon # Apache-2.0
croniter # MIT License croniter # MIT License
cryptography # BSD/Apache-2.0 cryptography!=2.0 # BSD/Apache-2.0
cursive # Apache-2.0 cursive # Apache-2.0
datetimerange # MIT datetimerange # MIT
decorator # BSD decorator # BSD
@ -34,25 +34,27 @@ django-debreach # BSD
django-formtools # BSD django-formtools # BSD
django-pyscss # BSD License (2 clause) django-pyscss # BSD License (2 clause)
Django<4.3 # BSD Django<4.3 # BSD
dnspython # http://www.dnspython.org/LICENSE # eventlet is not compatibile with 2.0.0: https://github.com/eventlet/eventlet/issues/619
dogpile.cache!=1.1.7 # BSD dnspython!=2.0.0,!=2.2.0 # http://www.dnspython.org/LICENSE
dogpile.cache!=0.9.1,!=1.1.7 # BSD
dogtag-pki # LGPLv3+ dogtag-pki # LGPLv3+
dulwich # Apache-2.0 dulwich!=0.19.3,!=0.19.7 # Apache-2.0
edgegrid-python # Apache-2.0 edgegrid-python # Apache-2.0
elasticsearch<3.0.0 # Apache-2.0 elasticsearch<3.0.0 # Apache-2.0
enmerkar # BSD enmerkar # BSD
# NOTE: New versions of eventlet should not be accepted lightly # NOTE: New versions of eventlet should not be accepted lightly
# as they have earned a reputation of frequently breaking things. # as they have earned a reputation of frequently breaking things.
eventlet!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,!=0.32.0,!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT
exabgp # BSD exabgp!=4.0.6 # BSD
falcon # Apache-2.0 falcon # Apache-2.0
fasteners # Apache-2.0 # https://github.com/harlowja/fasteners/issues/36
Flask # BSD fasteners!=0.15,!=0.16 # Apache-2.0
Flask!=0.11 # BSD
Flask-RESTful # BSD Flask-RESTful # BSD
GitPython # BSD License (3 clause) GitPython # BSD License (3 clause)
google-api-python-client # Apache-2.0 google-api-python-client # Apache-2.0
graphviz # MIT License graphviz!=0.5.0 # MIT License
greenlet # MIT greenlet!=0.4.14 # MIT
gunicorn # MIT gunicorn # MIT
httplib2 # MIT httplib2 # MIT
httpx # BSD httpx # BSD
@ -62,7 +64,7 @@ icalendar # BSD
# newer code than in [most] releases of the Python std library. # newer code than in [most] releases of the Python std library.
importlib-metadata # Apache-2.0 importlib-metadata # Apache-2.0
infinisdk # BSD-3 infinisdk # BSD-3
influxdb # MIT influxdb!=5.3.0 # MIT
influxdb-client # MIT influxdb-client # MIT
infoblox-client # Apache-2.0 infoblox-client # Apache-2.0
iso8601 # MIT iso8601 # MIT
@ -70,33 +72,33 @@ jaeger-client # Apache-2.0
Jinja2 # BSD License (3 clause) Jinja2 # BSD License (3 clause)
jira # BSD License (2 clause) jira # BSD License (2 clause)
jmespath # MIT jmespath # MIT
jsonpatch # BSD jsonpatch!=1.20 # BSD
jsonschema # MIT jsonschema # MIT
kazoo # Apache-2.0 kazoo # Apache-2.0
kombu # BSD kombu!=4.0.2 # BSD
kubernetes # Apache-2.0 kubernetes # Apache-2.0
ldap3 # LGPLv3 ldap3 # LGPLv3
libvirt-python # LGPLv2+ libvirt-python!=4.1.0,!=4.2.0 # LGPLv2+
lxml # BSD lxml!=3.7.0 # BSD
Mako # MIT Mako # MIT
msgpack # Apache-2.0 msgpack # Apache-2.0
munch # MIT munch # MIT
ncclient # Apache-2.0 ncclient # Apache-2.0
netaddr # BSD netaddr # BSD
netifaces # MIT netifaces!=0.10.0,!=0.10.1 # MIT
netmiko # MIT netmiko # MIT
networkx!=2.8.4 # BSD networkx!=2.8.4 # BSD
oauthlib # BSD oauthlib # BSD
opentelemetry-exporter-otlp # Apache-2.0 opentelemetry-exporter-otlp # Apache-2.0
opentelemetry-sdk # Apache-2.0 opentelemetry-sdk # Apache-2.0
ovs # Apache-2.0 ovs # Apache-2.0
packaging # Apache-2.0 packaging!=20.5,!=20.6,!=20.7 # Apache-2.0
paramiko # LGPLv2.1+ paramiko!=2.9.0,!=2.9.1 # LGPLv2.1+
passlib # BSD passlib # BSD
Paste # MIT Paste # MIT
PasteDeploy # MIT PasteDeploy # MIT
pecan # BSD pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,!=1.4.0 # BSD
pexpect # ISC License pexpect!=3.3 # ISC License
pika # BSD pika # BSD
Pillow # PIL License Pillow # PIL License
Pint # BSD Pint # BSD
@ -105,27 +107,27 @@ PrettyTable!=3.4.0 # BSD
prometheus-client # Apache-2.0 prometheus-client # Apache-2.0
protobuf # BSD License (3 clause) protobuf # BSD License (3 clause)
psutil # BSD psutil # BSD
pyasn1 # BSD pyasn1!=0.2.3 # BSD
pyasn1-lextudio # BSD pyasn1-lextudio # BSD
pyasn1-modules # BSD pyasn1-modules # BSD
pyasn1-modules-lextudio # BSD pyasn1-modules-lextudio # BSD
pycadf # Apache-2.0 pycadf!=2.0.0 # Apache-2.0
pycdlib # LGPLv2+ pycdlib # LGPLv2+
PyECLib # BSD PyECLib # BSD
pyghmi # Apache-2.0 pyghmi!=1.4.0,!=1.5.11 # Apache-2.0
PyJWT # MIT PyJWT # MIT
pykmip # Apache 2.0 License pykmip # Apache 2.0 License
pylxd # Apache-2.0 pylxd # Apache-2.0
pymemcache # Apache 2.0 License pymemcache!=1.3.0 # Apache 2.0 License
PyMI;sys_platform=='win32' # Apache 2.0 License PyMI;sys_platform=='win32' # Apache 2.0 License
pymongo # Apache-2.0 pymongo!=3.1 # Apache-2.0
PyMySQL # MIT License PyMySQL # MIT License
pyngus # Apache-2.0 pyngus # Apache-2.0
pyOpenSSL # Apache-2.0 pyOpenSSL # Apache-2.0
pyparsing # MIT pyparsing # MIT
pyroute2!=0.7.1;sys_platform != "win32" # Apache-2.0 (+ dual licensed GPL2) pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
pysaml2 # Apache-2.0 pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0
pyScss # MIT License pyScss!=1.3.5 # MIT License
pysnmp-lextudio # BSD pysnmp-lextudio # BSD
pystache # MIT pystache # MIT
# Only required for sasl/binary protocol # Only required for sasl/binary protocol
@ -140,7 +142,7 @@ pywinrm # MIT
PyYAML # MIT PyYAML # MIT
pyzabbix # LGPL pyzabbix # LGPL
rbd-iscsi-client # Apache-2.0 rbd-iscsi-client # Apache-2.0
requests # Apache-2.0 requests!=2.20.0,!=2.24.0 # Apache-2.0
requests-aws # BSD License (3 clause) requests-aws # BSD License (3 clause)
requests-kerberos # ISC requests-kerberos # ISC
requestsexceptions # Apache-2.0 requestsexceptions # Apache-2.0
@ -148,14 +150,18 @@ rfc3986 # Apache-2.0
Routes # MIT Routes # MIT
rtslib-fb # Apache-2.0 rtslib-fb # Apache-2.0
ruamel.yaml # MIT ruamel.yaml # MIT
salt # Apache-2.0 salt!=2019.2.1,!=2019.2.2 # Apache-2.0
scikit-learn # BSD scikit-learn # BSD
scipy # BSD scipy # BSD
# https://github.com/holgern/py-scrypt/issues/16 # https://github.com/holgern/py-scrypt/issues/16
scrypt!=0.8.21 # BSD scrypt!=0.8.21 # BSD
semantic-version # BSD semantic-version # BSD
setproctitle # BSD setproctitle # BSD
SQLAlchemy # MIT # NOTE(yamahata):
# bug work around of sqlalchemy
# https://bitbucket.org/zzzeek/sqlalchemy/issues/3952/
# The fix which is in git master branch is planned for 1.1.9
SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
sqlalchemy-filters # Apache-2.0 sqlalchemy-filters # Apache-2.0
sqlalchemy-migrate # Apache-2.0 sqlalchemy-migrate # Apache-2.0
SQLAlchemy-Utils # BSD License SQLAlchemy-Utils # BSD License
@ -215,7 +221,7 @@ XStatic-Spin # MIT License
XStatic-term.js # MIT License XStatic-term.js # MIT License
XStatic-tv4 # MIT XStatic-tv4 # MIT
# NOTE(anilvenkata): This is required for profiling oslo.service processes # NOTE(anilvenkata): This is required for profiling oslo.service processes
Yappi # MIT Yappi!=0.98,!=0.99 # MIT
zeroconf # LGPL zeroconf # LGPL
zipp # MIT zipp # MIT
zstd # BSD License (2 clause) zstd # BSD License (2 clause)
@ -224,7 +230,7 @@ zVMCloudConnector;sys_platform!='win32' # Apache 2.0 License
## section:testing ## section:testing
bashate # Apache-2.0 bashate # Apache-2.0
coverage # Apache-2.0 coverage!=4.4 # Apache-2.0
ddt # MIT ddt # MIT
django-nose # BSD django-nose # BSD
docker # Apache-2.0 docker # Apache-2.0
@ -239,7 +245,7 @@ ldappool # MPL
# Do not make mock conditional on Python version: we depend on newer code than # Do not make mock conditional on Python version: we depend on newer code than
# in [most] releases of the Python std library. # in [most] releases of the Python std library.
# https://github.com/testing-cabal/mock/issues/487 for 4.0.[0-1] blacklist # https://github.com/testing-cabal/mock/issues/487 for 4.0.[0-1] blacklist
mock # BSD mock!=4.0.0,!=4.0.1 # BSD
moto # Apache-2.0 moto # Apache-2.0
mypy # MIT mypy # MIT
nodeenv # BSD nodeenv # BSD
@ -261,13 +267,13 @@ pytest-xdist # MIT
python-consul # MIT License python-consul # MIT License
python-subunit # Apache-2.0/BSD python-subunit # Apache-2.0/BSD
pyzmq # LGPL+BSD pyzmq # LGPL+BSD
redis # MIT redis!=4.0.0 # MIT
requests-mock # Apache-2.0 requests-mock # Apache-2.0
retrying # Apache-2.0 retrying!=1.3.0 # Apache-2.0
sadisplay # BSD sadisplay # BSD
selenium<4.0.0 # Apache-2.0 selenium<4.0.0 # Apache-2.0
stestr # Apache-2.0 stestr!=2.3.0,!=3.0.0 # Apache-2.0
sushy # Apache-2.0 sushy!=1.9.0 # Apache-2.0
tabulate # MIT tabulate # MIT
tenacity # Apache-2.0 tenacity # Apache-2.0
testrepository # Apache-2.0/BSD testrepository # Apache-2.0/BSD
@ -283,7 +289,7 @@ typing # PSF
typing-extensions # PSF typing-extensions # PSF
tzdata # MIT tzdata # MIT
virtualbmc # Apache-2.0 virtualbmc # Apache-2.0
virtualenv # MIT virtualenv!=16.3.0 # MIT
WebTest # MIT WebTest # MIT
Werkzeug!=2.2.0 # BSD License Werkzeug!=2.2.0 # BSD License
whereto # Apache-2.0 whereto # Apache-2.0
@ -294,13 +300,13 @@ xvfbwrapper #license: MIT
## section:docs ## section:docs
blockdiag # Apache-2.0 blockdiag!=2.0.0 # Apache-2.0
doc8 # Apache-2.0 doc8 # Apache-2.0
pydot # MIT License pydot # MIT License
pydotplus # MIT License pydotplus # MIT License
Pygments # BSD license Pygments # BSD license
rst2txt # BSD rst2txt # BSD
sphinx # BSD sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2 # BSD
sphinxcontrib-actdiag # BSD sphinxcontrib-actdiag # BSD
sphinxcontrib-apidoc # BSD sphinxcontrib-apidoc # BSD
sphinxcontrib-blockdiag # BSD sphinxcontrib-blockdiag # BSD
@ -325,16 +331,16 @@ ceilometer # Apache-2.0
# ceilometermiddleware might not show up with a search of setup.cfg and # ceilometermiddleware might not show up with a search of setup.cfg and
# requirements files, but some projects use it via being installed by devstack # requirements files, but some projects use it via being installed by devstack
ceilometermiddleware # Apache-2.0 ceilometermiddleware # Apache-2.0
cliff # Apache-2.0 cliff!=2.9.0,!=2.17.0 # Apache-2.0
debtcollector # Apache-2.0 debtcollector # Apache-2.0
dib-utils # Apache-2.0 dib-utils # Apache-2.0
diskimage-builder # Apache-2.0 diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1 # Apache-2.0
etcd3gw # Apache-2.0 etcd3gw!=0.2.2,!=0.2.3,!=0.2.6 # Apache-2.0
futurist # Apache-2.0 futurist # Apache-2.0
glance-store # Apache-2.0 glance-store!=0.29.0 # Apache-2.0
heat-translator # Apache-2.0 heat-translator # Apache-2.0
horizon # Apache-2.0 horizon # Apache-2.0
ironic-lib # Apache-2.0 ironic-lib!=4.6.0 # Apache-2.0
keystoneauth1 # Apache-2.0 keystoneauth1 # Apache-2.0
keystonemiddleware # Apache-2.0 keystonemiddleware # Apache-2.0
kuryr-lib # Apache-2.0 kuryr-lib # Apache-2.0
@ -355,7 +361,7 @@ neutron-fwaas # Apache-2.0
neutron-lib # Apache-2.0 neutron-lib # Apache-2.0
octavia-lib # Apache-2.0 octavia-lib # Apache-2.0
os-apply-config # Apache-2.0 os-apply-config # Apache-2.0
os-brick # Apache-2.0 os-brick!=2.8.0 # Apache-2.0
os-client-config # Apache-2.0 os-client-config # Apache-2.0
os-collect-config # Apache-2.0 os-collect-config # Apache-2.0
os-ken # Apache-2.0 os-ken # Apache-2.0
@ -363,34 +369,36 @@ os-refresh-config # Apache-2.0
os-resource-classes # Apache-2.0 os-resource-classes # Apache-2.0
os-service-types # Apache-2.0 os-service-types # Apache-2.0
os-traits # Apache-2.0 os-traits # Apache-2.0
os-vif!=3.0.0 # Apache-2.0 os-vif!=1.8.0,!=1.12.0,!=3.0.0 # Apache-2.0
os-win # Apache-2.0 os-win # Apache-2.0
osc-lib # Apache-2.0 osc-lib # Apache-2.0
osc-placement # Apache-2.0 osc-placement # Apache-2.0
oslo.cache # Apache-2.0 oslo.cache!=1.31.1,!=2.1.0 # Apache-2.0
oslo.concurrency # Apache-2.0 oslo.concurrency # Apache-2.0
oslo.config # Apache-2.0 oslo.config!=4.3.0,!=4.4.0 # Apache-2.0
oslo.context # Apache-2.0 oslo.context # Apache-2.0
oslo.db # Apache-2.0 oslo.db # Apache-2.0
oslo.i18n # Apache-2.0 oslo.i18n # Apache-2.0
oslo.limit # Apache-2.0 oslo.limit # Apache-2.0
oslo.log!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0 oslo.log!=3.44.2,!=4.1.2,!=4.2.0,!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0
oslo.messaging # Apache-2.0 oslo.messaging!=9.0.0 # Apache-2.0
oslo.metrics # Apache-2.0 oslo.metrics # Apache-2.0
oslo.middleware # Apache-2.0 oslo.middleware # Apache-2.0
oslo.policy # Apache-2.0 oslo.policy!=3.0.0,!=3.6.1 # Apache-2.0
oslo.privsep # Apache-2.0 oslo.privsep # Apache-2.0
oslo.reports # Apache-2.0 oslo.reports # Apache-2.0
oslo.rootwrap # Apache-2.0 oslo.rootwrap # Apache-2.0
oslo.serialization # Apache-2.0 # NOTE(mriedem): oslo.serialization 2.19.1 is blocked for bug 1593641
oslo.service # Apache-2.0 oslo.serialization!=2.19.1 # Apache-2.0
oslo.service!=1.28.1 # Apache-2.0
oslo.upgradecheck # Apache-2.0 oslo.upgradecheck # Apache-2.0
oslo.utils # Apache-2.0 # NOTE(lajoskatona): oslo.utils version between 3.39.1 and 3.40.1 excluded due to bug 1812922
oslo.utils!=3.39.1,!=3.40.0,!=3.40.1 # Apache-2.0
oslo.versionedobjects # Apache-2.0 oslo.versionedobjects # Apache-2.0
oslo.vmware # Apache-2.0 oslo.vmware # Apache-2.0
osprofiler # Apache-2.0 osprofiler # Apache-2.0
pbr # Apache-2.0 pbr!=2.1.0 # Apache-2.0
stevedore # Apache-2.0 stevedore!=3.0.0 # Apache-2.0
tap-as-a-service # Apache-2.0 tap-as-a-service # Apache-2.0
taskflow # Apache-2.0 taskflow # Apache-2.0
tempest # Apache-2.0 tempest # Apache-2.0
@ -409,7 +417,7 @@ gnocchiclient # Apache-2.0
openstacksdk # Apache-2.0 openstacksdk # Apache-2.0
python-barbicanclient # Apache-2.0 python-barbicanclient # Apache-2.0
python-blazarclient # Apache-2.0 python-blazarclient # Apache-2.0
python-cinderclient # Apache-2.0 python-cinderclient!=4.0.0 # Apache-2.0
python-cloudkittyclient # Apache-2.0 python-cloudkittyclient # Apache-2.0
python-cyborgclient # Apache-2.0 python-cyborgclient # Apache-2.0
python-designateclient # Apache-2.0 python-designateclient # Apache-2.0
@ -417,12 +425,12 @@ python-freezerclient # Apache-2.0
python-glanceclient # Apache-2.0 python-glanceclient # Apache-2.0
python-heatclient # Apache-2.0 python-heatclient # Apache-2.0
python-ironic-inspector-client # Apache-2.0 python-ironic-inspector-client # Apache-2.0
python-ironicclient # Apache-2.0 python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0 # Apache-2.0
python-keystoneclient # Apache-2.0 python-keystoneclient!=2.1.0 # Apache-2.0
python-magnumclient # Apache-2.0 python-magnumclient # Apache-2.0
python-manilaclient # Apache-2.0 python-manilaclient # Apache-2.0
python-masakariclient # Apache-2.0 python-masakariclient # Apache-2.0
python-mistralclient # Apache-2.0 python-mistralclient!=3.2.0 # Apache-2.0
python-monascaclient # Apache-2.0 python-monascaclient # Apache-2.0
python-muranoclient # Apache-2.0 python-muranoclient # Apache-2.0
python-neutronclient # Apache-2.0 python-neutronclient # Apache-2.0
@ -445,7 +453,7 @@ python-zunclient # Apache-2.0
## ##
## Docs-related projects under openstack governance ## Docs-related projects under openstack governance
openstackdocstheme # Apache-2.0 openstackdocstheme!=2.1.0,!=2.1.1 # Apache-2.0
os-api-ref # Apache-2.0 os-api-ref # Apache-2.0
oslosphinx # Apache-2.0 oslosphinx # Apache-2.0
reno # Apache-2.0 reno # Apache-2.0
@ -475,7 +483,7 @@ python-linstor # LGPLv3
pywbem # LGPLv2.1+ pywbem # LGPLv2.1+
rsd-lib # Apache-2.0 rsd-lib # Apache-2.0
storops # Apache-2.0 storops # Apache-2.0
storpool # Apache-2.0 storpool!=5.2.0,!=5.3.0 # Apache-2.0
storpool.spopenstack # Apache-2.0 storpool.spopenstack # Apache-2.0
## section:internal ## section:internal
@ -502,7 +510,7 @@ extras # MIT
jsonpath-rw # Apache-2.0 jsonpath-rw # Apache-2.0
jsonpath-rw-ext # Apache-2.0 jsonpath-rw-ext # Apache-2.0
kafka-python # Apache-2.0 kafka-python # Apache-2.0
oauth2client # Apache-2.0 oauth2client!=4.0.0 # Apache-2.0
pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5' # MIT pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5' # MIT
# pysnmp library is not maintained since 4 years, it is # pysnmp library is not maintained since 4 years, it is
# not recommended to use it, use its fork pysnmp-lextudio instead # not recommended to use it, use its fork pysnmp-lextudio instead

122
tools/lint.py
View File

@ -12,20 +12,13 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from concurrent import futures
import datetime
import os import os
import sys
from packaging import requirements
import requests
GLOBAL_REQS = os.path.join( GLOBAL_REQS = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.path.dirname(os.path.realpath(__file__)),
'..', '..',
'global-requirements.txt', 'global-requirements.txt',
) )
MAX_EXCLUDE_AGE = datetime.timedelta(365 * 2)
def sort() -> None: def sort() -> None:
@ -82,120 +75,5 @@ def sort() -> None:
fh.write(dep) fh.write(dep)
def validate_excludes(
name: str, specifiers: requirements.SpecifierSet
) -> tuple[str, str]:
data = requests.get(f'https://pypi.org/pypi/{name}/json').json()
latest_release = max(data['releases'])
result = []
for specifier in specifiers:
if specifier.operator != '!=':
result.append(specifier)
# non-exclusion specifier
continue
exclude = specifier.version
if exclude == latest_release:
print(
f'Release {exclude} is the latest release for package {name}. '
f'Skipping checks.'
)
result.append(specifier)
continue
release = data['releases'].get(exclude)
if not release:
print(
f'Failed to find release {exclude} for package {name}',
file=sys.stderr,
)
continue
if all(r['yanked'] for r in release):
print(f'Release {exclude} for package {name} was yanked')
continue
now = datetime.datetime.now(datetime.timezone.utc)
age = min(
(now - datetime.datetime.fromisoformat(r['upload_time_iso_8601']))
for r in release
)
if age >= MAX_EXCLUDE_AGE:
print(
f'Release {exclude} for package {name} is older than the '
f'upper limit for age '
f'({age.days} days >= {MAX_EXCLUDE_AGE.days} days)'
)
continue
# exclude is recent enough and not yanked so keep it
result.append(specifier)
return name, ','.join(sorted(str(r) for r in result))
def remove_old_excludes():
"""Remove excludes for old package versions.
If we exclude e.g. v1.22 of a package but that version was release over 2
years ago and said package is currently at v1.45, then there's no reason to
keep that exclude around.
"""
deps: dict[str, set[str]] = {}
with open(GLOBAL_REQS) as fh:
for line in fh.readlines():
if not line.strip() or line.startswith('#'):
# ignore blank lines and comments
continue
req = requirements.Requirement(line.split(' #')[0])
if req.name in ('setuptools',):
# ignore certain packages where we want to retain all excludes
continue
# these shouldn't be in our global-requirements file so we don't
# handle them...but make sure
assert not req.extras, f'unexpected extras: {req}'
assert not req.url, f'unexpected url: {req}'
if any(s.operator == '!=' for s in req.specifier):
deps[req.name] = req.specifier
with futures.ThreadPoolExecutor() as executor:
res = executor.map(validate_excludes, *zip(*deps.items()))
deps.update(dict(res))
with open(GLOBAL_REQS) as fh:
data = fh.read()
with open(GLOBAL_REQS, 'w') as fh:
for i, line in enumerate(data.split('\n')):
if i != 0:
fh.write('\n')
if line.startswith('#') or not line.strip():
# skipped (empty or comment)
fh.write(line)
continue
dep, comment, license = line.partition(' #')
req = requirements.Requirement(dep)
if req.name not in deps:
# skipped (no cap)
fh.write(line)
continue
req.specifier = deps[req.name]
# requirements.Requirement.__str__ adds a space after the semicolon
# which we don't want
fh.write(str(req).replace('; ', ';') + comment + license)
if __name__ == '__main__': if __name__ == '__main__':
remove_old_excludes()
sort() sort()