From e9d96699c7ee7a480b20916d9c5a36ecf1fcc7a8 Mon Sep 17 00:00:00 2001 From: Saul Wold Date: Wed, 21 Aug 2019 15:41:41 -0700 Subject: [PATCH] Add speclint and associated zuul job This adds the scripts for checking RPM Specfiles and a zuul project template to be added to the StarlingX repos. Additionally a tox.ini entry will be required to launch the check via tox. See the tox.ini for the rpm-packaging-lint entry. Depends-On: https://review.opendev.org/677739 Change-Id: I844491aaab34159941e6dd2f1a4528ad5e87f789 Signed-off-by: Saul Wold --- scripts/macros.openstack-singlespec | 878 ++++++++++++++++++++++++++++ scripts/speclint.sh | 94 +++ tox.ini | 19 + zuul.d/jobs.yaml | 8 + zuul.d/project-templates.yaml | 15 +- 5 files changed, 1011 insertions(+), 3 deletions(-) create mode 100644 scripts/macros.openstack-singlespec create mode 100755 scripts/speclint.sh diff --git a/scripts/macros.openstack-singlespec b/scripts/macros.openstack-singlespec new file mode 100644 index 0000000..8f2124e --- /dev/null +++ b/scripts/macros.openstack-singlespec @@ -0,0 +1,878 @@ +# Added from the openstack/rpm-packaging since we need to customize this +# tis_patch_ver +# +%tis_patch_ver 1 + +%prepare_alternative(t:) \ +%define alternative_target %{-t:%{-t*}}%{!-t:%{_bindir}/%1} \ +rm -f %{buildroot}%{alternative_target} \ +alternative_target="%{alternative_target}" \ +if [[ "$alternative_target" == %{_mandir}* ]]; then \ + rm -f %{buildroot}${alternative_target%%%%%{ext_man}} \ + rm -f %{buildroot}%{alternative_target}%{ext_man} \ +fi \ +mkdir -p %{buildroot}%{_sysconfdir}/alternatives \ +touch %{buildroot}%{_sysconfdir}/alternatives/%1 \ +ln -sf %{_sysconfdir}/alternatives/%1 %{buildroot}%{alternative_target} \ +%{nil} + +%install_alternative(s:t:p:n:) \ +%define alternative_name %{-n:%{-n*}}%{!-n:%1} \ +%define alternative_source %{-s:%{-s*}}%{!-s:%{_bindir}/%{alternative_name}} \ +%define alternative_target %{-t:%{-t*}}%{!-t:%2} \ +%define alternative_priority %{-p:%{-p*}}%{!-p:%3} \ +update-alternatives --install \\\ + %{alternative_source} \\\ + %{alternative_name} \\\ + %{alternative_target} \\\ + %{alternative_priority} + +%uninstall_alternative(n:t:) \ +%define alternative_name %{-n:%{-n*}}%{!-n:%1} \ +%define alternative_target %{-t:%{-t*}}%{!-t:%2} \ +if [ ! -e "%{alternative_target}" ]; then \ + update-alternatives --quiet --remove "%{alternative_name}" "%{alternative_target}" \ +fi \ +%{nil} + +%alternative_for() \ +%1 \ +%ghost %{_sysconfdir}/alternatives/%{basename:%1} + +%system_python python2 +%python_for_executables python3 + +##### common functionality ##### + +%_python_sysconfig_path() %(%1 -c "import sysconfig as s; print(s.get_paths().get('%2'))") +%_python_sysconfig_var() %(%1 -c "import sysconfig as s; print(s.get_config_var('%2'))") + +%_rec_macro_helper %{lua: + rpm.define("_rec_macro_helper %{nil}") + function expand_macro(name, args) + local interp = rpm.expand("%python_flavor") + local args = args and rpm.expand(args) or "" + print(rpm.expand("%{" .. interp .. "_" .. name .. " " .. args .."}")) + end + function call_sysconfig(which, interp) + local arg = rpm.expand("%1") + print(rpm.expand("%{_python_sysconfig_" .. which .. " " .. interp .. " " .. arg .. "}")) + end +} + +##### fedora compatibility ##### + +%py_setup setup.py +%py_shbang_opts -s + +##### binary suffixes for flavors ##### + +%python2_bin_suffix %python2_version +%python3_bin_suffix %python3_version +%pypy3_bin_suffix pp%{pypy3_version} + +##### preferred configuration ##### + +%python_sitelib %{_python_sysconfig_path %python_flavor purelib} +%python_sitearch %{_python_sysconfig_path %python_flavor platlib} +%python_version %{_python_sysconfig_var %python_flavor py_version_short} +%python_version_nodots %{_python_sysconfig_var %python_flavor py_version_nodot} + +%python_prefix %{_rec_macro_helper}%{lua:expand_macro("prefix")} +%python_bin_suffix %{_rec_macro_helper}%{lua:expand_macro("bin_suffix")} + +%python_sysconfig_path() %{_rec_macro_helper}%{lua:call_sysconfig("path", "%python_flavor")} +%python_sysconfig_var() %{_rec_macro_helper}%{lua:call_sysconfig("var", "%python_flavor")} + +%python_alternative() %{_rec_macro_helper}%{lua:expand_macro("alternative", "%**")} +%python_install_alternative() %{_rec_macro_helper}%{lua:expand_macro("install_alternative", "%**")} +%python_uninstall_alternative() %{_rec_macro_helper}%{lua:expand_macro("uninstall_alternative", "%**")} + +%py_ver %python_version + +##### macro definitions for flavor "pypy3" ##### + +%__pypy3 /usr/bin/pypy3 + +%pypy3_shbang_opts %py_shbang_opts + +%pypy3_prefix pypy3 +%pypy3_sitelib %{_python_sysconfig_path pypy3 purelib} +%pypy3_sitearch %{_python_sysconfig_path pypy3 platlib} +%pypy3_version %{_python_sysconfig_var pypy3 py_version_short} +%pypy3_version_nodots %{_python_sysconfig_var pypy3 py_version_nodot} + +%pypy3_sysconfig_path() %{_rec_macro_helper}%{lua:call_sysconfig("path", "pypy3")} +%pypy3_sysconfig_var() %{_rec_macro_helper}%{lua:call_sysconfig("var", "pypy3")} + +%ifpypy3 %if "%{python_flavor}" == "pypy3" + +%pypy3_only() %if "%{python_flavor}" == "pypy3" \ +%** \ +%endif + +%pypy3_build \ +%{_python_use_flavor pypy3} \ +%__pypy3 %{py_setup} %{?py_setup_args} build \\\ + --executable="%__pypy3 %pypy3_shbang_opts" + +%pypy3_install \ +%{_python_use_flavor pypy3} \ +%__pypy3 %{py_setup} %{?py_setup_args} install \\\ + -O1 --skip-build --force --root %{buildroot} --prefix %{_prefix} + +%pypy3_alternative() %{_python_macro_init} \ +%{lua:local link, name, path = python_alternative_names(rpm.expand("%1"), rpm.expand("%pypy3_bin_suffix")) \ +print(rpm.expand("%ghost %{_sysconfdir}/alternatives/" .. name .. "\\\n")) \ +print(link .. "\\\n") \ +print(path .. "\\\n") } + +%pypy3_install_alternative() %{_python_macro_init} \ +%{lua:python_install_alternative("pypy3")} + +%pypy3_uninstall_alternative() \ +%{uninstall_alternative -n %1 -t %{_bindir}/%1-%pypy3_bin_suffix} + +##### macro definitions for flavor "python2" ##### + +%__python2 /usr/bin/python2 + +%python2_shbang_opts %py_shbang_opts + +%python2_prefix python2 +%python2_sitelib %{_python_sysconfig_path python2 purelib} +%python2_sitearch %{_python_sysconfig_path python2 platlib} +%python2_version %{_python_sysconfig_var python2 py_version_short} +%python2_version_nodots %{_python_sysconfig_var python2 py_version_nodot} + +%python2_sysconfig_path() %{_rec_macro_helper}%{lua:call_sysconfig("path", "python2")} +%python2_sysconfig_var() %{_rec_macro_helper}%{lua:call_sysconfig("var", "python2")} + +%ifpython2 %if "%{python_flavor}" == "python2" + +%python2_only() %if "%{python_flavor}" == "python2" \ +%** \ +%endif + +%python2_build \ +%{_python_use_flavor python2} \ +%__python2 %{py_setup} %{?py_setup_args} build \\\ + --executable="%__python2 %python2_shbang_opts" + +%python2_install \ +%{_python_use_flavor python2} \ +%__python2 %{py_setup} %{?py_setup_args} install \\\ + -O1 --skip-build --force --root %{buildroot} --prefix %{_prefix} + +%python2_alternative() %{_python_macro_init} \ +%{lua:local link, name, path = python_alternative_names(rpm.expand("%1"), rpm.expand("%python2_bin_suffix")) \ +print(rpm.expand("%ghost %{_sysconfdir}/alternatives/" .. name .. "\\\n")) \ +print(link .. "\\\n") \ +print(path .. "\\\n") } + +%python2_install_alternative() %{_python_macro_init} \ +%{lua:python_install_alternative("python2")} + +%python2_uninstall_alternative() \ +%{uninstall_alternative -n %1 -t %{_bindir}/%1-%python2_bin_suffix} + +##### macro definitions for flavor "python3" ##### + +%__python3 /usr/bin/python3 + +%python3_shbang_opts %py_shbang_opts + +%python3_prefix python3 +%python3_sitelib %{_python_sysconfig_path python3 purelib} +%python3_sitearch %{_python_sysconfig_path python3 platlib} +%python3_version %{_python_sysconfig_var python3 py_version_short} +%python3_version_nodots %{_python_sysconfig_var python3 py_version_nodot} + +%python3_sysconfig_path() %{_rec_macro_helper}%{lua:call_sysconfig("path", "python3")} +%python3_sysconfig_var() %{_rec_macro_helper}%{lua:call_sysconfig("var", "python3")} + +%ifpython3 %if "%{python_flavor}" == "python3" + +%python3_only() %if "%{python_flavor}" == "python3" \ +%** \ +%endif + +%python3_build \ +%{_python_use_flavor python3} \ +%__python3 %{py_setup} %{?py_setup_args} build \\\ + --executable="%__python3 %python3_shbang_opts" + +%python3_install \ +%{_python_use_flavor python3} \ +%__python3 %{py_setup} %{?py_setup_args} install \\\ + -O1 --skip-build --force --root %{buildroot} --prefix %{_prefix} + +%python3_alternative() %{_python_macro_init} \ +%{lua:local link, name, path = python_alternative_names(rpm.expand("%1"), rpm.expand("%python3_bin_suffix")) \ +print(rpm.expand("%ghost %{_sysconfdir}/alternatives/" .. name .. "\\\n")) \ +print(link .. "\\\n") \ +print(path .. "\\\n") } + +%python3_install_alternative() %{_python_macro_init} \ +%{lua:python_install_alternative("python3")} + +%python3_uninstall_alternative() \ +%{uninstall_alternative -n %1 -t %{_bindir}/%1-%python3_bin_suffix} + +##### compatibility short-name macros ##### + +# fedora expects %py_shbang_opts and %pyX_shbang_opts, possibly to be redefinable? +# we expect everything to start with binary name, so we actually use %pythonX_shbang_opts +# so if a specfile redefines the %pyX_, the correct one will be used +%py2_shbang_opts %py_shbang_opts +%python2_shbang_opts %py2_shbang_opts +%py3_shbang_opts %py_shbang_opts +%python3_shbang_opts %py3_shbang_opts + +%py2_build %python2_build +%py2_install %python2_install +%py3_build %python3_build +%py3_install %python3_install + +%py2_ver %python2_version +%py3_ver %python3_version + +%python2_prefix %{?python2_package_prefix}%{?!python2_package_prefix:python} + +%pythons %{?!skip_python2:python2} %{?!skip_python3:python3} + +# This method for generating python_modules gets too deep to expand at about 5 python flavors. +# It is replaced by a Lua macro in macros.lua +# However, OBS has a much higher expansion depth, so this works fine. +%python_module_iter(a:) %{-a*}-%{args} %{expand:%%{?!python_module_iter_%1:%%{python_module_iter -a %*}}} +%python_module_iter_STOP stop +%python_module() %{expand:%%define args %{**}} %{expand:%%{python_module_iter -a %{pythons} STOP}} + +%add_python() %{expand:%%define pythons %pythons %1} + +%python_flavor %{_python_macro_init}%{lua: print(flavor)} + +%if_python_kind() %if "%{python_flavor}" == "%1" +%if_not_python_kind() %if "%{python_flavor}" != "%1" + +%ifpycache %if "%{python_flavor}" != "python2" + +%pycache_only() %ifpycache \ +%** \ +%endif + +%_python_use_flavor() \ +python_flavor=`[ -f _current_flavor ] && cat _current_flavor || true` \ +if [ -z "$python_flavor" ]; then python_flavor="tmp"; fi \ +if [ "$python_flavor" != "%1" ]; then \ + if [ -d build ]; then mv build _build.$python_flavor; fi \ + if [ -d _build.%1 ]; then mv _build.%1 build; fi \ +fi \ +echo %1 > _current_flavor \ +%{nil} + +%_python_stash_flavor() \ +if [ -d build ]; then mv build _build.%1; fi \ +if [ -d _build.tmp ]; then mv _build.tmp build; fi \ +%{nil} + + +### LUA-MACROS ### +%_python_definitions %{lua: +-- declare common functions +function string.startswith(str, prefix) + return str:sub(1, prefix:len()) == prefix +end +function string.endswith(str, suffix) + return str:sub(-suffix:len()) == suffix +end +function string.basename(str) + while true do + local idx = str:find("/") + if not idx then return str end + str = str:sub(idx + 1) + end +end +function lookup_table(tbl) + local result = {} + for _,v in ipairs(tbl) do result[v] = true end + return result +end +-- macro replacements +SHORT_FLAVORS = { + -- ?? + python = "py", + -- ?? + python2 = "py2", + python3 = "py3", + pypy = "pypy", +} +function replace_macros(str, targetflavor) + local LONG_MACROS = { "sitelib", "sitearch", + "alternative", "install_alternative", "uninstall_alternative", + "version", "version_nodots", "bin_suffix", "prefix"} + local SHORT_MACROS = { "ver" } + for _, srcflavor in ipairs({flavor, "python"}) do + str = str:gsub("%%__" .. srcflavor, "%%__" .. targetflavor) + for _, macro in ipairs(LONG_MACROS) do + local from = string.format("%s_%s", srcflavor, macro) + local to = string.format("%s_%s", targetflavor, macro) + str = str:gsub("%%" .. from, "%%" .. to) + str = str:gsub("%%{" .. from .. "}", "%%{" .. to .. "}") + str = str:gsub("%%{" .. from .. "(%s+.-)}", "%%{" .. to .. "%1}") + end + for _, macro in ipairs(SHORT_MACROS) do + local from = string.format("%s_%s", SHORT_FLAVORS[srcflavor], macro) + local to = string.format("%s_%s", SHORT_FLAVORS[targetflavor], macro) + str = str:gsub("%%" .. from, "%%" .. to) + str = str:gsub("%%{" .. from .. "}", "%%{" .. to .. "}") + end + end + return str +end +function package_name(flavor, modname, subpkg, append) + if flavor == "python2" and old_python2 then + flavor = "python" + end + local name = flavor .. "-" .. modname + if subpkg and subpkg ~= "" then + name = name .. "-" .. subpkg + end + if append and append ~= "" then + name = name .. " " .. append + end + return name +end +function pkgname_from_param(param) + if param == modname then + return "" + elseif param:startswith(modname .. "-") then + return param:sub(modname:len() + 2) + else + return "-n " .. param + end +end +-- alternative-related +local bindir = rpm.expand("%{_bindir}") +local mandir = rpm.expand("%{_mandir}") +local ext_man, ext_man_expr +ext_man = rpm.expand("%{ext_man}") +if ext_man == "" then + ext_man_expr = "%.%d$" +else + -- ASSUMPTION: ext_man:startswith(".") + ext_man_expr = "%.%d%" .. ext_man .. "$" +end +function python_alternative_names(arg, binsuffix, keep_path_unmangled) + local link, name, path + name = arg:basename() + local man_ending = arg:match(ext_man_expr) or arg:match("%.%d$") + if arg:startswith("/") then + link = arg + elseif man_ending then + link = mandir .. "/man" .. man_ending:sub(2,2) .. "/" .. arg + else + link = bindir .. "/" .. arg + end + if man_ending then + path = link:sub(1, -man_ending:len()-1) .. "-" .. binsuffix .. man_ending + else + path = link .. "-" .. binsuffix + end + -- now is the time to append ext_man if appropriate + -- "link" and "name" get ext_man always + if ext_man ~= "" and man_ending and not arg:endswith(ext_man) then + link = link .. ext_man + name = name .. ext_man + if not keep_path_unmangled then path = path .. ext_man end + end + return link, name, path +end +function python_install_alternative(flavor) + local prio = rpm.expand("%" .. flavor .. "_version_nodots") + local binsuffix = rpm.expand("%" .. flavor .. "_bin_suffix") + local params = {} + for p in string.gmatch(rpm.expand("%*"), "%S+") do + table.insert(params, p) + end + if #params == 0 then + print("error") + return + end + local link, name, path = python_alternative_names(params[1], binsuffix) + print(string.format("update-alternatives --install %s %s %s %s", link, name, path, prio)) + table.remove(params, 1) + for _, v in ipairs(params) do + print(string.format(" \\\\\\n --slave %s %s %s", python_alternative_names(v, binsuffix))) + end +end +} +%_python_scan_spec() %{lua: \ + local last_python = rpm.expand("%python_for_executables")\ + local insert_last_python = false\ +\ + pythons = {}\ + -- make sure that last_python is the last item in the list\ + for str in string.gmatch(rpm.expand("%pythons"), "%S+") do\ + if str == last_python then\ + insert_last_python = true\ + else\ + table.insert(pythons, str)\ + end\ + end\ + -- ...but check that it is actually in the buildset\ + if insert_last_python then table.insert(pythons, last_python) end\ +\ + modname = rpm.expand("%name")\ + local spec_name_prefix = "python"\ + -- modname from name\ + local name = modname\ + for _,py in ipairs(pythons) do\ + if name:find(py .. "%-") == 1 then\ + spec_name_prefix = py\ + modname = name:sub(py:len() + 2)\ + break\ + end\ + end\ + -- try to match "python-"\ + if name == modname and name:find("python%-") == 1 then\ + spec_name_prefix = "python"\ + modname = name:sub(8)\ + end\ + -- if not found, modname == %name, spec_name_prefix == "python"\ +\ + system_python = rpm.expand("%system_python")\ + -- is the package built for python2 as "python-foo" ?\ + old_python2 = rpm.expand("%python2_prefix") == "python"\ + is_called_python = spec_name_prefix == "python"\ +\ + -- detect `flavor`, used for evaluating %ifmacros\ + if is_called_python then\ + -- either system_python (if found in %pythons)\ + -- or the last entry of %pythons\ + for _,py in ipairs(pythons) do\ + flavor = py\ + if flavor == system_python then break end\ + end\ + else\ + -- specname is something other than "python-", and it is a valid\ + -- python flavor (otherwise spec_name_prefix defaults to "python"\ + -- so `is_called_python` is true), so we use it literally\ + flavor = spec_name_prefix\ + end\ +\ + -- find the spec file\ + specpath = name .. ".spec"\ + local locations = { rpm.expand("%_sourcedir"), rpm.expand("%_specdir"), "." }\ + for _,loc in ipairs(locations) do\ + local filename = loc .. "/" .. specpath\ + if posix.stat(filename, "mode") ~= nil then\ + specpath = filename\ + break\ + end\ + end\ +} + +%python_subpackages() %{lua: \ + rpm.expand("%_python_macro_init")\ + _python_subpackages_emitted = true\ +\ + local current_flavor = flavor\ + local original_flavor = rpm.expand("%python_flavor")\ +\ + -- line processing functions\ + local function print_altered(line)\ + -- set %name macro to proper flavor-name\ + line = line:gsub("%%{?name}?", current_flavor .. "-" .. modname)\ + -- print expanded\ + print(rpm.expand(replace_macros(line, current_flavor)) .. "\\n")\ + end\ +\ + local function ignore_line(line) end\ +\ + local function files_line(line)\ + -- unexpand %license at start of line\ + if line:startswith("%license") then\ + line = "%" .. line\ + end\ + return print_altered(line)\ + end\ +\ + local PROPERTY_COPY_UNMODIFIED = lookup_table { "Epoch:", "Summary:", "Version:", "BuildArch:" }\ + local PROPERTY_COPY_MODIFIED = lookup_table {\ + "Requires:", "Provides:",\ + "Recommends:", "Suggests:",\ + "Conflicts:", "Obsoletes:",\ + "Supplements:", "Enhances:",\ + "%requires_eq", "%requires_ge",\ + "Requires(pre):", "Requires(preun):", "Requires(post):", "Requires(postun):",\ + "Requires(pretrans):", "Requires(posttrans):",\ + }\ +\ + local function process_package_line(line)\ + -- This function processes lines like "Requires: something something".\ + -- "Requires: python-foo" -> "Requires: python3-foo"\ + -- "Requires: %{name} = %{version}" -> "Requires: python3-modname = %{version}"\ + -- "Supplements: packageand(python-a:python-b)" -> "Supplements: packageand(python3-a:python3-b)"\ + -- you get the idea.\ + -- TODO implement %$flavor_only support here?\ +\ + -- first split Property: value\ + local property, value = line:match("^([A-Z%%]%S+)%s*(.*)$")\ +\ + -- "python-foo" -> "python3-foo"\ + local function rename_package(package, flavor)\ + if package == "python" or package == flavor then\ + -- specialcase plain "python"\ + package = current_flavor\ + else\ + package = package:gsub("^" .. flavor .. "(%W)", current_flavor .. "%1")\ + package = package:gsub("^python(%W)", current_flavor .. "%1")\ + end\ + return package\ + end\ +\ + -- split and rewrite "packageand(a:b:c)", using rename_package() for each of a, b, c\ + local function fix_packageand(packageand, flavor)\ + local inner = packageand:match("^packageand%((.*)%)$")\ + if not inner then return packageand end\ + local eat = inner\ + local result = "packageand("\ + while eat do\ + local idx = eat:find(":")\ + local n = ""\ + if idx then\ + n = eat:sub(1, idx)\ + eat = eat:sub(idx+1)\ + else\ + n = eat\ + eat = nil\ + end\ + n = n:gsub("^%s*", "")\ + result = result .. rename_package(n, flavor)\ + end\ + return result .. ")"\ + end\ +\ + if PROPERTY_COPY_UNMODIFIED[property] then\ + print_altered(line)\ + elseif PROPERTY_COPY_MODIFIED[property] then\ + -- specifically handle %name macro before expansion\ + line = line:gsub("%%{?name}?", current_flavor .. "-" .. modname)\ + -- convert value using the appropriate function\ + if value:startswith("packageand") then\ + value = fix_packageand(value, flavor)\ + else\ + value = rename_package(value, flavor)\ + end\ + -- rely on print_altered to perform expansion on the result\ + print_altered(string.format("%s %s", property, value))\ + end\ + end\ +\ + local auto_posttrans = {}\ + local auto_posttrans_current = {}\ + local auto_posttrans_backslash = false\ +\ + local function expect_alternatives(line)\ + if auto_posttrans_backslash then\ + local apc = auto_posttrans_current\ + apc[#apc] = apc[#apc] .. "\\n" .. line\ + auto_posttrans_backslash = line:endswith("\\\\")\ + elseif line:startswith("%python_install_alternative")\ + or line:startswith("%{python_install_alternative") -- "}"\ + or line:startswith("%" .. flavor .. "_install_alternative")\ + or line:startswith("%{" .. flavor .. "_install_alternative") -- "}"\ + then\ + table.insert(auto_posttrans_current, line)\ + auto_posttrans_backslash = line:endswith("\\\\")\ + else\ + auto_posttrans_backslash = false\ + end\ + return print_altered(line)\ + end\ + -- end line processing functions\ +\ + local function print_obsoletes(modname)\ + if current_flavor == "python2" then\ + print(rpm.expand("Obsoletes: python-" .. modname .. " < %{?epoch:%{epoch}:}%{version}-%{release}\\n"))\ + print(rpm.expand("Provides: python-" .. modname .. " = %{?epoch:%{epoch}:}%{version}-%{release}\\n"))\ + end\ + end\ +\ + local function files_headline(flavor, param)\ + if not param then param = "" end\ + local append = param:match("(%-f%s+%S+)")\ + local nof = param:gsub("%-f%s+%S+%s*", "")\ + local python_files = param:match("%%{?python_files}?")\ + local subpkg = param:match("%%{python_files%s*(.-)}")\ + if subpkg then python_files = true end\ +\ + if is_called_python and not python_files then\ + -- kingly hack. but RPM's native %error does not work.\ + local errmsg =\ + 'error: Package with "python-" prefix must not contain unmarked "%files" sections.\\n' ..\ + 'error: Use "%files %python_files" or "%files %{python_files foo} instead.\\n'\ + io.stderr:write(errmsg)\ + print(errmsg)\ + error('Invalid spec file')\ + end\ +\ + local mymodname = nof\ + if python_files then mymodname = subpkg end\ + return "%files -n " .. package_name(flavor, modname, mymodname, append) .. "\\n"\ + end\ +\ + local function section_headline(section, flavor, param)\ + if section == "files" then\ + return files_headline(flavor, param)\ + else\ + return "%" .. section .. " -n " .. package_name(flavor, modname, param) .. "\\n"\ + end\ + end\ +\ + local python2_binsuffix = rpm.expand("%python2_bin_suffix")\ + local function dump_alternatives_posttrans()\ + if not old_python2 and current_flavor == "python2" then\ + for label, value in pairs(auto_posttrans) do\ + if value ~= false then\ + print(section_headline("posttrans", current_flavor, label))\ + for _,line in ipairs(value) do\ + -- RPM needs {} characters in Lua macros to match, so\ + -- this is an opening "{" for this one: ----------v\ + firstarg = line:match("install_alternative%s+([^%s}]+)")\ + if firstarg then\ + local _,_,path = python_alternative_names(firstarg, python2_binsuffix)\ + print(string.format('if [ -e "%s" ]; then\\n', path))\ + print_altered(line)\ + print("fi\\n")\ + end\ + end\ + end\ + end\ + end\ + auto_posttrans = {}\ + end\ +\ + local function should_expect_alternatives(section, param)\ + if old_python2 or current_flavor ~= "python2" then return false end\ + if param == nil then param = "" end\ + if section == "posttrans" then\ + auto_posttrans[param] = false\ + return false\ + end\ + if section == "post" and auto_posttrans[param] ~= false then\ + auto_posttrans_current = {}\ + auto_posttrans[param] = auto_posttrans_current\ + return true\ + end\ + return false\ + end\ +\ + local function match_braces(line)\ + local count = 0\ + for c in line:gmatch(".") do\ + if c == "{" then count = count + 1\ + elseif c == "}" and count > 0 then count = count - 1\ + end\ + end\ + return count == 0\ + end\ +\ + local KNOWN_SECTIONS = lookup_table {"package", "description", "files", "prep",\ + "build", "install", "check", "clean", "pre", "post", "preun", "postun",\ + "pretrans", "posttrans", "changelog"}\ + local COPIED_SECTIONS = lookup_table {"description", "files",\ + "pre", "post", "preun", "postun", "pretrans", "posttrans"}\ +\ + -- before we start, print Provides: python2-modname\ + if is_called_python and old_python2 then\ + print(rpm.expand("Provides: python2-" .. modname .. " = %{?epoch:%{epoch}:}%{version}-%{release}\\n"))\ + end\ +\ + for _,python in ipairs(pythons) do\ + local is_current_flavor = python == flavor\ + -- "python-foo" case:\ + if is_called_python then\ + if old_python2 then\ + -- if we're in old-style package, "python" == "python2"\ + is_current_flavor = python == "python2"\ + else\ + -- else nothing is current flavor, always generate\ + is_current_flavor = false\ + end\ + end\ +\ + current_flavor = python\ +\ + -- rescan spec for each flavor\ + if not is_current_flavor then\ + local spec, err = io.open(specpath, "r")\ + if err then print ("bad spec " .. specpath) return end\ +\ + rpm.define("python_flavor " .. python)\ +\ + local section_function = process_package_line\ + print(section_headline("package", current_flavor, nil))\ + print_obsoletes(modname)\ +\ + while true do\ + -- collect lines until braces match. it's what rpm does, kind of.\ + local eof = false\ + local line = spec:read()\ + if line == nil then break end\ + while not match_braces(line) do\ + local nl = spec:read()\ + if nl == nil then eof = true break end\ + line = line .. "\\n" .. nl\ + end\ + if eof then break end\ + --io.stderr:write(current_flavor .. " >".. tostring(line) .."<\\n")\ +\ + -- match section delimiter\ + local section_noparam = line:match("^%%(%S+)(%s*)$")\ + local section_withparam, param = line:match("^%%(%S+)%s+(.+)$")\ + local newsection = section_noparam or section_withparam\ +\ + if KNOWN_SECTIONS[newsection] then\ + -- enter new section\ + if param and param:startswith("-n") then\ + -- ignore named section\ + section_function = ignore_line\ + elseif newsection == "package" then\ + print(section_headline("package", current_flavor, param))\ + print_obsoletes(modname .. "-" .. param)\ + section_function = process_package_line\ + elseif newsection == "files" and current_flavor == flavor then\ + section_function = ignore_line\ + elseif COPIED_SECTIONS[newsection] then\ + print(section_headline(newsection, current_flavor, param))\ + if should_expect_alternatives(newsection, param) then\ + section_function = expect_alternatives\ + elseif newsection == "files" then\ + section_function = files_line\ + else\ + section_function = print_altered\ + end\ + else\ + section_function = ignore_line\ + end\ + elseif line:startswith("%python_subpackages") then\ + -- ignore\ + elseif line:startswith("%if") then\ + -- RPM handles %if on top level, whole sections can be conditional.\ + -- We must copy the %if declarations always, even if they are part\ + -- of non-copied sections. Otherwise we miss this:\ + -- %files A\ + -- /bin/something\ + -- %if %condition\ + -- %files B\ + -- /bin/otherthing\ + -- %endif\ + print_altered(line)\ + -- We are, however, copying expanded versions. This way, specifically,\ + -- macros like %ifpython3 are evaluated differently in the top-level spec\ + -- itself and in the copied sections.\ + --io.stderr:write(rpm.expand(line) .. "\\n")\ + elseif line:startswith("%else") or line:startswith("%endif") then\ + print(line .. "\\n")\ + --io.stderr:write(line .. "\\n")\ + else\ + section_function(line)\ + end\ + end\ +\ + dump_alternatives_posttrans()\ +\ + spec:close()\ + end\ + end\ +\ + -- restore %python_flavor for further processing\ + rpm.define("python_flavor " .. original_flavor)\ +} + +%python_exec(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-=) %{lua: \ + local args = rpm.expand("%**")\ + print(rpm.expand("%{python_expand %__$python " .. args .. "}"))\ +} + +%python_expand(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-=) %{lua: \ + -- force spec scan\ + rpm.expand("%_python_macro_init")\ + local args = rpm.expand("%**")\ + for _, python in ipairs(pythons) do\ + print(rpm.expand("%{_python_use_flavor " .. python .. "}\\n"))\ + local cmd = replace_macros(args, python)\ + cmd = cmd:gsub("$python", python)\ + print(rpm.expand(cmd .. "\\n"))\ + end\ +} + +%python_build(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-=) %{lua: \ + rpm.expand("%_python_macro_init")\ + for _, python in ipairs(pythons) do\ + print(rpm.expand("%" .. python .. "_build %**"))\ + end\ +} + +%python_install(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-=) %{lua: \ + rpm.expand("%_python_macro_init")\ + for _, python in ipairs(pythons) do\ + print(rpm.expand("%" .. python .. "_install %**"))\ + end\ +} + +%python_files() %{lua: \ + rpm.expand("%_python_macro_init")\ + local nparams = rpm.expand("%#")\ + local param = ""\ + if tonumber(nparams) > 0 then param = rpm.expand("%1") end\ +\ + print("-n " .. package_name(flavor, modname, param))\ +\ + if not _python_subpackages_emitted then\ + print("\\n/%python_subpackages_macro_not_present\\n")\ + io.stderr:write("%python_subpackages macro not present\\n"\ + .. "(To get rid of this error, either add a %python_subpackages macro to preamble "\ + .. "or remove %python_files.\\n")\ + error("%python_subpackages macro not present\\n")\ + end\ +} + +%python_clone(a) %{lua: \ + rpm.expand("%_python_macro_init")\ + local param = rpm.expand("%1")\ + local link, name, path\ + for _, python in ipairs(pythons) do\ + local binsuffix = rpm.expand("%" .. python .. "_bin_suffix")\ + link,name,path = python_alternative_names(param, binsuffix, true)\ + print(rpm.expand(string.format("cp %s %s\\n", param, path)))\ + print(rpm.expand(string.format("sed -ri '1s@#!.*python.*@#!/usr/bin/%s@' %s\\n", python, path)))\ + end\ +\ + -- %python_clone -a\ + if rpm.expand("%{?-a}") == "-a" then\ + local buildroot = rpm.expand("%{buildroot}")\ + if link:startswith(buildroot) then link = link:sub(buildroot:len() + 1) end\ + print(rpm.expand(string.format("%%{prepare_alternative -t %s %s}\\n", link, name)))\ + end\ +} + +%python_module() %{lua: \ + rpm.expand("%_python_macro_init")\ + local params = rpm.expand("%**")\ + for _, python in ipairs(pythons) do\ + if python == "python2" then\ + print(rpm.expand("%python2_prefix") .. "-" .. params)\ + else\ + print(python .. "-" .. params)\ + end\ + print(" ")\ + end\ +} +### LUA-MACROS ### + + +%_python_macro_init %{_python_definitions}%{_python_scan_spec}%{lua: rpm.define("_python_macro_init %{nil}")} + diff --git a/scripts/speclint.sh b/scripts/speclint.sh new file mode 100755 index 0000000..be2a67c --- /dev/null +++ b/scripts/speclint.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# +# Derived from the openstack/rpm-packaging scripts + +set -e + +usage() { + echo "" + echo "Usage: " + echo " Provide lint-like info about specfiles:" + echo " speclint.sh " + exit 1 +} + +if [ ${#@} -eq 0 ]; then + usage +fi + +while getopts "h" o; do + case "${o}" in + *) + usage + ;; + esac +done + +tmpdir=$(mktemp -d) +MAXPROC=4 +if [ -d ../zuul-jobs ]; then + echo "Fetching from zuul-jobs" + cat ../zuul-jobs/scripts/macros.openstack-singlespec > $tmpdir/.rpmmacros +else + echo "Fetching from git" + wget -q -O $tmpdir/.rpmmacros https://opendev.org/openstack/rpm-packaging/raw/branch/master/openstack/openstack-macros/macros.openstack-singlespec + echo "%tis_patch_ver 1" >> $tmpdir/.rpmmacros +fi + +failed=0 +for spec in $@; do + if [ ! -f $spec ]; then + echo "$spec does not exisit, please pass valid RPM specfiles on the cmdline" + failed=1 + continue 1 + fi + echo "Checking $spec" + specname=$(basename $spec) + egrep -q '^Source:' $spec && { + echo "$spec should not have Source: lines. Please use Source0: instead." + failed=1 + } + egrep -q '^%setup' $spec && { + echo "$spec should not use '%setup'. Please use '%autosetup' instead." + failed=1 + } + egrep -q '%{__python[23]}' $spec && { + echo "$spec should not use '%{__python[23]}'. Please use 'python2' or 'python3' instead." + failed=1 + } + + (cd $(dirname $spec); HOME=$tmpdir rpmspec -q --qf "%{VERSION}\n" $specname) >/dev/null || { + echo "$spec does not parse properly. Please check your syntax." + failed=1 + } + + echo "spec-cleaner checking $specname" + # Make a copy to do some fix-ups required by spec-cleaner + cp $spec $tmpdir/$specname + # NOTE(toabctl):spec-cleaner can not ignore epochs currently + sed -i '/^Epoch:.*/d' $tmpdir/$specname + # NOTE(jpena): spec-cleaner wants python2/python3 instead of + # %{__python2}/%{__python3} + # https://github.com/openSUSE/spec-cleaner/issues/173 + sed -i 's/%{__python2}/python2/g' $tmpdir/$specname + sed -i 's/%{__python3}/python3/g' $tmpdir/$specname + spec-cleaner -m -d --no-copyright --diff-prog "diff -uw" \ + $tmpdir/$specname > $tmpdir/$specname.cleaner.diff & + let count+=1 + [[ count -eq $MAXPROC ]] && wait && count=0 +done + +# check if some diffs are available +for specdiff in $tmpdir/*.cleaner.diff; do + if [ -s "$specdiff" ]; then + echo "##### `basename ${specdiff} .cleaner.diff` ##### " + cat $specdiff + failed=1 + fi +done + +rm -rf $tmpdir + +exit $failed diff --git a/tox.ini b/tox.ini index 0626851..5c8593e 100644 --- a/tox.ini +++ b/tox.ini @@ -87,3 +87,22 @@ deps = commands = flake8 +[testenv:rpm-packaging-lint] +basepython = python3 +setenv = + VIRTUAL_ENV={envdir} + LC_ALL=en_US.utf-8 +install_command = pip install -U \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ + {opts} {packages} +deps = -r{toxinidir}/test-requirements.txt +whitelist_externals = bash +commands = + bash -c 'if [ -d {toxinidir}/../zuul-jobs ]; then jobsdir={toxinidir}/../zuul-jobs; \ + else jobsdir={toxinidir}/../stx-zuul-jobs; \ + fi; \ + find . \ + -not \( -type d -name .?\* -prune \) \ + -not \( -type d -name centos -prune \) \ + -type f -name \*.spec \ + -print0 | xargs -t -0 $jobsdir/tools/spec-tools/speclint.sh' diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 4ec098a..0f33d96 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -15,3 +15,11 @@ tox_envlist: linters tox_environment: ANSIBLE_ROLES_PATH: ~/src/opendev.org/starlingx/zuul-jobs/roles:~/src/opendev.org/opendev/base-jobs/roles:~/src/opendev.org/zuul/zuul-jobs/roles:~/src/opendev.org/openstack/openstack-zuul-jobs/roles:~/src/opendev.org/openstack/project-config/roles:~/src/opendev.org/opendev/system-config/roles + +- job: + name: stx-check-specfile + parent: tox + description: Sanity check RPM spec files + vars: + tox_envlist: rpm-packaging-lint + files: .*\.spec diff --git a/zuul.d/project-templates.yaml b/zuul.d/project-templates.yaml index 9e9b4d0..df2f2e3 100644 --- a/zuul.d/project-templates.yaml +++ b/zuul.d/project-templates.yaml @@ -1,7 +1,7 @@ - project-template: name: stx-api-ref-jobs description: | - Build and publish api-ref document for starlingx. + Build and publish api-ref document for StarlingX. check: jobs: - build-openstack-api-ref: @@ -25,7 +25,7 @@ - project-template: name: publish-stx-docs description: | - Runs and publishes documentation to starlingx site. + Runs and publishes documentation to StarlingX site. check: jobs: - openstack-tox-docs @@ -39,7 +39,7 @@ - project-template: name: stx-release-notes-jobs description: | - Runs the release notes test and publish jobs for starlingx. + Runs the release notes test and publish jobs for StarlingX. check: jobs: - build-openstack-releasenotes @@ -49,3 +49,12 @@ post: jobs: - publish-stx-releasenotes + +- project-template: + name: stx-lint-specfile-jobs + description: | + Lint RPM Specfiles for syntax and correctness + check: + jobs: + - stx-check-specfile: + voting: false