Merge branch 'stable-3.2' into stable-3.3

* stable-3.2:
  Add script for incremental reindexing during upgrade
  Add query option allowing administrators to skip visibility filtering

Change-Id: Iaaca54cd3ca811a7a45861c21ad4ef1f45753446

Documentation/rest-api-changes.txt

@@ -150,6 +150,12 @@ limit or a supplied `n` query parameter, the last change object has a
 The `S` or `start` query parameter can be supplied to skip a number
 of changes from the list.
 
+Administrators can use the `skip-visibility` query parameter to skip visibility filtering.
+This can be used to ensure that no changes are missed e.g. when querying for changes which
+need to be reindexed. Without this parameter query results the user has no permission to read
+are filtered out. REST requests with the skip-visibility option are rejected when the current
+user doesn't have the ADMINISTRATE_SERVER capability.
+
 Clients are allowed to specify more than one query by setting the `q`
 parameter multiple times. In this case the result is an array of
 arrays, one per query in the same order the queries were given in.

contrib/reindex/.flake8

@@ -0,0 +1,9 @@
+[flake8]
+max-line-length=100
+ignore=
+    # E203 whitespace before ':'
+    E203,
+    # W503: Line break before binary operator
+    W503,
+    # W504: Line break after binary operator
+    W504

contrib/reindex/.gitignore

@@ -0,0 +1 @@
+changes-to-reindex.list

contrib/reindex/Pipfile

@@ -0,0 +1,19 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pygerrit2 = "*"
+requests = "*"
+tqdm = "*"
+
+[dev-packages]
+flake8 = "*"
+black = "*"
+
+[requires]
+python_version = "3.9"
+
+[pipenv]
+allow_prereleases = true

contrib/reindex/Pipfile.lock

@@ -0,0 +1,248 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "37be5a74a22d0e084ebfe168bfdcd7bcaa87ad7b42be66b1d9fbff5e936ebe72"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.9"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+            ],
+            "version": "==2020.12.5"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==4.0.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
+        },
+        "pbr": {
+            "hashes": [
+                "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9",
+                "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00"
+            ],
+            "markers": "python_version >= '2.6'",
+            "version": "==5.5.1"
+        },
+        "pygerrit2": {
+            "hashes": [
+                "sha256:d12cff5cc514dd61281d997ea86771e7f818030c3d2ef230b25bb14dae7d3f86"
+            ],
+            "index": "pypi",
+            "version": "==2.0.14"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+            ],
+            "index": "pypi",
+            "version": "==2.25.1"
+        },
+        "tqdm": {
+            "hashes": [
+                "sha256:38b658a3e4ecf9b4f6f8ff75ca16221ae3378b2e175d846b6b33ea3a20852cf5",
+                "sha256:d4f413aecb61c9779888c64ddf0c62910ad56dcbe857d8922bb505d4dbff0df1"
+            ],
+            "index": "pypi",
+            "version": "==4.54.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.2"
+        }
+    },
+    "develop": {
+        "appdirs": {
+            "hashes": [
+                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+            ],
+            "version": "==1.4.4"
+        },
+        "black": {
+            "hashes": [
+                "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"
+            ],
+            "index": "pypi",
+            "version": "==20.8b1"
+        },
+        "click": {
+            "hashes": [
+                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
+                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==7.1.2"
+        },
+        "flake8": {
+            "hashes": [
+                "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839",
+                "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"
+            ],
+            "index": "pypi",
+            "version": "==3.8.4"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "pathspec": {
+            "hashes": [
+                "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
+                "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"
+            ],
+            "version": "==0.8.1"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367",
+                "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.6.0"
+        },
+        "pyflakes": {
+            "hashes": [
+                "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92",
+                "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.2.0"
+        },
+        "regex": {
+            "hashes": [
+                "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538",
+                "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4",
+                "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc",
+                "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa",
+                "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444",
+                "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1",
+                "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af",
+                "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8",
+                "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9",
+                "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88",
+                "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba",
+                "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364",
+                "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e",
+                "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7",
+                "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0",
+                "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31",
+                "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683",
+                "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee",
+                "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b",
+                "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884",
+                "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c",
+                "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e",
+                "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562",
+                "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85",
+                "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c",
+                "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6",
+                "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d",
+                "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b",
+                "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70",
+                "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b",
+                "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b",
+                "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f",
+                "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0",
+                "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5",
+                "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5",
+                "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f",
+                "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e",
+                "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512",
+                "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d",
+                "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917",
+                "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"
+            ],
+            "version": "==2020.11.13"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072",
+                "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298",
+                "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "version": "==3.7.4.3"
+        }
+    }
+}

contrib/reindex/README.md

@@ -0,0 +1,63 @@
+# Incremental reindexing during upgrade of large gerrit site
+
+In order to shorten the downtime needed to reindex changes during a
+Gerrit upgrade the following strategy can be used:
+
+- index preparation
+  - create a full consistent backup
+  - note down the timestamp when the backup was created (backup-time)
+  - create a complete copy of the production system from the backup
+  - upgrade this copy to the new Gerrit version
+  - online reindex this copy
+- upgrade of the production system
+  - make system unavailable so that users can't reach it anymore
+    e.g. by changing port numbers (downtime starts)
+  - take a full backup
+  - run
+
+    ``` bash
+    ./reindex.py -u gerrit-url -s backup-time
+    ```
+
+    to write the list of changes which have been created or modified
+    since the backup for the index preparation was created to a file
+    "changes-to-reindex.list"
+  - upgrade the production system to the new gerrit version skipping
+    reindexing
+  - copy the bulk of the new index from the copy system to the
+    production system
+  - run
+
+    ``` bash
+    ./reindex.py -u gerrit-url
+    ```
+
+    this reindexes all changes which have been created or modified after
+    the backup was taken reading these changes from the file
+    "changes-to-reindex.list"
+  - smoketest the system
+  - make the production system available to the users again
+    (downtime ends)
+
+## Online help
+
+For help on all available options run
+
+``` bash
+./reindex -h
+```
+
+## Python environment
+
+Prerequisites:
+
+- python 3.9
+- pipenv
+
+Install virtual python environment and run the script
+
+``` bash
+pipenv sync
+pipenv shell
+./reindex <options>
+```

contrib/reindex/reindex.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser, RawTextHelpFormatter
+from itertools import islice
+import getpass
+import logging
+import os
+
+from pygerrit2 import GerritRestAPI, HTTPBasicAuth, HTTPBasicAuthFromNetrc
+from tqdm import tqdm
+
+EPILOG = """\
+To query the list of changes which have been created or modified since the
+given timestamp and write them to a file "changes-to-reindex.list" run
+$ ./reindex.py -u gerrit-url -s timestamp
+
+To reindex the list of changes in file "changes-to-reindex.list" run
+$ ./reindex.py -u gerrit-url
+"""
+
+
+def _parse_options():
+    parser = ArgumentParser(
+        formatter_class=RawTextHelpFormatter,
+        epilog=EPILOG,
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        dest="url",
+        help="gerrit url",
+    )
+    parser.add_argument(
+        "-s",
+        "--since",
+        dest="time",
+        help=(
+            "changes modified after the given 'TIME', inclusive. Must be in the\n"
+            "format '2006-01-02[ 15:04:05[.890][ -0700]]', omitting the time defaults\n"
+            "to 00:00:00 and omitting the timezone defaults to UTC."
+        ),
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        default="changes-to-reindex.list",
+        dest="file",
+        help=(
+            "file path to store list of changes if --since is given,\n"
+            "otherwise file path to read list of changes from"
+        ),
+    )
+    parser.add_argument(
+        "-c",
+        "--chunk",
+        default=100,
+        dest="chunksize",
+        help="chunk size defining how many changes are reindexed per request",
+        type=int,
+    )
+    parser.add_argument(
+        "--cert",
+        dest="cert",
+        type=str,
+        help="path to file containing custom ca certificates to trust",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbose",
+        action="store_true",
+        help="verbose debugging output",
+    )
+    parser.add_argument(
+        "-n",
+        "--netrc",
+        default=True,
+        dest="netrc",
+        action="store_true",
+        help=(
+            "read credentials from .netrc, default to environment variables\n"
+            "USERNAME and PASSWORD, otherwise prompt for credentials interactively"
+        ),
+    )
+    return parser.parse_args()
+
+
+def _chunker(iterable, chunksize):
+    it = map(lambda s: s.strip(), iterable)
+    while True:
+        chunk = list(islice(it, chunksize))
+        if not chunk:
+            return
+        yield chunk
+
+
+class Reindexer:
+    """Class for reindexing Gerrit changes"""
+
+    def __init__(self):
+        self.options = _parse_options()
+        self._init_logger()
+        credentials = self._authenticate()
+        if self.options.cert:
+            certs = os.path.expanduser(self.options.cert)
+            self.api = GerritRestAPI(
+                url=self.options.url, auth=credentials, verify=certs
+            )
+        else:
+            self.api = GerritRestAPI(url=self.options.url, auth=credentials)
+
+    def _init_logger(self):
+        self.logger = logging.getLogger("Reindexer")
+        self.logger.setLevel(logging.DEBUG)
+        h = logging.StreamHandler()
+        if self.options.verbose:
+            h.setLevel(logging.DEBUG)
+        else:
+            h.setLevel(logging.INFO)
+        formatter = logging.Formatter("%(message)s")
+        h.setFormatter(formatter)
+        self.logger.addHandler(h)
+
+    def _authenticate(self):
+        username = password = None
+        if self.options.netrc:
+            auth = HTTPBasicAuthFromNetrc(url=self.options.url)
+            username = auth.username
+            password = auth.password
+        if not username:
+            username = os.environ.get("USERNAME")
+        if not password:
+            password = os.environ.get("PASSWORD")
+        while not username:
+            username = input("user: ")
+        while not password:
+            password = getpass.getpass("password: ")
+        auth = HTTPBasicAuth(username, password)
+        return auth
+
+    def _query(self):
+        start = 0
+        more_changes = True
+        while more_changes:
+            query = f"since:{self.options.time}&start={start}&skip-visibility"
+            for change in self.api.get(f"changes/?q={query}"):
+                more_changes = change.get("_more_changes") is not None
+                start += 1
+                yield change.get("_number")
+            break
+
+    def _query_to_file(self):
+        self.logger.debug(
+            f"writing changes since {self.options.time} to file {self.options.file}:"
+        )
+        with open(self.options.file, "w") as output:
+            for id in self._query():
+                self.logger.debug(id)
+                output.write(f"{id}\n")
+
+    def _reindex_chunk(self, chunk):
+        self.logger.debug(f"indexing {chunk}")
+        response = self.api.post(
+            "/config/server/index.changes",
+            chunk,
+        )
+        self.logger.debug(f"response: {response}")
+
+    def _reindex(self):
+        self.logger.debug(f"indexing changes from file {self.options.file}")
+        with open(self.options.file, "r") as f:
+            with tqdm(unit="changes", desc="Indexed") as pbar:
+                for chunk in _chunker(f, self.options.chunksize):
+                    self._reindex_chunk(chunk)
+                    pbar.update(len(chunk))
+
+    def execute(self):
+        if self.options.time:
+            self._query_to_file()
+        else:
+            self._reindex()
+
+
+def main():
+    reindexer = Reindexer()
+    reindexer.execute()
+
+
+if __name__ == "__main__":
+    main()

java/com/google/gerrit/server/restapi/change/QueryChanges.java

@@ -27,8 +27,11 @@ import com.google.gerrit.extensions.restapi.TopLevelResource;
 import com.google.gerrit.index.query.QueryParseException;
 import com.google.gerrit.index.query.QueryRequiresAuthException;
 import com.google.gerrit.index.query.QueryResult;
+import com.google.gerrit.server.CurrentUser;
 import com.google.gerrit.server.DynamicOptions;
 import com.google.gerrit.server.change.ChangeJson;
+import com.google.gerrit.server.permissions.GlobalPermission;
+import com.google.gerrit.server.permissions.PermissionBackend;
 import com.google.gerrit.server.permissions.PermissionBackendException;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gerrit.server.query.change.ChangeQueryBuilder;
@@ -49,10 +52,13 @@ public class QueryChanges implements RestReadView<TopLevelResource>, DynamicOpti
   private final ChangeQueryBuilder qb;
   private final Provider<ChangeQueryProcessor> queryProcessorProvider;
   private final HashMap<String, DynamicOptions.DynamicBean> dynamicBeans = new HashMap<>();
+  private final Provider<CurrentUser> userProvider;
+  private final PermissionBackend permissionBackend;
   private EnumSet<ListChangesOption> options;
   private Integer limit;
   private Integer start;
   private Boolean noLimit;
+  private Boolean skipVisibility;
 
   @Option(
       name = "--query",
@@ -94,6 +100,15 @@ public class QueryChanges implements RestReadView<TopLevelResource>, DynamicOpti
     this.noLimit = on;
   }
 
+  @Option(name = "--skip-visibility", usage = "Skip visibility check, only for administrators")
+  public void skipVisibility(boolean on) throws AuthException, PermissionBackendException {
+    if (on) {
+      CurrentUser user = userProvider.get();
+      permissionBackend.user(user).check(GlobalPermission.ADMINISTRATE_SERVER);
+    }
+    skipVisibility = on;
+  }
+
   @Override
   public void setDynamicBean(String plugin, DynamicOptions.DynamicBean dynamicBean) {
     dynamicBeans.put(plugin, dynamicBean);
@@ -103,10 +118,14 @@ public class QueryChanges implements RestReadView<TopLevelResource>, DynamicOpti
   QueryChanges(
       ChangeJson.Factory json,
       ChangeQueryBuilder qb,
-      Provider<ChangeQueryProcessor> queryProcessorProvider) {
+      Provider<ChangeQueryProcessor> queryProcessorProvider,
+      Provider<CurrentUser> userProvider,
+      PermissionBackend permissionBackend) {
     this.json = json;
     this.qb = qb;
     this.queryProcessorProvider = queryProcessorProvider;
+    this.userProvider = userProvider;
+    this.permissionBackend = permissionBackend;
 
     options = EnumSet.noneOf(ListChangesOption.class);
   }
@@ -152,6 +171,9 @@ public class QueryChanges implements RestReadView<TopLevelResource>, DynamicOpti
     if (noLimit != null) {
       queryProcessor.setNoLimit(noLimit);
     }
+    if (skipVisibility != null) {
+      queryProcessor.enforceVisibility(!skipVisibility);
+    }
     dynamicBeans.forEach((p, b) -> queryProcessor.setDynamicBean(p, b));
 
     if (queries == null || queries.isEmpty()) {

javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java

@@ -25,21 +25,27 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.gerrit.acceptance.AbstractDaemonTest;
 import com.google.gerrit.acceptance.NoHttpd;
+import com.google.gerrit.acceptance.PushOneCommit;
 import com.google.gerrit.acceptance.UseClockStep;
 import com.google.gerrit.acceptance.config.GerritConfig;
 import com.google.gerrit.acceptance.testsuite.account.AccountOperations;
 import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.acceptance.testsuite.request.RequestScopeOperations;
+import com.google.gerrit.entities.AccessSection;
 import com.google.gerrit.entities.Account;
 import com.google.gerrit.entities.Patch;
 import com.google.gerrit.entities.Permission;
 import com.google.gerrit.entities.Project;
 import com.google.gerrit.extensions.api.changes.ReviewInput;
 import com.google.gerrit.extensions.common.ChangeInfo;
+import com.google.gerrit.extensions.restapi.AuthException;
 import com.google.gerrit.extensions.restapi.BadRequestException;
 import com.google.gerrit.extensions.restapi.TopLevelResource;
+import com.google.gerrit.server.project.ProjectConfig;
 import com.google.gerrit.server.restapi.change.QueryChanges;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import java.util.Arrays;
 import java.util.List;
 import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
 import org.eclipse.jgit.junit.TestRepository;
@@ -50,6 +56,7 @@ public class QueryChangesIT extends AbstractDaemonTest {
   @Inject private AccountOperations accountOperations;
   @Inject private ProjectOperations projectOperations;
   @Inject private Provider<QueryChanges> queryChangesProvider;
+  @Inject private RequestScopeOperations requestScopeOperations;
 
   @Test
   @SuppressWarnings("unchecked")
@@ -283,9 +290,91 @@ public class QueryChangesIT extends AbstractDaemonTest {
     assertThat(result.get(1).get(0)._number).isEqualTo(numericId2);
   }
 
+  @Test
+  public void skipVisibility_rejectedForNonAdmin() throws Exception {
+    requestScopeOperations.setApiUser(user.id());
+    final QueryChanges queryChanges = queryChangesProvider.get();
+    String query = "is:open repo:" + project.get();
+    queryChanges.addQuery(query);
+    AuthException thrown =
+        assertThrows(AuthException.class, () -> queryChanges.skipVisibility(true));
+    assertThat(thrown).hasMessageThat().isEqualTo("administrate server not permitted");
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void skipVisibility_noReadPermission() throws Exception {
+    createChange().getChangeId();
+    requestScopeOperations.setApiUser(admin.id());
+    QueryChanges queryChanges = queryChangesProvider.get();
+
+    queryChanges.addQuery("is:open repo:" + project.get());
+    List<List<ChangeInfo>> result =
+        (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+    assertThat(result).hasSize(1);
+
+    try (ProjectConfigUpdate u = updateProject(allProjects)) {
+      ProjectConfig cfg = u.getConfig();
+      removeAllBranchPermissions(cfg, Permission.READ);
+      u.save();
+    }
+
+    queryChanges = queryChangesProvider.get();
+    queryChanges.addQuery("is:open repo:" + project.get());
+    List<List<ChangeInfo>> result2 =
+        (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+    assertThat(result2).hasSize(0);
+
+    queryChanges = queryChangesProvider.get();
+    queryChanges.addQuery("is:open repo:" + project.get());
+    queryChanges.skipVisibility(true);
+    List<List<ChangeInfo>> result3 =
+        (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+    assertThat(result3).hasSize(1);
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void skipVisibility_privateChange() throws Exception {
+    TestRepository<InMemoryRepository> userRepo = cloneProject(project, user);
+    PushOneCommit.Result result =
+        pushFactory.create(user.newIdent(), userRepo).to("refs/for/master");
+    requestScopeOperations.setApiUser(user.id());
+    gApi.changes().id(result.getChangeId()).setPrivate(true);
+
+    requestScopeOperations.setApiUser(admin.id());
+    QueryChanges queryChanges = queryChangesProvider.get();
+
+    queryChanges.addQuery("is:open repo:" + project.get());
+    List<List<ChangeInfo>> result2 =
+        (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+    assertThat(result2).hasSize(0);
+
+    queryChanges = queryChangesProvider.get();
+    queryChanges.addQuery("is:open repo:" + project.get());
+    queryChanges.skipVisibility(true);
+    List<List<ChangeInfo>> result3 =
+        (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+    assertThat(result3).hasSize(1);
+  }
+
   private static void assertNoChangeHasMoreChangesSet(List<ChangeInfo> results) {
     for (ChangeInfo info : results) {
       assertThat(info._moreChanges).isNull();
     }
   }
+
+  private static void removeAllBranchPermissions(ProjectConfig cfg, String... permissions) {
+    for (AccessSection s : ImmutableList.copyOf(cfg.getAccessSections())) {
+      if (s.getName().startsWith("refs/heads/")
+          || s.getName().startsWith("refs/for/")
+          || s.getName().equals("refs/*")) {
+        cfg.upsertAccessSection(
+            s.getName(),
+            updatedSection -> {
+              Arrays.stream(permissions).forEach(p -> updatedSection.remove(Permission.builder(p)));
+            });
+      }
+    }
+  }
 }