From b2607979ab25ceaf1353c34149cec9e35af57329 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 25 Mar 2024 09:38:48 -0700 Subject: [PATCH] Update etherpad to v2.0.3 This updates changes how Etherpad is built and how authentication is managed for API requests. This ends up changing a lot of our tooling around etherpad but etherpad itself (other than the auth changes) doesn't seem to change much. In response to this I update our admin docs on common api tasks to use the new process. Then update our testinfra testing as well to cover that to ensure it all continues to work properly after this change. Note the Dockerfile updates are all adapted from upstream. I'm actually not fond of the decisions they have made in this image build, but being in sync is probably more important than fixing the multistage builds and being different. This change jumps us from v1.9.7 to 2.0.3 (covers releases 2.0.0, 2.0.1, and 2.0.2 too). A changelog can be found here: https://github.com/ether/etherpad-lite/blob/v2.0.3/CHANGELOG.md Change-Id: Ia7c4f26d893b4fc4a178262e1a6b9f3fa80d2a5c --- doc/source/etherpad.rst | 33 +++-- docker/etherpad/Dockerfile | 109 ++++++++++++---- docker/etherpad/settings.json.docker | 23 +++- .../roles/etherpad/templates/settings.json.j2 | 22 +++- .../host_vars/etherpad99.opendev.org.yaml.j2 | 1 + testinfra/test_etherpad.py | 118 ++++++++++++++++-- zuul.d/docker-images/etherpad.yaml | 1 + 7 files changed, 257 insertions(+), 50 deletions(-) diff --git a/doc/source/etherpad.rst b/doc/source/etherpad.rst index 0b776c144c..547e2eb67a 100644 --- a/doc/source/etherpad.rst +++ b/doc/source/etherpad.rst @@ -39,19 +39,34 @@ Manual Administrative Tasks The following sections describe tasks that individuals with root access may need to perform on rare occasions. +All interaction with the Etherpad admin API requires retrieving a bearer +token for authorization and authentication to the API. The secret used +to authenticate when requesting a token is found in the Etherpad +settings.json file under the ``client_secret`` key:: + + grep '"client_secret":' /etc/etherpad/settings.json | \ + sed -e 's/\s\+"client_secret": "\(.*\)",/\1/' + +With this secret we can make a request to get a token:: + + curl --data grant_type=client_credentials \ + --data client_id=api_admin \ + --data client_secret="SECRETHERE" \ + http://localhost:9001/oidc/token + +This will return a Bearer token that is valid for one hour. You will need +this token to perform the actions described below. + Deleting a Pad -------------- On occasion it may be necessary to delete a pad, so as to redact sensitive or illegal data posted to it (the revision history it keeps makes this harder than just clearing the current contents through a -browser). This is fairly easily accomplished via the `HTTP API`_, but -you need the key which is saved in a file on the server so it's easiest -if done when SSH'd into it locally:: +browser). This is fairly easily accomplished via the `HTTP API`_:: - wget -qO- "http://localhost:9001/api/1/deletePad?apikey=$(sudo \ - docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec etherpad \ - cat /opt/etherpad-lite/APIKEY.txt)&padID=XXXXXXXXXX" ; echo + curl -H "Authorization: Bearer TOKENFROMABOVEPROCESS" \ + "http://localhost:9001/api/1/deletePad?padID=XXXXXXXXXX" ; echo ...where XXXXXXXXXX is the pad's name as it appears at the end of its URL (the trailing echo is just because the API response doesn't end with @@ -75,6 +90,6 @@ on it, not being aware that this changes the document for everyone instead of just locally. Via the revision slider you can identify the last good version and then restore it via the API:: - wget -qO- "http://localhost:9001/api/1.2.11/restoreRevision?apikey=$(sudo \ - docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec etherpad \ - cat /opt/etherpad-lite/APIKEY.txt)&padID=XXXXXXXXXX&rev=NNN" ; echo + curl -H "Authorization: Bearer TOKENFROMABOVEPROCESS" \ + "http://localhost:9001/api/1.2.11/restoreRevision?padID=XXXXXXXXXX&rev=NNN" \ + ; echo diff --git a/docker/etherpad/Dockerfile b/docker/etherpad/Dockerfile index fec5df5d70..ff9ad87024 100644 --- a/docker/etherpad/Dockerfile +++ b/docker/etherpad/Dockerfile @@ -22,15 +22,48 @@ # # Author: muxator -FROM node:20-bookworm-slim +# We set defaults here so that we can make use of them in different +# stages of the multi stage build. +ARG EP_DIR=/opt/etherpad-lite +ARG SETTINGS=./settings.json.docker +ARG ETHERPAD_PLUGINS="ep_headings" + +FROM node:20-bookworm-slim as adminBuild +ARG EP_DIR +WORKDIR "${EP_DIR}" + +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get -qq update && \ + apt-get -qq dist-upgrade && \ + apt-get -qq --no-install-recommends install ca-certificates git && \ + apt-get -qq clean && \ + rm -rf /var/lib/apt/lists/* +RUN git clone https://github.com/ether/etherpad-lite ${EP_DIR} +RUN git checkout v2.0.3 +RUN cd ./admin && npm install -g pnpm && pnpm install && pnpm run build --outDir ./dist +RUN cd ./ui && pnpm install && pnpm run build --outDir ./dist + + +FROM node:20-bookworm-slim as build LABEL maintainer="infra-root@openstack.org" +# Set these arguments when building the image from behind a proxy +ARG http_proxy= +ARG https_proxy= +ARG no_proxy= + ARG TIMEZONE= + RUN \ [ -z "${TIMEZONE}" ] || { \ ln -sf /usr/share/zoneinfo/"${TIMEZONE#/usr/share/zoneinfo/}" /etc/localtime; \ dpkg-reconfigure -f noninteractive tzdata; \ } +ENV TIMEZONE=${TIMEZONE} + +# Control the configuration file to be copied into the container. +# We bind mount over this file +ARG SETTINGS # plugins to install while building the container. By default no plugins are # installed. @@ -38,7 +71,15 @@ RUN \ # # EXAMPLE: # ETHERPAD_PLUGINS="ep_codepad ep_author_neat" -ARG ETHERPAD_PLUGINS="ep_headings" +ARG ETHERPAD_PLUGINS + +# local plugins to install while building the container. By default no plugins are +# installed. +# If given a value, it has to be a space-separated, quoted list of plugin names. +# +# EXAMPLE: +# ETHERPAD_LOCAL_PLUGINS="../ep_my_plugin ../ep_another_plugin" +ARG ETHERPAD_LOCAL_PLUGINS= # Control whether abiword will be installed, enabling exports to DOC/PDF/ODT formats. # By default, it is not installed. @@ -56,12 +97,6 @@ ARG INSTALL_ABIWORD= # INSTALL_LIBREOFFICE=true ARG INSTALL_SOFFICE= -# By default, Etherpad container is built and run in "production" mode. This is -# leaner (development dependencies are not installed) and runs faster (among -# other things, assets are minified & compressed). -ENV NODE_ENV=production -ENV ETHERPAD_PRODUCTION=true - # Follow the principle of least privilege: run as unprivileged user. # # Running as non-root enables running this image in platforms like OpenShift @@ -73,25 +108,26 @@ ARG EP_HOME= ARG EP_UID=5001 ARG EP_GID=0 ARG EP_SHELL= + RUN groupadd --system ${EP_GID:+--gid "${EP_GID}" --non-unique} etherpad && \ useradd --system ${EP_UID:+--uid "${EP_UID}" --non-unique} --gid etherpad \ ${EP_HOME:+--home-dir "${EP_HOME}"} --create-home \ ${EP_SHELL:+--shell "${EP_SHELL}"} etherpad -ARG EP_DIR=/opt/etherpad-lite +ARG EP_DIR RUN mkdir -p "${EP_DIR}" && chown etherpad:etherpad "${EP_DIR}" # the mkdir is needed for configuration of openjdk-11-jre-headless, see # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 RUN export DEBIAN_FRONTEND=noninteractive; \ mkdir -p /usr/share/man/man1 && \ - npm install npm@6 -g && \ + npm install pnpm -g && \ apt-get -qq update && \ apt-get -qq dist-upgrade && \ apt-get -qq --no-install-recommends install \ ca-certificates \ - git \ curl \ + git \ ${INSTALL_ABIWORD:+abiword} \ ${INSTALL_SOFFICE:+libreoffice} \ && \ @@ -102,28 +138,47 @@ USER etherpad RUN git clone https://github.com/ether/etherpad-lite ${EP_DIR} WORKDIR "${EP_DIR}" -RUN git checkout v1.9.7 +RUN git checkout v2.0.3 + +FROM build as development +ARG ETHERPAD_PLUGINS + +COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/admin/dist ./src/templates/admin +COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/ui/dist ./src/static/oidc + +RUN bin/installDeps.sh && \ + if [ ! -z "${ETHERPAD_PLUGINS}" ] || [ ! -z "${ETHERPAD_LOCAL_PLUGINS}" ]; then \ + pnpm run install-plugins ${ETHERPAD_PLUGINS} ${ETHERPAD_LOCAL_PLUGINS:+--path ${ETHERPAD_LOCAL_PLUGINS}}; \ + fi + + +FROM build as production +ARG EP_DIR +ARG SETTINGS +ARG ETHERPAD_PLUGINS + +ENV NODE_ENV=production +ENV ETHERPAD_PRODUCTION=true + +COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/admin/dist ./src/templates/admin +COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/ui/dist ./src/static/oidc + +RUN bin/installDeps.sh && rm -rf ~/.npm && rm -rf ~/.local && rm -rf ~/.cache && \ + if [ ! -z "${ETHERPAD_PLUGINS}" ] || [ ! -z "${ETHERPAD_LOCAL_PLUGINS}" ]; then \ + pnpm run install-plugins ${ETHERPAD_PLUGINS} ${ETHERPAD_LOCAL_PLUGINS:+--path ${ETHERPAD_LOCAL_PLUGINS}}; \ + fi -# Plugins must be installed before installing Etherpad's dependencies, otherwise -# npm will try to hoist common dependencies by removing them from -# src/node_modules and installing them in the top-level node_modules. As of -# v6.14.10, npm's hoist logic appears to be buggy, because it sometimes removes -# dependencies from src/node_modules but fails to add them to the top-level -# node_modules. Even if npm correctly hoists the dependencies, the hoisting -# seems to confuse tools such as `npm outdated`, `npm update`, and some ESLint -# rules. -RUN { [ -z "${ETHERPAD_PLUGINS}" ] || \ - npm install --no-save ${ETHERPAD_PLUGINS}; } && \ - src/bin/installDeps.sh && \ - rm -rf ~/.npm # Copy the configuration file. -COPY --chown=etherpad:etherpad ./settings.json.docker "${EP_DIR}"/settings.json +COPY --chown=etherpad:etherpad ${SETTINGS} "${EP_DIR}"/settings.json # Fix group permissions RUN chmod -R g=u . -HEALTHCHECK --interval=20s --timeout=3s CMD curl -f http://localhost:9001 || exit 1 +USER etherpad + +HEALTHCHECK --interval=5s --timeout=3s \ + CMD curl --silent http://localhost:9001/health | grep -E "pass|ok|up" > /dev/null || exit 1 EXPOSE 9001 -CMD ["node", "src/node/server.js"] +CMD ["pnpm", "run", "prod"] diff --git a/docker/etherpad/settings.json.docker b/docker/etherpad/settings.json.docker index 79f72f05e5..d969318220 100644 --- a/docker/etherpad/settings.json.docker +++ b/docker/etherpad/settings.json.docker @@ -527,7 +527,7 @@ /* * Restrict socket.io transport methods */ - "socketTransportProtocols" : ["xhr-polling", "jsonp-polling", "htmlfile"], + "socketTransportProtocols" : ["websocket", "polling"], "socketIo": { /* @@ -650,5 +650,24 @@ /* * Enable/Disable case-insensitive pad names. */ - "lowerCasePadIds": "${LOWER_CASE_PAD_IDS:false}" + "lowerCasePadIds": "${LOWER_CASE_PAD_IDS:false}", + "sso": { + "issuer": "${SSO_ISSUER:http://localhost:9001}", + "clients": [ + { + "client_id": "${ADMIN_CLIENT:admin_client}", + "client_secret": "${ADMIN_SECRET:admin}", + "grant_types": ["authorization_code"], + "response_types": ["code"], + "redirect_uris": ["${ADMIN_REDIRECT:http://localhost:9001/admin/}"] + }, + { + "client_id": "${USER_CLIENT:user_client}", + "client_secret": "${USER_SECRET:user}", + "grant_types": ["authorization_code"], + "response_types": ["code"], + "redirect_uris": ["${USER_REDIRECT:http://localhost:9001/}"] + } + ] + } } diff --git a/playbooks/roles/etherpad/templates/settings.json.j2 b/playbooks/roles/etherpad/templates/settings.json.j2 index 0d8748d4cd..ed58da9eb8 100644 --- a/playbooks/roles/etherpad/templates/settings.json.j2 +++ b/playbooks/roles/etherpad/templates/settings.json.j2 @@ -537,7 +537,7 @@ * Restrict socket.io transport methods */ // OpenDev: Add websocket for our Jitsi-Meet integration. - "socketTransportProtocols" : ["websocket", "xhr-polling", "jsonp-polling", "htmlfile"], + "socketTransportProtocols" : ["websocket", "polling"], "socketIo": { /* @@ -660,5 +660,23 @@ /* * Enable/Disable case-insensitive pad names. */ - "lowerCasePadIds": false + "lowerCasePadIds": false, + "sso": { + "issuer": "${SSO_ISSUER:http://localhost:9001}", + "clients": [ + { + "client_id": "api_admin", + "client_secret": "{{ etherpad_admin_api_secret }}", + "grant_types": ["client_credentials"], + "response_types": [], + "redirect_uris": [], + "extraParams": [ + { + "name": "admin", + "value": "true" + } + ] + } + ] + } } diff --git a/playbooks/zuul/templates/host_vars/etherpad99.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/etherpad99.opendev.org.yaml.j2 index 6a3443f0b5..8dbcaffa89 100644 --- a/playbooks/zuul/templates/host_vars/etherpad99.opendev.org.yaml.j2 +++ b/playbooks/zuul/templates/host_vars/etherpad99.opendev.org.yaml.j2 @@ -1,2 +1,3 @@ etherpad_db_root_password: rootpassword etherpad_db_password: password +etherpad_admin_api_secret: YH0FIYlSeTuMab3DKmfCs3Vlb5w9SuU63dGyJW5HZHE371YJ diff --git a/testinfra/test_etherpad.py b/testinfra/test_etherpad.py index 6c28dddef6..f8c22313bf 100644 --- a/testinfra/test_etherpad.py +++ b/testinfra/test_etherpad.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import json import urllib.parse testinfra_hosts = ['etherpad99.opendev.org'] @@ -36,6 +37,19 @@ def test_etherpad_logs(host): assert mariadb_log_file.exists assert mariadb_log_file.contains('mysqld: ready for connections') +def _get_bearer_token(host): + # Get a Beaer token via Oauth2 negotiation for API interaction + cmd = host.run('grep \'"client_secret":\' /etc/etherpad/settings.json |' + ' sed -e \'s/\\s\\+"client_secret": "\\(.*\\)",/\\1/\'') + secret = cmd.stdout.strip() + cmd = host.run('curl --data grant_type=client_credentials ' + '--data client_id=api_admin --data client_secret="%s" ' + 'http://localhost:9001/oidc/token' % secret) + ret = json.loads(cmd.stdout) + assert ret['token_type'] == 'Bearer' + token = ret['access_token'] + return token + def test_etherpad_4_byte_utf8(host): # The 🖖 utf8 character is a four byte character. This test ensures we # can set that value in a pad and get it back out again. This test both @@ -43,18 +57,102 @@ def test_etherpad_4_byte_utf8(host): # utf8 chars teststr = '🖖 Live long and prosper 🖖' urlstr = urllib.parse.quote(teststr) - cmd = host.run('sudo docker-compose -f ' - '/etc/etherpad-docker/docker-compose.yaml ' - 'exec -T etherpad cat /opt/etherpad-lite/APIKEY.txt') - token = cmd.stdout.strip() - cmd = host.run('wget -qO- "http://localhost:9001/api/1/createPad?' - 'apikey=%s&padID=testing"' % token) + + token = _get_bearer_token(host) + + # Test the API works and that 4 byte chars are happy. + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/createPad?' + 'padID=testing"' % token) assert '"code":0' in cmd.stdout assert '"message":"ok"' in cmd.stdout - cmd = host.run('wget -qO- "http://localhost:9001/api/1/setText?' - 'apikey=%s&padID=testing&text=%s"' % (token, urlstr)) + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/setText?' + 'padID=testing&text=%s"' % (token, urlstr)) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/getText?' + 'padID=testing"' % token) assert '"code":0' in cmd.stdout assert '"message":"ok"' in cmd.stdout - cmd = host.run('wget -qO- "http://localhost:9001/api/1/getText?' - 'apikey=%s&padID=testing"' % token) assert teststr in cmd.stdout + +def test_etherpad_delete_pad(host): + urlstr = urllib.parse.quote('test pad delete string') + token = _get_bearer_token(host) + + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/createPad?' + 'padID=testing_delete"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/setText?' + 'padID=testing_delete&text=%s"' % (token, urlstr)) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1.2.1/listAllPads"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + pads = json.loads(cmd.stdout)['data']['padIDs'] + assert 'testing_delete' in pads + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/deletePad?' + 'padID=testing_delete"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1.2.1/listAllPads"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + pads = json.loads(cmd.stdout)['data']['padIDs'] + assert 'testing_delete' not in pads + +def test_etherpad_restore_pad(host): + firststr = urllib.parse.quote('first') + secondstr = urllib.parse.quote('second') + token = _get_bearer_token(host) + + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/createPad?' + 'padID=testing_restore"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/setText?' + 'padID=testing_restore&text=%s"' % (token, firststr)) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/setText?' + 'padID=testing_restore&text=%s"' % (token, secondstr)) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/getText?' + 'padID=testing_restore"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + assert firststr not in cmd.stdout + assert secondstr in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/getRevisionsCount?' + 'padID=testing_restore"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + revisions = json.loads(cmd.stdout)['data']['revisions'] + old_rev = urllib.parse.quote(str(revisions - 1)) + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1.2.11/restoreRevision?' + 'padID=testing_restore&rev=%s"' % (token, old_rev)) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + cmd = host.run('curl -H "Authorization: Bearer %s" ' + '"http://localhost:9001/api/1/getText?' + 'padID=testing_restore"' % token) + assert '"code":0' in cmd.stdout + assert '"message":"ok"' in cmd.stdout + assert firststr in cmd.stdout + assert secondstr not in cmd.stdout diff --git a/zuul.d/docker-images/etherpad.yaml b/zuul.d/docker-images/etherpad.yaml index dbfe4fa7a1..2a87898e1d 100644 --- a/zuul.d/docker-images/etherpad.yaml +++ b/zuul.d/docker-images/etherpad.yaml @@ -7,6 +7,7 @@ vars: ðerpad_vars docker_images: - context: docker/etherpad + target: production repository: opendevorg/etherpad files: ðerpad_files - docker/etherpad/