diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index a891802ac8..629a48dcc7 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: path: synapse @@ -65,7 +65,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 39ddb61918..8ecf8fb723 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 74f9d76437..957e84d21a 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -19,7 +19,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -30,7 +30,7 @@ jobs: mdbook-version: "0.5.2" - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -56,7 +56,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index ec8e8d6daa..a4aa4dfcb5 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -54,7 +54,7 @@ jobs: contents: write steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -76,7 +76,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index e0817698f4..88decb33d9 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 815593ffcd..0b39c5c372 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -93,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: pip install .[all,test] @@ -151,7 +151,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 7d04f53f38..098419980e 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -19,8 +19,8 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: pip install tomli diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 6f4c966cdc..34f86520b0 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event_name == 'push' with: ref: master diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c6b9f60baf..a54722e95b 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,8 +27,8 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - id: set-distros @@ -61,7 +61,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: path: src @@ -70,7 +70,7 @@ jobs: uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set up docker layer caching - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -78,7 +78,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" @@ -129,9 +129,9 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -167,8 +167,8 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 976090fe82..9a8cb12ec6 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -18,8 +18,8 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -44,8 +44,8 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - name: Install PyYAML diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aa6250612b..58974ad7db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,7 @@ jobs: integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }} linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} + golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }} steps: - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter @@ -89,6 +90,13 @@ jobs: - 'poetry.lock' - '.github/workflows/tests.yml' + golangci: + - 'complement/**/*.go' + - 'complement/.golangci.yml' + - 'complement/go.mod' + - 'complement/go.sum' + - '.github/workflows/tests.yml' + linting_readme: - 'README.rst' @@ -98,7 +106,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -118,8 +126,8 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" @@ -128,8 +136,8 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -141,7 +149,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -163,7 +171,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -186,7 +194,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | .mypy_cache @@ -199,7 +207,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -208,11 +216,11 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -226,7 +234,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -245,7 +253,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -263,7 +271,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -299,7 +307,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -312,6 +320,23 @@ jobs: - run: cargo fmt --check + lint-golangci: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.golangci == 'true' }} + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + cache-dependency-path: complement/go.sum + go-version-file: complement/go.mod + + - name: Run golangci-lint + working-directory: ./complement + run: go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.1 run ./... --max-issues-per-linter=0 + # This is to detect issues with the rst file, which can otherwise cause issues # when uploading packages to PyPi. lint-readme: @@ -319,8 +344,8 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -341,6 +366,7 @@ jobs: - lint-clippy-nightly - lint-rust - lint-rustfmt + - lint-golangci - lint-readme runs-on: ubuntu-latest steps: @@ -359,6 +385,7 @@ jobs: lint-clippy-nightly lint-rust lint-rustfmt + lint-golangci lint-readme calculate-test-jobs: @@ -366,8 +393,8 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - id: get-matrix @@ -387,7 +414,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -443,7 +470,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -458,7 +485,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.10" @@ -506,7 +533,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -556,7 +583,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -603,7 +630,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -646,7 +673,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -699,7 +726,7 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -720,7 +747,7 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 85d7be7b34..f6880fef1f 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 1b906f7f44..80984ceeb3 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -115,7 +115,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index f9873b6bbd..1e2937230c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ __pycache__/ /media_store/ /uploads /homeserver-config-overrides.d +tmp/ # For direnv users /.envrc diff --git a/CHANGES.md b/CHANGES.md index 2a769e7ca4..938282f7f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,100 @@ +# Synapse 1.157.2 (2026-07-28) + +This security release addresses several vulnerabilities. + +Please upgrade when you can, particularly if your homeserver participates in open federation +and/or has untrusted local users. + +## Security Fixes + +High severity: + +- Fix [ELEMENTSEC-2026-1071](https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9) +- Fix [ELEMENTSEC-2024-1520](https://github.com/element-hq/synapse/security/advisories/GHSA-rgv2-84w7-5j9p) +- Fix [ELEMENTSEC-2026-1717](https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76) +- Fix [ELEMENTSEC-2026-1721](https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq) +- Fix [ELEMENTSEC-2026-1729](https://github.com/element-hq/synapse/security/advisories/GHSA-cjh7-rcpx-xpf8) +- Fix [ELEMENTSEC-2026-1740](https://github.com/element-hq/synapse/security/advisories/GHSA-6wjm-9p2x-gvpm) + +Moderate severity: + +- Fix [ELEMENTSEC-2026-1714](https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r) +- Fix [ELEMENTSEC-2026-1718](https://github.com/element-hq/synapse/security/advisories/GHSA-r66v-qhwx-8rg4) +- Fix [ELEMENTSEC-2026-1751](https://github.com/element-hq/synapse/security/advisories/GHSA-jhcg-5392-5mjw) + +Low severity: + +- Fix [ELEMENTSEC-2026-1703](https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq) +- Fix [ELEMENTSEC-2026-1760](https://github.com/element-hq/synapse/security/advisories/GHSA-hgcg-p9gx-fq5f) + +## Famedly additions for v1.157.2_1 + +- feat: Add requester as an argument to the third party module callback "check_event_allowed" (Jason Little) + + +# Synapse 1.157.1 (2026-07-22) + +## Bugfixes + +- Fix config regression around falsy `experimental_features` no longer being accepted. ([\#19987](https://github.com/element-hq/synapse/issues/19987)) + + +# Synapse 1.157.0 (2026-07-21) + +No significant changes since 1.157.0rc1. + +Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11570) as this release removes support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`). + + +# Synapse 1.157.0rc1 (2026-07-14) + +## Features + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. ([\#19663](https://github.com/element-hq/synapse/issues/19663)) +- Add before and after time filters to the ['Redact events of a user'](https://element-hq.github.io/synapse/v1.157/admin_api/user_admin_api.html#redact-events-of-a-user) Admin API. ([\#19802](https://github.com/element-hq/synapse/issues/19802)) +- Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). ([\#19808](https://github.com/element-hq/synapse/issues/19808)) +- Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. ([\#19935](https://github.com/element-hq/synapse/issues/19935)) + +## Bugfixes + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. ([\#19826](https://github.com/element-hq/synapse/issues/19826)) +- Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. ([\#19901](https://github.com/element-hq/synapse/issues/19901)) +- Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. + Contributed by @m4us1ne. ([\#19902](https://github.com/element-hq/synapse/issues/19902)) +- Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. ([\#19928](https://github.com/element-hq/synapse/issues/19928)) +- Fix a bug causing device list pruning to skip some rows when the transaction gets retried. ([\#19947](https://github.com/element-hq/synapse/issues/19947)) +- Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. ([\#19948](https://github.com/element-hq/synapse/issues/19948)) +- Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. ([\#19949](https://github.com/element-hq/synapse/issues/19949)) + +## Deprecations and Removals + +- Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). ([\#19895](https://github.com/element-hq/synapse/issues/19895)) + +## Internal Changes + +- Port the synchronous core of client event serialization to Rust. ([\#19837](https://github.com/element-hq/synapse/issues/19837), [\#19922](https://github.com/element-hq/synapse/issues/19922)) +- Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). ([\#19871](https://github.com/element-hq/synapse/issues/19871), [\#19879](https://github.com/element-hq/synapse/issues/19879)) +- Allow Rust code to have database access via Python database connection pool. ([\#19878](https://github.com/element-hq/synapse/issues/19878)) +- Add `golangci-lint` to CI. ([\#19888](https://github.com/element-hq/synapse/issues/19888)) +- Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. ([\#19890](https://github.com/element-hq/synapse/issues/19890)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. ([\#19896](https://github.com/element-hq/synapse/issues/19896)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. ([\#19897](https://github.com/element-hq/synapse/issues/19897)) +- Fix storage type mismatches where values were bound with a type that didn't match their database column. ([\#19911](https://github.com/element-hq/synapse/issues/19911)) +- Speed up deletion of old sliding sync connections by adding an index. ([\#19912](https://github.com/element-hq/synapse/issues/19912)) +- Add note to 3PID email token request unit tests that the endpoint being tested can have an expected, artificial delay of up to 1s. ([\#19916](https://github.com/element-hq/synapse/issues/19916)) +- Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. ([\#19923](https://github.com/element-hq/synapse/issues/19923)) +- Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. ([\#19929](https://github.com/element-hq/synapse/issues/19929)) +- Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). ([\#19936](https://github.com/element-hq/synapse/issues/19936)) +- Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). ([\#19938](https://github.com/element-hq/synapse/issues/19938)) +- Minor presence performance improvements for large servers. ([\#19939](https://github.com/element-hq/synapse/issues/19939)) +- Reduce replication traffic caused by presence. ([\#19941](https://github.com/element-hq/synapse/issues/19941)) +- Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. ([\#19942](https://github.com/element-hq/synapse/issues/19942)) + + + + # Synapse 1.156.0 (2026-07-07) No significant changes since 1.156.0rc1. diff --git a/Cargo.lock b/Cargo.lock index a6d98135cd..7ca2265aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -67,9 +78,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cc" @@ -707,9 +718,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -835,9 +846,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c2ec80932c5c3b2d4fbc578c9b56b2d4502098587edb8bef5b6bfcad43682e" +checksum = "f64083bd3a16a353d9d62335808e8e13d0552d2a2b83fdb084496192dcfa9fcd" dependencies = [ "arc-swap", "log", @@ -981,9 +992,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -1004,9 +1015,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -1319,6 +1330,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "base64", "blake2", "bytes", diff --git a/complement/go.mod b/complement/go.mod index dfd8e73f22..cea013573b 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -51,10 +51,10 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.42.0 // indirect ) diff --git a/complement/go.sum b/complement/go.sum index 325ad216c6..4858bc78c8 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -122,8 +122,8 @@ go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXd golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -134,8 +134,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -146,12 +146,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/complement/tests/oidc_test.go b/complement/tests/oidc_test.go index deabf49950..59d8bb435f 100644 --- a/complement/tests/oidc_test.go +++ b/complement/tests/oidc_test.go @@ -52,6 +52,10 @@ oidc_providers: // `/_matrix/client/v3/login/sso/redirect/oidc-test_provider` endpoint. func TestOIDCProviderUnavailable(t *testing.T) { // Deploy a single homeserver + // + // FIXME: Since we're modifying the homeserver config, this should be using a clean + // deploy that won't affect subsequent tests because Complement will re-use the + // deployment, see https://github.com/element-hq/synapse/issues/19937 deployment := complement.Deploy(t, 1) defer deployment.Destroy(t) diff --git a/debian/changelog b/debian/changelog index c1522f068d..38a6625e8b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,27 @@ +matrix-synapse-py3 (1.157.2) stable; urgency=medium + + * New Synapse release 1.157.2. + + -- Synapse Packaging team Tue, 28 Jul 2026 14:01:25 +0100 + +matrix-synapse-py3 (1.157.1) stable; urgency=medium + + * New synapse release 1.157.1. + + -- Synapse Packaging team Wed, 22 Jul 2026 14:30:55 +0000 + +matrix-synapse-py3 (1.157.0) stable; urgency=medium + + * New synapse release 1.157.0. + + -- Synapse Packaging team Tue, 21 Jul 2026 15:17:02 +0000 + +matrix-synapse-py3 (1.157.0~rc1) stable; urgency=medium + + * New synapse release 1.157.0rc1. + + -- Synapse Packaging team Tue, 14 Jul 2026 21:18:16 +0000 + matrix-synapse-py3 (1.156.0) stable; urgency=medium * New Synapse release 1.156.0. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 26c8556eff..38f8649b44 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1070,14 +1070,21 @@ def generate_worker_files( # Determine the load-balancing upstreams to configure nginx_upstream_config = "" for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items(): + # We use `max_fails=0` to prevent nginx from marking an upstream as unavailable + # after it fails to contact the Synapse worker. Otherwise, if nginx sees a + # Synapse worker as unavailable once, it will be marked as unavailable for 10 + # seconds (`fail_timeout` default). + # + # This is necessary because we use `COMPLEMENT_ENABLE_DIRTY_RUNS` (re-uses + # deployments/homeservers) and we don't want any cross-test pollution from + # stopping/starting homeservers. body = "" if using_unix_sockets: for port in upstream_worker_ports: - body += f" server unix:/run/worker.{port};\n" - + body += f" server unix:/run/worker.{port} max_fails=0;\n" else: for port in upstream_worker_ports: - body += f" server localhost:{port};\n" + body += f" server localhost:{port} max_fails=0;\n" # Add to the list of configured upstreams nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format( diff --git a/docs/admin_api/account_validity.md b/docs/admin_api/account_validity.md index dfa69e515b..fbe1f985b8 100644 --- a/docs/admin_api/account_validity.md +++ b/docs/admin_api/account_validity.md @@ -1,6 +1,6 @@ # Account validity API -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) This API allows a server administrator to manage the validity of an account. To use it, you must enable the account validity feature (under diff --git a/docs/admin_api/register_api.md b/docs/admin_api/register_api.md index e9a235ada5..ffe7d9fa17 100644 --- a/docs/admin_api/register_api.md +++ b/docs/admin_api/register_api.md @@ -1,6 +1,7 @@ # Shared-Secret Registration -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. +Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-register-user) instead. This API allows for the creation of users in an administrative and non-interactive way. This is generally used for bootstrapping a Synapse diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 9e2b756208..44d0985697 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -227,7 +227,7 @@ The following parameters should be set in the URL: - `name` - Is optional and filters to only return users with user ID localparts **or** displaynames that contain this value. - `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users. - Defaults to `true` to include guest users. This parameter is not supported when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) + Defaults to `true` to include guest users. This parameter is not supported when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) - `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results. - `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users. @@ -444,7 +444,7 @@ To unsuspend a user, use the same endpoint with a body of: ## Reset password -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-set-password) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Changes the password of another user. This will automatically log the user out of all their devices. @@ -469,7 +469,7 @@ The parameter `logout_devices` is optional and defaults to `true`. ## Get whether a user is a server administrator or not -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. The api is: @@ -488,7 +488,7 @@ A response body like the following is returned: ## Change whether a user is a server administrator or not -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-promote-admin) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Note that you cannot demote yourself. @@ -910,7 +910,7 @@ delete largest/smallest or newest/oldest files first. ## Login as a user -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use [Personal sessions](https://element-hq.github.io/matrix-authentication-service/topics/authorization.html#personal-sessions-personal-access-tokens) through the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Get an access token that can be used to authenticate as that user. Useful for when admins wish to do actions on behalf of a user. @@ -1512,24 +1512,28 @@ Returns a `404` HTTP status code if no user was found, with a response body like _Added in Synapse 1.72.0._ -## Redact all the events of a user +## Redact events of a user This endpoint allows an admin to redact the events of a given user. There are no restrictions on redactions for a local user. By default, we puppet the user who sent the message to redact it themselves. Redactions for non-local users are issued using the admin user, and will fail in rooms where the admin user is not admin/does not have the specified power level to issue redactions. An option -is provided to override the default and allow the admin to issue the redactions in all cases. +is provided to override the default and allow the admin to issue the redactions in all cases. +There are optional parameters to filter for events that happened in the given time period. The API is ``` POST /_synapse/admin/v1/user//redact { - "rooms": ["!roomid1", "!roomid2"] + "rooms": ["!roomid1", "!roomid2"], + "after_ts": 1779564103728, + "before_ts": 1779564103730 } ``` If an empty list is provided as the key for `rooms`, all events in all the rooms the user is member of will be redacted, otherwise all the events in the rooms provided in the request will be redacted. +If neither `after_ts` nor `before_ts` is provided, events will be redacted regardless of when they happened. If only one parameter is provided, all events occurring on or before/after given time will be redacted. The API starts redaction process running, and returns immediately with a JSON body with a redact id which can be used to query the status of the redaction process: @@ -1557,7 +1561,9 @@ The following JSON body parameters are optional: - `limit` - a limit on the number of the user's events to search for ones that can be redacted (events are redacted newest to oldest) in each room, defaults to 1000 if not provided. - `use_admin` - If set to `true`, the admin user is used to issue the redactions, rather than puppeting the user. Useful when the admin is also the moderator of the rooms that require redactions. Note that the redactions will fail in rooms - where the admin does not have the sufficient power level to issue the redactions. + where the admin does not have the sufficient power level to issue the redactions. +- `after_ts` - Redact only events that were sent at this time or after. Format: milliseconds timestamp. _Added in Synapse 1.157.0._ +- `before_ts` - Redact only events that were sent at this time or before. Format: milliseconds timestamp. _Added in Synapse 1.157.0._ _Added in Synapse 1.116.0._ @@ -1599,5 +1605,3 @@ The following fields are returned in the JSON response body: the corresponding error that caused the redaction to fail _Added in Synapse 1.116.0._ - - diff --git a/docs/upgrade.md b/docs/upgrade.md index 5c69446c91..f32035c72f 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,18 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.157.0 + +## MSC3861 Auth Delegation must be migrated to stable Matrix Authentication Service integration + +Support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`) +has been dropped in this version, in favour of the stable Matrix Authentication Service +integration. + +See [the previous upgrade notes](#stable-integration-with-matrix-authentication-service) +and the [`matrix_authentication_service` section in the Configuration Manual](usage/configuration/config_documentation.md#matrix_authentication_service) +for more information. + # Upgrading to v1.152.0 ## Workers which quarantine media must be stream writers @@ -327,7 +339,8 @@ using these metrics. Support for [Matrix Authentication Service (MAS)](https://github.com/element-hq/matrix-authentication-service) is now stable, with a simplified configuration. This stable integration requires MAS 0.20.0 or later. -The existing `experimental_features.msc3861` configuration option is now deprecated and will be removed in Synapse v1.137.0. +The existing `experimental_features.msc3861` configuration option is now deprecated and will be removed in Synapse v1.157.0. +(*Note*: this previously read v1.137.0 but the removal date was missed.) Synapse deployments already using MAS should now use the new configuration options: diff --git a/docs/usage/administration/admin_api/registration_tokens.md b/docs/usage/administration/admin_api/registration_tokens.md index ba95bcf038..c6cc433e35 100644 --- a/docs/usage/administration/admin_api/registration_tokens.md +++ b/docs/usage/administration/admin_api/registration_tokens.md @@ -1,6 +1,6 @@ # Registration Tokens -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-issue-user-registration-token) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. This API allows you to manage tokens which can be used to authenticate registration requests, as proposed in diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 8fa47ff084..90b001bcfe 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -318,6 +318,24 @@ This setting has the following sub-options: * `include_offline_users_on_sync` (boolean): When clients perform an initial or `full_state` sync, presence results for offline users are not included by default. Setting `include_offline_users_on_sync` to `true` will always include offline users in the results. Defaults to `false`. +* `last_active_granularity` (duration): How long after a user was last active that they are still shown as "currently active" to other users. Larger values reduce the rate of presence updates sent to other users and servers. + + *Added in Synapse 1.156.0.* + + Defaults to `"1m"`. + +* `sync_online_timeout` (duration): How long after a client's last sync request their presence is switched to offline. Clients are expected to keep a sync request open at (almost) all times while online, so this only needs to cover the gap between two consecutive sync requests. Note that if `rc_presence` is set to ratelimit how often syncs can affect presence, this must be greater than the ratelimit's interval or users will incorrectly be marked as offline in between syncs. + + *Added in Synapse 1.156.0.* + + Defaults to `"30s"`. + +* `idle_timeout` (duration): How long after a user was last active that their presence is switched to "unavailable" (idle) while they remain connected. Must be greater than `last_active_granularity`. + + *Added in Synapse 1.156.0.* + + Defaults to `"5m"`. + Example configuration: ```yaml presence: @@ -3954,6 +3972,25 @@ push: jitter_delay: 10s ``` --- +### `push_rules` + +*(object)* Options for push rules + +This setting has the following sub-options: + +* `limits` (object): Limits on the size of push rules that users can have + + This setting has the following sub-options: + + * `rule_count` (integer): This is the total number of push rules that each user can have. Power users may expect to have one push rule per room. Defaults to `10000`. + + * `rule_id_length` (integer): This is the maximum length of a push rule ID, in bytes. Push rule IDs need to be allowed to be at least as long as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit. Defaults to `300`. + + * `rule_size` (integer): This is the maximum size of a push rule's body, in bytes. + The exact mechanism for calculating this size is currently an implementation detail, subject to change. This limit should be treated as a coarse sanity limit rather than something to fine-tune. + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit and a mechanism for calculating it. Defaults to `1024`. +--- ## Rooms Config options relating to rooms. @@ -4326,6 +4363,16 @@ exclude_rooms_from_sync: - '!foo:example.com' ``` --- +### `exclude_rooms_from_presence` + +*(array)* A list of rooms to exclude from presence updates. Presence will not be routed between two users solely because they share one of these rooms. Users who also share a non-excluded room continue to exchange presence as normal. Defaults to `[]`. + +Example configuration: +```yaml +exclude_rooms_from_presence: +- '!foo:example.com' +``` +--- ## Opentracing Configuration options related to Opentracing support. diff --git a/docs/workers.md b/docs/workers.md index d987ff8980..92b606607c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -338,14 +338,6 @@ For multiple workers not handling the SSO endpoints properly, see [#7530](https://github.com/matrix-org/synapse/issues/7530) and [#9427](https://github.com/matrix-org/synapse/issues/9427). -Additionally, when MSC3861 is enabled (`experimental_features.msc3861.enabled` -set to `true`), the following endpoints can be handled by the worker: - - ^/_synapse/admin/v2/users/[^/]+$ - ^/_synapse/admin/v1/username_available$ - ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ - ^/_synapse/admin/v1/users/[^/]+/devices$ - Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) with `client` and `federation` `resources` must be configured in the [`worker_listeners`](usage/configuration/config_documentation.md#worker_listeners) diff --git a/pyproject.toml b/pyproject.toml index 118b03d8aa..cbedbabbc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.156.0" +version = "1.157.2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 162bc98182..612ab09f6d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,6 +22,7 @@ crate-type = ["lib", "cdylib"] name = "synapse.synapse_rust" [dependencies] +async-trait = "0.1.89" anyhow = "1.0.63" base64 = "0.22.1" bytes = "1.6.0" diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs new file mode 100644 index 0000000000..d79d12a83a --- /dev/null +++ b/rust/src/config/mod.rs @@ -0,0 +1,76 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::collections::BTreeSet; + +use pyo3::prelude::*; + +pub mod types; + +/// A Rust-side view of Synapse's Python `HomeServerConfig`. +/// +/// This only mirrors the subset of config that the Rust handlers need, rather +/// than the whole thing. Thanks to `#[derive(FromPyObject)]`, each field is +/// pulled directly off the corresponding attribute of the Python `config` +/// object, so you can populate it in one shot with: +/// ```ignore +/// let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; +/// ``` +#[derive(FromPyObject, Clone)] +pub struct SynapseHomeServerConfig { + pub room: RoomConfig, + pub auth: AuthConfig, + pub server: ServerConfig, + pub experimental: ExperimentalConfig, +} + +#[derive(FromPyObject, Clone)] +pub struct RoomConfig { + pub encryption_enabled_by_default_for_room_presets: BTreeSet, +} + +#[derive(FromPyObject, Clone)] +pub struct AuthConfig { + pub login_via_existing_enabled: bool, +} +#[derive(FromPyObject, Clone)] +pub struct ServerConfig { + pub msc4140_enabled: bool, +} + +#[derive(FromPyObject, Clone)] +pub struct ExperimentalConfig { + pub msc3026_enabled: bool, + pub msc3773_enabled: bool, + pub msc2815_enabled: bool, + pub msc3881_enabled: bool, + pub msc3874_enabled: bool, + pub msc3912_enabled: bool, + pub msc3391_enabled: bool, + pub msc4069_profile_inhibit_propagation: bool, + pub msc4028_push_encrypted_events: bool, + pub msc4108_enabled: bool, + pub msc4108_delegation_endpoint: Option, + pub msc3575_enabled: bool, + pub msc4133_enabled: bool, + pub msc4155_enabled: bool, + pub msc4306_enabled: bool, + pub msc4169_enabled: bool, + pub msc4354_enabled: bool, + pub msc4222_enabled: bool, + pub msc4491_enabled: bool, + pub msc4143_enabled: bool, + pub msc4446_enabled: bool, +} diff --git a/rust/src/config/types.rs b/rust/src/config/types.rs new file mode 100644 index 0000000000..8990b52f65 --- /dev/null +++ b/rust/src/config/types.rs @@ -0,0 +1,60 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::str::FromStr; + +use pyo3::{exceptions::PyAssertionError, prelude::*}; + +/// The presets available when creating a Matrix room according to the Matrix spec: +/// +/// > Preset | join_rules | history_visibility | guest_access | Other +/// > --- | --- | --- | --- | --- +/// > `private_chat` | `invite` | ``shared`` | `can_join` | . +/// > `trusted_private_chat` | `invite` | shared | `can_join` | All invitees are given the same power level as the room creator. +/// > `public_chat` | `public` | `shared` | `forbidden` | . +/// > +/// > *-- https://spec.matrix.org/v1.18/client-server-api/#post_matrixclientv3createroom* +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum RoomCreationPreset { + PrivateChat, + PublicChat, + TrustedPrivateChat, +} + +impl FromStr for RoomCreationPreset { + type Err = PyErr; + + fn from_str(s: &str) -> Result { + Ok(match s { + "private_chat" => RoomCreationPreset::PrivateChat, + "public_chat" => RoomCreationPreset::PublicChat, + "trusted_private_chat" => RoomCreationPreset::TrustedPrivateChat, + other => { + return Err(PyAssertionError::new_err(format!( + "Unknown variant {other:?} does not translate to `RoomCreationPreset`. \ + This is a Synapse programming error." + ))) + } + }) + } +} + +impl<'a, 'py> FromPyObject<'a, 'py> for RoomCreationPreset { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + value.extract::<&str>()?.parse() + } +} diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 91d5a08f7d..62a1ce6b90 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -13,10 +13,17 @@ * */ -use std::{future::Future, sync::OnceLock}; +use std::{ + future::Future, + sync::{Arc, Mutex}, +}; use once_cell::sync::OnceCell; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::{ + create_exception, exceptions::PyException, exceptions::PyRuntimeError, intern, prelude::*, + types::PyCFunction, +}; +use tokio::sync::oneshot; use crate::tokio_runtime::runtime; @@ -51,6 +58,16 @@ fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { .bind(py)) } +/// A reference to the `synapse.logging.context` module. +static LOGGING_CONTEXT_MODULE: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.logging.context` module. +fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(LOGGING_CONTEXT_MODULE + .get_or_try_init(|| py.import("synapse.logging.context").map(Into::into))? + .bind(py)) +} + /// Creates a twisted deferred from the given future, spawning the task on the /// tokio runtime. /// @@ -110,18 +127,161 @@ where make_deferred_yieldable(py, &deferred) } -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); +/// Runs a Python awaitable to completion on the Twisted reactor and resolves +/// with its result. +/// +/// This is the inverse of [`create_deferred`]: where that turns a Rust future +/// into a Twisted `Deferred`, this turns a Python awaitable into a Rust future. +/// +/// Despite returning a future, the awaitable is kicked off in the background running in +/// the Twisted reactor and runs to completion regardless of whether the returned Rust +/// future is ever polled; awaiting it only observes the result. +pub(crate) async fn run_python_awaitable( + reactor: Py, + make_awaitable: F, +) -> PyResult> +where + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, +{ + // Resolves when the awaitable completes; carries the resolved value or error. + let (tx, rx) = oneshot::channel::>>(); + // Shared between the success and error callbacks (only one ever fires). + let sender = Arc::new(Mutex::new(Some(tx))); + + Python::attach(|py| -> PyResult<()> { + // Create some deferred success/error callback functions that we will use to get + // the result from Python to Rust. + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let value = args.get_item(0)?.unbind(); + if let Some(tx) = success_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `success_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Ok(value)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let err = failure_to_pyerr(&args.get_item(0)?); + if let Some(tx) = error_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `error_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Err(err)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + // Wrap `make_awaitable` as a Python callable so we can hand it to + // `run_in_background`, which calls it (in the active logcontext) to produce + // the awaitable it then drives. + let awaitable_factory = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + Ok(make_awaitable(py)?.unbind()) + }, + )? + .unbind(); + + // Create a function that we will run with the Twisted reactor that will drive + // the Python awaitable. + let starter = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + + // We fire-and-forget using `run_in_background`. Re-using + // `run_in_background` also makes sure the awaitable gets run with the + // current logcontext while following the logcontext rules. + // + // FIXME: Currently runs in the sentinel logcontext because we don't manage it here + let deferred = logging_context_module(py)?.call_method1( + intern!(py, "run_in_background"), + (awaitable_factory.bind(py),), + ); + + let deferred = deferred?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(py.None()) + }, + )?; + + reactor + .bind(py) + .call_method1(intern!(py, "callFromThread"), (starter,))?; + + Ok(()) + })?; + + match rx.await { + Ok(result) => result, + Err(_) => Err(PyRuntimeError::new_err( + "run_python_awaitable channel closed before the awaitable completed", + )), + } +} + +/// Convert a Twisted `Failure` (as passed to an Deferred errback) into a [`PyErr`]. +/// +/// A Twisted `Failure` carries the original exception instance in its `.value` +/// attribute, which we re-raise so callers see the real error. If the `Failure` is +/// mangled, we fallback to raising a generic [`PyRuntimeError`] explaining what we saw +/// instead. +fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { + match failure.getattr(intern!(failure.py(), "value")) { + Ok(value) => PyErr::from_value(value), + Err(_) => PyRuntimeError::new_err(format!( + "Expected Python object passed here to be a Twisted `Failure` with a `value` attribute \ + but saw something else: {}", + failure + .str() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".to_owned()), + )), + } +} + +static MAKE_DEFERRED_YIELDABLE: OnceCell> = OnceCell::new(); /// Given a deferred, make it follow the Synapse logcontext rules fn make_deferred_yieldable<'py>( py: Python<'py>, deferred: &Bound<'py, PyAny>, ) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_try_init(|| { + logging_context_module(py)? + .getattr("make_deferred_yieldable") + .map(Into::into) + })?; make_deferred_yieldable .call1(py, (deferred,))? @@ -133,6 +293,8 @@ fn make_deferred_yieldable<'py>( pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { // Make sure we fail early if we can't load some modules defer(py)?; + // We can't check this here because of circular import issues + // logging_context_module(py)?; Ok(()) } diff --git a/rust/src/duration.rs b/rust/src/duration.rs index 6c2e2653d1..6863e3349c 100644 --- a/rust/src/duration.rs +++ b/rust/src/duration.rs @@ -40,6 +40,11 @@ impl SynapseDuration { Self { milliseconds } } + /// Returns the duration as a number of milliseconds. + pub const fn as_millis(&self) -> u64 { + self.milliseconds + } + /// Creates a `SynapseDuration` from a number of hours. pub const fn from_hours(hours: u32) -> Self { // We take a u32 here so that we know the multiplication won't overflow. diff --git a/rust/src/events/constants.rs b/rust/src/events/constants.rs index 811794d48c..51f3b572f9 100644 --- a/rust/src/events/constants.rs +++ b/rust/src/events/constants.rs @@ -74,6 +74,40 @@ pub mod unsigned_field { pub const AGE_TS: &str = "age_ts"; /// Unsigned field: redacted_because pub const REDACTED_BECAUSE: &str = "redacted_because"; + /// Unsigned field: redacted_by + pub const REDACTED_BY: &str = "redacted_by"; + /// Unsigned field: transaction_id + pub const TRANSACTION_ID: &str = "transaction_id"; + /// Unsigned field: org.matrix.msc4140.delay_id + pub const DELAY_ID: &str = "org.matrix.msc4140.delay_id"; + /// Unsigned field: membership (MSC4115) + pub const MEMBERSHIP: &str = "membership"; + /// Unsigned field: msc4354_sticky_duration_ttl_ms (MSC4354) + pub const STICKY_TTL: &str = "msc4354_sticky_duration_ttl_ms"; + /// Unsigned field: io.element.synapse.soft_failed (admin metadata) + pub const SOFT_FAILED: &str = "io.element.synapse.soft_failed"; + /// Unsigned field: io.element.synapse.policy_server_spammy (admin metadata) + pub const POLICY_SERVER_SPAMMY: &str = "io.element.synapse.policy_server_spammy"; + /// Unsigned field: invite_room_state + pub const INVITE_ROOM_STATE: &str = "invite_room_state"; + /// Unsigned field: knock_room_state + pub const KNOCK_ROOM_STATE: &str = "knock_room_state"; + /// Unsigned field: m.relations + pub const M_RELATIONS: &str = "m.relations"; + /// Unsigned field: replaces_state + pub const REPLACES_STATE: &str = "replaces_state"; + /// Unsigned field: prev_content + pub const PREV_CONTENT: &str = "prev_content"; +} + +/// Relation types (the `rel_type` of an `m.relates_to`). +pub mod relation_type { + /// Relation type: m.reference + pub const REFERENCE: &str = "m.reference"; + /// Relation type: m.replace + pub const REPLACE: &str = "m.replace"; + /// Relation type: m.thread + pub const THREAD: &str = "m.thread"; } /// Membership Event Fields diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs index 86023add17..9b6e636e68 100644 --- a/rust/src/events/formats/mod.rs +++ b/rust/src/events/formats/mod.rs @@ -95,9 +95,10 @@ pub use vmsc4242::EventFormatVMSC4242; /// pyclass. /// /// The `signatures` and `unsigned` fields are kept separate from the other -/// fields as they are mutable (and must be deep-copied if the event is cloned). -/// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that -/// the serialised JSON is a single flat object matching the Matrix spec. +/// fields as they are mutable. Use [`FormattedEvent::deep_copy`] when an +/// independently-mutable copy is required. `common_fields` and +/// `specific_fields` are both `#[serde(flatten)]`ed so that the serialised JSON +/// is a single flat object matching the Matrix spec. /// /// Note, deserialization of this struct must not be done from /// [`serde_json::Value`] nor [`pythonize::depythonize`], due to a bug with diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index d61bf0d48c..0778fbfeaa 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -498,6 +498,44 @@ impl EventInternalMetadata { .write() .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) } + + /// The event ID of the redaction event, if this event has been redacted. + pub fn redacted_by(&self) -> PyResult> { + Ok(self.read_inner()?.redacted_by.clone()) + } + + /// The transaction ID, if set when the event was created. + /// + /// The transaction ID comes from the `txn_id` path parameter of the + /// client-server API request used to send the event. + pub fn txn_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_txn_id().map(|s| s.to_owned())) + } + + /// The device ID of the sender, if set. + pub fn device_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_device_id().map(|s| s.to_owned())) + } + + /// The access token ID of the sender, if set. + pub fn token_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_token_id()) + } + + /// The delay ID, set only if the event was a delayed event. + pub fn delay_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_delay_id().map(|s| s.to_owned())) + } + + /// Whether the event has been soft failed. + pub fn soft_failed(&self) -> PyResult { + Ok(self.read_inner()?.is_soft_failed()) + } + + /// Whether the policy server marked this event as spammy. + pub fn policy_server_spammy(&self) -> PyResult { + Ok(self.read_inner()?.get_policy_server_spammy()) + } } /// Helper to convert `None` to an `AttributeError` for a property getter. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index bb4877d482..3143fec9cb 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -17,14 +17,16 @@ use std::{collections::BTreeMap, sync::Arc}; use pyo3::{ exceptions::{PyKeyError, PyTypeError}, + prelude::Borrowed, pyclass, pymethods, types::{ PyAnyMethods, PyIterator, PyList, PyListMethods, PyMapping, PySet, PySetMethods, PyTuple, }, - Bound, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyResult, Python, + Bound, FromPyObject, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyErr, PyResult, Python, }; use pythonize::{depythonize, pythonize}; use serde::{Deserialize, Serialize}; +use serde_json::Value; /// A generic class for representing immutable JSON objects. /// @@ -40,34 +42,46 @@ pub struct JsonObject { object: Arc, serde_json::Value>>, } -#[pymethods] -impl JsonObject { - #[new] - #[pyo3(signature = (content = None))] - fn new<'a, 'py>(content: Option<&'a Bound<'py, PyAny>>) -> PyResult { - let Some(content) = content else { - // If no content is provided, default to an empty object. - return Ok(Self::default()); - }; +// We implement `FromPyObject` to allow `JsonObject` to be used as function +// arguments. +impl<'py> FromPyObject<'_, 'py> for JsonObject { + type Error = PyErr; - if let Ok(content) = content.cast::() { - // If the content is already a JsonObject, we can just clone the - // underlying map (this is safe as the object is immutable). + fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { + // Fast path: already a JsonObject, so just share the underlying map + // (cheap, as it's immutable and behind an `Arc`). + if let Ok(json_obj) = obj.cast::() { return Ok(JsonObject { - object: content.get().object.clone(), + object: json_obj.get().object.clone(), }); } - let Ok(content) = content.cast::() else { - return Err(PyTypeError::new_err("'content' must be a mapping")); - }; - - // Use pythonize to try and convert from a mapping. - let content = depythonize(content)?; - Ok(Self { - object: Arc::new(content), + // Otherwise accept any mapping and convert it via pythonize. Unlike the + // `#[new]` constructor we don't accept `None` here: an absent value is + // represented as `Option` at the field/argument level. + let mapping = obj + .cast::() + .map_err(|_| PyTypeError::new_err("expected a mapping"))?; + let object: BTreeMap, Value> = depythonize(&mapping)?; + Ok(JsonObject { + object: Arc::new(object), }) } +} + +#[pymethods] +impl JsonObject { + #[new] + #[pyo3(signature = (content = None))] + fn new(content: Option<&Bound<'_, PyAny>>) -> PyResult { + match content { + // If no content is provided, default to an empty object. + None => Ok(Self::default()), + // Otherwise reuse the `FromPyObject` path, which accepts an + // existing `JsonObject` or any Python mapping. + Some(content) => JsonObject::extract(content.as_borrowed()), + } + } fn __len__(&self) -> usize { self.object.len() @@ -197,6 +211,29 @@ impl JsonObject { pub fn get_field(&self, key: &str) -> Option<&serde_json::Value> { self.object.get(key) } + + /// Returns a reference to the underlying map of this object's entries. + pub fn as_map(&self) -> &BTreeMap, Value> { + &self.object + } + + /// Whether the object has no entries. + pub fn is_empty(&self) -> bool { + self.object.is_empty() + } + + pub fn iter(&self) -> impl Iterator, &Value)> { + self.object.iter() + } +} + +impl<'a> IntoIterator for &'a JsonObject { + type Item = (&'a Box, &'a serde_json::Value); + type IntoIter = std::collections::btree_map::Iter<'a, Box, serde_json::Value>; + + fn into_iter(self) -> Self::IntoIter { + self.object.as_ref().iter() + } } /// Helper class returned by `JsonObject.keys()` to act as a view into the keys diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 83900e14ba..0d746a2ec8 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -58,6 +58,7 @@ use pyo3::{ wrap_pyfunction, Bound, IntoPyObject, PyAny, PyResult, Python, }; use pythonize::{depythonize, pythonize}; +use serde_json::Value; use crate::events::{ constants::event_type::M_ROOM_MEMBER, @@ -87,6 +88,8 @@ pub mod filter; pub mod formats; pub mod internal_metadata; pub mod json_object; +pub mod relations; +pub mod serialize; pub mod signatures; pub mod unsigned; pub mod utils; @@ -107,9 +110,21 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; child_module.add_function(wrap_pyfunction!(redact_event_py, m)?)?; child_module.add_function(wrap_pyfunction!(redact_event_dict, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::serialize_events, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_raw, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_for_client_v1, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_for_client_v2, m)?)?; + child_module.add_function(wrap_pyfunction!( + serialize::format_event_for_client_v2_without_room_id, + m + )?)?; m.add_submodule(&child_module)?; @@ -129,7 +144,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> /// metadata, rejection reason, and a reference to the room version that /// produced this event). See the module-level docs for the high-level /// design. -#[pyclass(frozen, weakref)] +#[pyclass(frozen, weakref, skip_from_py_object)] pub struct Event { /// The parsed event JSON. parsed_event: FormattedEvent, @@ -593,8 +608,19 @@ impl Event { } } - #[getter] - fn redacts<'py>(&self, py: Python<'py>) -> PyResult>> { + /// Returns the `redacts` field of this event, if it has one. + #[getter(redacts)] + fn redacts_py<'py>(&self, py: Python<'py>) -> PyResult>> { + let value = self.redacts(); + value + .map(|v| pythonize(py, v).map_err(Into::into)) + .transpose() + } +} + +impl Event { + /// Returns the `redacts` field of this event, if it has one. + pub fn redacts(&self) -> Option<&Value> { let common = &self.parsed_event.common_fields; let value = if self.room_version.updated_redaction_rules { common.content.get_field(REDACTS) @@ -602,8 +628,6 @@ impl Event { common.other_fields.get(REDACTS) }; value - .map(|v| pythonize(py, v).map_err(Into::into)) - .transpose() } } diff --git a/rust/src/events/relations.rs b/rust/src/events/relations.rs new file mode 100644 index 0000000000..07cd4f88cf --- /dev/null +++ b/rust/src/events/relations.rs @@ -0,0 +1,144 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +//! The bundled aggregations attached to an event for client serialization. +//! +//! These mirror the Matrix "server-side aggregation" data (references, edits +//! and thread summaries) that is folded into an event's `unsigned.m.relations` +//! section when serializing for clients. They are built by the Python +//! `RelationsHandler` and consumed by [`serialize_events`](crate::events::serialize::serialize_events). +//! +//! The events they reference ([`Event`]) are stored by value rather than as +//! Python handles; cloning an `Event` is cheap (it shares the underlying data +//! behind `Arc`s) and the events are only ever read here. + +use pyo3::{pyclass, pymethods, Py, PyTraverseError, PyVisit}; + +use crate::events::{json_object::JsonObject, Event}; + +/// A thread's bundled summary: its latest event, the number of events in the +/// thread, and whether the requesting user has participated. +#[pyclass(frozen, skip_from_py_object, get_all)] +pub struct ThreadAggregation { + /// The latest event in the thread. + pub latest_event: Py, + /// The total number of events in the thread. + pub count: i64, + /// Whether the requesting user has sent an event to the thread. + pub current_user_participated: bool, +} + +#[pymethods] +impl ThreadAggregation { + #[new] + fn new(latest_event: Py, count: i64, current_user_participated: bool) -> Self { + Self { + latest_event, + count, + current_user_participated, + } + } + + #[getter] + fn latest_event(&self) -> &Py { + &self.latest_event + } + + #[getter] + fn count(&self) -> i64 { + self.count + } + + #[getter] + fn current_user_participated(&self) -> bool { + self.current_user_participated + } + + /// The Python GC needs to know that this object references the latest + /// event. + /// + /// Note that we don't need to implement `__clear__` because we cannot have + /// reference cycles. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.latest_event)?; + Ok(()) + } +} + +/// The bundled aggregations for a single event. +/// +/// Some values require additional processing during serialization (the edit +/// and the thread's latest event are themselves serialized). +#[pyclass(frozen, skip_from_py_object, get_all)] +pub struct BundledAggregations { + /// The `m.reference` aggregation (e.g. `{"chunk": [{"event_id": ...}]}`). + pub references: Option, + /// The edit (`m.replace`) event that applies to this event. + pub replace: Option>, + /// The thread (`m.thread`) summary for this event. + pub thread: Option>, +} + +#[pymethods] +impl BundledAggregations { + #[new] + #[pyo3(signature = (references = None, replace = None, thread = None))] + fn new( + references: Option, + replace: Option>, + thread: Option>, + ) -> Self { + Self { + references, + replace, + thread, + } + } + + #[getter] + fn references(&self) -> Option { + self.references.clone() + } + + #[getter] + fn replace(&self) -> Option<&Py> { + self.replace.as_ref() + } + + #[getter] + fn thread(&self) -> Option<&Py> { + self.thread.as_ref() + } + + /// Whether there are any aggregations to bundle. + /// + /// Matches the Python `bool(self.references or self.replace or self.thread)`: + /// an empty `references` mapping counts as falsey. + fn __bool__(&self) -> bool { + self.references.as_ref().is_some_and(|r| !r.is_empty()) + || self.replace.is_some() + || self.thread.is_some() + } + + /// The Python GC needs to know that this object references the latest + /// event. + /// + /// Note that we don't need to implement `__clear__` because we cannot have + /// reference cycles. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.replace)?; + visit.call(&self.thread)?; + Ok(()) + } +} diff --git a/rust/src/events/serialize.rs b/rust/src/events/serialize.rs new file mode 100644 index 0000000000..bb3eb4df06 --- /dev/null +++ b/rust/src/events/serialize.rs @@ -0,0 +1,821 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +//! The synchronous core of client event serialization. +//! +//! This module turns events from their internal/federation shape into the JSON +//! shape sent to clients: applying the requested [`EventFormat`], folding in +//! redactions, module-callback unsigned additions, bundled aggregations +//! (references, edits and thread summaries) and field filtering. +//! +//! It operates purely on already-fetched data — all DB/IO (fetching redactions, +//! running module callbacks, resolving the admin/MSC4354 config) is performed +//! up front by the Python caller and passed in. +//! +//! The entry point is [`serialize_events`], which reads the Python inputs (the +//! redaction map, bundled aggregations and module-callback additions) once per +//! batch and then serializes each event — recursing entirely in Rust into +//! redactions and bundled aggregations via [`serialize_event`]. + +use std::collections::HashMap; + +use pyo3::{ + exceptions::{PyTypeError, PyValueError}, + pyclass, pyfunction, pymethods, + types::{PyAnyMethods, PyDict, PyDictMethods}, + Bound, PyAny, PyResult, Python, +}; +use pythonize::pythonize; +use serde_json::{Map, Number, Value}; + +use crate::{ + events::{ + constants::{ + event_field::{self, CONTENT, EVENT_ID, ROOM_ID, SENDER, UNSIGNED}, + event_type::{M_ROOM_CREATE, M_ROOM_REDACTION}, + redaction_field::REDACTS, + relation_type, unsigned_field, + }, + json_object::JsonObject, + relations::BundledAggregations, + Event, + }, + types::Requester, +}; + +/// The top-level `user_id` field copied from `sender` by the v1 client format. +const USER_ID: &str = "user_id"; + +/// Keys dropped by the v2 client event format. +const V2_DROP_KEYS: [&str; 7] = [ + event_field::AUTH_EVENTS, + event_field::PREV_EVENTS, + event_field::HASHES, + event_field::SIGNATURES, + event_field::DEPTH, + event_field::ORIGIN, + event_field::PREV_STATE, +]; + +/// Keys copied from `unsigned` to the top level by the v1 client event format. +const V1_COPY_KEYS: [&str; 6] = [ + unsigned_field::AGE, + unsigned_field::REDACTED_BECAUSE, + unsigned_field::REPLACES_STATE, + unsigned_field::PREV_CONTENT, + unsigned_field::INVITE_ROOM_STATE, + unsigned_field::KNOCK_ROOM_STATE, +]; + +/// The format used to convert an event from its federation shape to the shape +/// sent to clients. +#[pyclass(eq, eq_int, frozen, from_py_object)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum EventFormat { + /// Return the event dict unchanged (federation format). + Raw, + /// The legacy `/events`-style client format. + ClientV1, + /// The `/sync`-style client format. + ClientV2, + /// Like `ClientV2`, but also strips `room_id`. + ClientV2WithoutRoomId, +} + +/// Configuration for serializing an event for clients. +/// +/// The output shape is chosen by [`EventFormat`]. The field `requester`, when +/// set, controls whether sender-only fields (such as the transaction ID) are +/// included. +#[pyclass(frozen, skip_from_py_object, get_all)] +#[derive(Clone)] +pub struct SerializeEventConfig { + /// Whether to apply the client event format transform (v1/v2/raw). When + /// `false`, the federation-format PDU event is returned as-is. + /// + /// FIXME: Can we remove this and rely on [`Self::event_format`]? + as_client_event: bool, + /// Which client event format variant to apply (only used when + /// `as_client_event` is `true`). + event_format: EventFormat, + /// The entity requesting the event. Used to gate sender-only fields such as + /// `transaction_id` and `delay_id`. + requester: Option, + /// If set, only include these field paths in the output. An empty list or + /// `None` returns all fields. + /// + /// The fields can be "dotted" fields, e.g. `content.body`. + event_field_allowlist: Option>, + /// Whether to include `invite_room_state` / `knock_room_state` in + /// `unsigned`. These are stripped by default and only included for specific + /// endpoints (e.g. `/sync` invite/knock handling). + include_stripped_room_state: bool, + /// When `true`, add server-admin-only metadata to `unsigned` + /// (`io.element.synapse.soft_failed`, + /// `io.element.synapse.policy_server_spammy`). + include_admin_metadata: bool, + /// Whether MSC4354 (sticky events) is enabled. When `true`, the remaining + /// stickiness TTL is computed and added to `unsigned`. + msc4354_enabled: bool, +} + +#[pymethods] +impl SerializeEventConfig { + #[new] + #[allow(clippy::too_many_arguments)] + fn new( + as_client_event: bool, + event_format: EventFormat, + requester: Option>, + event_field_allowlist: Option>, + include_stripped_room_state: bool, + include_admin_metadata: bool, + msc4354_enabled: bool, + ) -> PyResult { + let requester = requester.map(|r| r.get().clone()); + + Ok(Self { + as_client_event, + event_format, + requester, + event_field_allowlist, + include_stripped_room_state, + include_admin_metadata, + msc4354_enabled, + }) + } +} + +/// Synchronously serialize a batch of events for clients. +/// +/// `events` is a list of `(event, membership)` pairs, where `event` is a +/// `FilteredEvent.event` and `membership` the corresponding +/// `FilteredEvent.membership`. All DB/IO must already have been performed by the +/// Python caller: `redaction_map` maps redaction event IDs to events, +/// `unsigned_additions` maps event IDs to module-callback unsigned fields, and +/// `bundle_aggregations` maps event IDs to their bundled aggregations. +/// +/// These three maps are shared across the whole batch, so they are read out of +/// Python once and then reused for every event. +#[pyfunction] +#[pyo3(signature = ( + events, + time_now_ms, + config, + *, + bundle_aggregations = None, + redaction_map = None, + unsigned_additions = None, +))] +pub fn serialize_events<'py>( + py: Python<'py>, + events: Vec<(Bound<'py, Event>, Option)>, + time_now_ms: i64, + config: &SerializeEventConfig, + bundle_aggregations: Option>>, + redaction_map: Option>>, + unsigned_additions: Option>, +) -> PyResult>> { + let redaction_map = redaction_map.unwrap_or_default(); + let unsigned_additions = unsigned_additions.unwrap_or_default(); + + events + .iter() + .map(|(event, membership)| { + let serialized = serialize_event( + event.get(), + time_now_ms, + config, + membership.as_deref(), + bundle_aggregations.as_ref(), + &redaction_map, + &unsigned_additions, + )?; + Ok(pythonize(py, &Value::Object(serialized))?) + }) + .collect() +} + +/// The recursive core: serialize a single event, fold in its redaction, +/// module-callback additions and field filtering, then recurse into any +/// bundled aggregations. +#[allow(clippy::too_many_arguments)] +fn serialize_event( + event: &Event, + time_now_ms: i64, + config: &SerializeEventConfig, + membership: Option<&str>, + bundle_aggregations: Option<&HashMap>>, + redaction_map: &HashMap>, + unsigned_additions: &HashMap, +) -> PyResult> { + let mut serialized = serialize_event_value(event, time_now_ms, config, membership)?; + + // If the event was redacted, include the (pre-fetched) redaction event in + // the serialized event's unsigned section. + if let Some(redacted_by) = event.internal_metadata.redacted_by()? { + unsigned_mut(&mut serialized)?.insert( + unsigned_field::REDACTED_BY.to_owned(), + Value::String(redacted_by.clone()), + ); + + if let Some(redaction_event) = redaction_map.get(&redacted_by) { + let serialized_redaction = Value::Object(serialize_event_value( + redaction_event.get(), + time_now_ms, + config, + None, + )?); + unsigned_mut(&mut serialized)?.insert( + unsigned_field::REDACTED_BECAUSE.to_owned(), + serialized_redaction.clone(), + ); + // The v1 client format (apply_event_format) copies redacted_because + // up to the top level, but since we add it after that runs, do it + // here too. + if config.as_client_event && config.event_format == EventFormat::ClientV1 { + serialized.insert( + unsigned_field::REDACTED_BECAUSE.to_owned(), + serialized_redaction, + ); + } + } + } + + // Merge in the module-callback additions. Start from a copy of the additions + // and overlay the event's own unsigned on top, so modules can't clobber + // existing fields. + if let Some(adds) = unsigned_additions.get(event.event_id()) { + let unsigned = unsigned_mut(&mut serialized)?; + for (key, value) in adds { + // Don't let modules clobber existing unsigned fields. + if let serde_json::map::Entry::Vacant(entry) = unsigned.entry(&**key) { + entry.insert(value.clone()); + } + } + } + + // Only include fields that the client has requested. + if let Some(fields) = &config.event_field_allowlist { + if !fields.is_empty() { + serialized = only_fields(&serialized, fields)?; + } + } + + // Inject any bundled aggregations. Note this happens after field filtering; + // aggregations are always returned. + if let Some(bundles) = bundle_aggregations { + if let Some(aggregation) = bundles.get(event.event_id()) { + inject_bundled_aggregations( + time_now_ms, + config, + aggregation.get(), + &mut serialized, + bundle_aggregations, + redaction_map, + unsigned_additions, + )?; + } + } + + Ok(serialized) +} + +/// Inject an event's bundled aggregations (references, edit, thread summary) +/// into the `m.relations` section of its serialized `unsigned`. +#[allow(clippy::too_many_arguments)] +fn inject_bundled_aggregations( + time_now_ms: i64, + config: &SerializeEventConfig, + aggregation: &BundledAggregations, + serialized_event: &mut Map, + bundle_aggregations: Option<&HashMap>>, + redaction_map: &HashMap>, + unsigned_additions: &HashMap, +) -> PyResult<()> { + let mut serialized_aggregations = Map::new(); + + if let Some(references) = &aggregation.references { + if !references.is_empty() { + serialized_aggregations.insert( + relation_type::REFERENCE.to_owned(), + Value::Object( + references + .iter() + .map(|(k, v)| (k.clone().into_string(), v.clone())) + .collect(), + ), + ); + } + } + + if let Some(replace) = &aggregation.replace { + // Bundle the *whole* edit event (serialized without its own bundled + // aggregations). The spec (v1.5) only requires event_id/origin_server_ts/ + // sender, but per MSC3925 we include the full edit. + // https://spec.matrix.org/v1.5/client-server-api/#server-side-aggregation-of-mreplace-relationships + let serialized = serialize_event( + replace.get(), + time_now_ms, + config, + None, + None, + redaction_map, + unsigned_additions, + )?; + serialized_aggregations + .insert(relation_type::REPLACE.to_owned(), Value::Object(serialized)); + } + + if let Some(thread) = &aggregation.thread { + let thread = thread.get(); + // The thread's latest event is serialized with the same bundle map, so + // it may recurse further. + let serialized_latest = serialize_event( + thread.latest_event.get(), + time_now_ms, + config, + None, + bundle_aggregations, + redaction_map, + unsigned_additions, + )?; + + let mut thread_summary = Map::new(); + thread_summary.insert("latest_event".to_owned(), Value::Object(serialized_latest)); + thread_summary.insert( + "count".to_owned(), + Value::Number(Number::from(thread.count)), + ); + thread_summary.insert( + "current_user_participated".to_owned(), + Value::Bool(thread.current_user_participated), + ); + serialized_aggregations.insert( + relation_type::THREAD.to_owned(), + Value::Object(thread_summary), + ); + } + + if !serialized_aggregations.is_empty() { + let unsigned = unsigned_mut(serialized_event)?; + let relations = object_entry_mut(unsigned, unsigned_field::M_RELATIONS)?; + for (key, value) in serialized_aggregations { + relations.insert(key, value); + } + } + + Ok(()) +} + +/// Serialize a single event to its client JSON shape, without recursing into +/// redactions or bundled aggregations (those are handled by the caller). +fn serialize_event_value( + event: &Event, + time_now_ms: i64, + config: &SerializeEventConfig, + membership: Option<&str>, +) -> PyResult> { + let mut d: Map = match serde_json::to_value(&event.parsed_event) { + Ok(Value::Object(map)) => map, + Ok(_) => { + return Err(PyValueError::new_err( + "event did not serialize to a JSON object", + )) + } + Err(err) => { + return Err(PyValueError::new_err(format!( + "Failed to serialize event: {err}" + ))) + } + }; + + // Always include the `event_id` field in a client event. For room version + // v3+, these aren't in the PDU event JSON. + d.insert( + EVENT_ID.to_owned(), + Value::String(event.event_id().to_owned()), + ); + + // Replace `age_ts` with `age`, with `age` calculated as the difference + // between the current time and `age_ts`. This is an optional field in the + // spec. + // + // We might not have an `age_ts`, e.g. if a remote server did not include + // the `age` field in the event it sent us. Since `age_ts` is generated by + // us, it *should* be an integer, but it is possible for it to be out of i64 + // range (e.g. if the original `age` was close to the maximum i64 value). In + // that case, just omit `age` rather than erroring (otherwise a once valid + // event could start failing). + let unsigned = unsigned_mut(&mut d)?; + if let Some(age_ts) = event.unsigned().age_ts()?.and_then(|n| n.as_i64()) { + unsigned.insert( + unsigned_field::AGE.to_owned(), + Value::Number(Number::from(time_now_ms - age_ts)), + ); + unsigned.remove(unsigned_field::AGE_TS); + } + + // Include the transaction_id / delay_id in the unsigned section if the event + // was sent by the same session (or, where appropriate, the same sender) as + // the one requesting the event. + if let Some(requester) = &config.requester { + if requester.user_id == event.sender() { + if let Some(txn_id) = event.internal_metadata.txn_id()? { + if let Some(event_device_id) = event.internal_metadata.device_id()? { + if Some(event_device_id.as_str()) == requester.device_id.as_deref() { + unsigned_mut(&mut d)?.insert( + unsigned_field::TRANSACTION_ID.to_owned(), + Value::String(txn_id), + ); + } + } else { + // No device ID is stored for some events: old events, and + // those created by appservices, guests, or with admin-API + // tokens. For those, fall back to the access token: only + // include the transaction ID if the event was sent from the + // same token (or for guests/appservices, which we can't + // check, so assume the same session). + let event_token_id = event.internal_metadata.token_id()?; + let token_matches = event_token_id.is_some() + && requester.access_token_id.is_some() + && event_token_id == requester.access_token_id; + if token_matches || requester.is_guest || requester.app_service_id.is_some() { + unsigned_mut(&mut d)?.insert( + unsigned_field::TRANSACTION_ID.to_owned(), + Value::String(txn_id), + ); + } + } + } + + if let Some(delay_id) = event.internal_metadata.delay_id()? { + unsigned_mut(&mut d)? + .insert(unsigned_field::DELAY_ID.to_owned(), Value::String(delay_id)); + } + } + } + + // Strip invite/knock room state unless requested. + if !config.include_stripped_room_state { + let unsigned = unsigned_mut(&mut d)?; + unsigned.remove(unsigned_field::INVITE_ROOM_STATE); + unsigned.remove(unsigned_field::KNOCK_ROOM_STATE); + } + + if config.as_client_event { + apply_event_format(config.event_format, &mut d); + } + + // Ensure the room_id field is set for create events in MSC4291 rooms. + if event.r#type() == M_ROOM_CREATE && event.room_version.msc4291_room_ids_as_hashes { + d.insert( + ROOM_ID.to_owned(), + Value::String(event.room_id().to_owned()), + ); + } + + // A redaction stores the redacted event ID in different places depending + // on the room version (top-level `redacts` vs `content.redacts`). It's + // already in the version-correct place; copy it to the *other* one too, + // for forwards/backwards-compatibility with clients. + if event.r#type() == M_ROOM_REDACTION { + let redacts = event.redacts(); + // Skip a present-but-null value: the Python `e.redacts` property + // surfaced JSON null as `None`, and the old code guarded with + // `e.redacts is not None`. + if let Some(redacts) = redacts.filter(|v| !v.is_null()) { + let redacts = redacts.clone(); + if event.room_version.updated_redaction_rules { + d.insert(REDACTS.to_owned(), redacts); + } else { + object_entry_mut(&mut d, CONTENT)?.insert(REDACTS.to_owned(), redacts); + } + } + } + + let unsigned = unsigned_mut(&mut d)?; + if config.include_admin_metadata { + if event.internal_metadata.soft_failed()? { + unsigned.insert(unsigned_field::SOFT_FAILED.to_owned(), Value::Bool(true)); + } + if event.internal_metadata.policy_server_spammy()? { + unsigned.insert( + unsigned_field::POLICY_SERVER_SPAMMY.to_owned(), + Value::Bool(true), + ); + } + } + + if config.msc4354_enabled { + if let Some(sticky_duration) = event.sticky_duration() { + // min() ensures the origin server can't claim a time in the future + // to exceed the stickiness duration limit. + // + // The `as i64` cast is safe as sticky duration are capped to an + // hour, which is well within the i64 range. + let expires_at = std::cmp::min(event.origin_server_ts(), time_now_ms) + + sticky_duration.as_millis() as i64; + if expires_at > time_now_ms { + unsigned.insert( + unsigned_field::STICKY_TTL.to_owned(), + Value::Number(Number::from(expires_at - time_now_ms)), + ); + } + } + } + + if let Some(membership) = membership { + unsigned.insert( + unsigned_field::MEMBERSHIP.to_owned(), + Value::String(membership.to_owned()), + ); + } + + Ok(d) +} + +/// Apply the client event format transform in place. +fn apply_event_format(format: EventFormat, d: &mut Map) { + match format { + EventFormat::Raw => {} + EventFormat::ClientV2 => format_for_client_v2(d), + EventFormat::ClientV2WithoutRoomId => { + format_for_client_v2(d); + d.remove(ROOM_ID); + } + EventFormat::ClientV1 => { + format_for_client_v2(d); + + let sender = d.get(SENDER).filter(|v| !v.is_null()).cloned(); + if let Some(sender) = sender { + d.insert(USER_ID.to_owned(), sender); + } + + let mut to_copy = Vec::new(); + if let Some(Value::Object(unsigned)) = d.get(UNSIGNED) { + for key in V1_COPY_KEYS { + if let Some(value) = unsigned.get(key) { + to_copy.push((key.to_owned(), value.clone())); + } + } + } + for (key, value) in to_copy { + d.insert(key, value); + } + } + } +} + +fn format_for_client_v2(d: &mut Map) { + for key in V2_DROP_KEYS { + d.remove(key); + } +} + +// Standalone versions of the client format transforms, re-exported from +// `synapse.events.utils` purely as a backwards compatibility hack: they have +// never been part of the module API and modules shouldn't be pulling them in, +// but some in the wild import them from there anyway. They may be removed in +// the future; nothing in Synapse itself should use them. +// +// Unlike [`apply_event_format`], these mutate a Python dict in place and +// return it, matching the original Python implementations that used to live +// in `synapse/events/utils.py`. + +/// Return the event dict unchanged (federation format). +#[pyfunction] +pub fn format_event_raw(d: Bound<'_, PyDict>) -> Bound<'_, PyDict> { + d +} + +/// Apply the legacy `/events`-style v1 client format to `d` in place. +#[pyfunction] +pub fn format_event_for_client_v1(d: Bound<'_, PyDict>) -> PyResult> { + let d = format_event_for_client_v2(d)?; + + if let Some(sender) = d.get_item(SENDER)? { + if !sender.is_none() { + d.set_item(USER_ID, sender)?; + } + } + + // As in the original Python (`d["unsigned"]`), a missing `unsigned` key + // raises `KeyError`. + let unsigned = d.as_any().get_item(UNSIGNED)?; + for key in V1_COPY_KEYS { + if unsigned.contains(key)? { + d.set_item(key, unsigned.get_item(key)?)?; + } + } + + Ok(d) +} + +/// Apply the `/sync`-style v2 client format to `d` in place. +#[pyfunction] +pub fn format_event_for_client_v2(d: Bound<'_, PyDict>) -> PyResult> { + for key in V2_DROP_KEYS { + // Equivalent to `d.pop(key, None)`. + if d.contains(key)? { + d.del_item(key)?; + } + } + Ok(d) +} + +/// Apply the v2 client format to `d` in place, additionally stripping `room_id`. +#[pyfunction] +pub fn format_event_for_client_v2_without_room_id( + d: Bound<'_, PyDict>, +) -> PyResult> { + let d = format_event_for_client_v2(d)?; + if d.contains(ROOM_ID)? { + d.del_item(ROOM_ID)?; + } + Ok(d) +} + +/// Return a mutable reference to `map["unsigned"]`, creating it as an empty +/// object if it is missing or not an object. +fn unsigned_mut(map: &mut Map) -> PyResult<&mut Map> { + object_entry_mut(map, UNSIGNED) +} + +/// Return a mutable reference to `map[key]`, creating it as an empty object if +/// missing or not an object. +fn object_entry_mut<'a>( + map: &'a mut Map, + key: &str, +) -> PyResult<&'a mut Map> { + let entry = map + .entry(key.to_owned()) + .or_insert_with(|| Value::Object(Map::new())); + + let Some(obj) = entry.as_object_mut() else { + return Err(PyTypeError::new_err(format!( + "Expected an object for key '{key}'" + ))); + }; + + Ok(obj) +} + +/// Return a new map containing only the given (possibly dotted) field paths, +/// implementing the `event_field_allowlist` client filter. +fn only_fields(dictionary: &Map, fields: &[String]) -> PyResult> { + let mut output = Map::new(); + for field in fields { + copy_field(dictionary, &mut output, &split_field(field))?; + } + Ok(output) +} + +/// Copy a single (possibly nested) field path from `src` into `dst`, creating +/// intermediate objects in `dst` as needed. A missing path is a no-op. +fn copy_field( + src: &Map, + dst: &mut Map, + field: &[String], +) -> PyResult<()> { + if field.is_empty() { + return Ok(()); + } + if field.len() == 1 { + if let Some(value) = src.get(&field[0]) { + dst.insert(field[0].clone(), value.clone()); + } + return Ok(()); + } + + let (key_to_move, parents) = field.split_last().expect("field is non-empty"); + + // Drill down into `src`. + let mut sub = src; + for parent in parents { + match sub.get(parent) { + Some(Value::Object(obj)) => sub = obj, + _ => return Ok(()), + } + } + + let Some(value) = sub.get(key_to_move) else { + return Ok(()); + }; + let value = value.clone(); + + // Build the nested objects in `dst` as required. + let mut out = dst; + for parent in parents { + out = object_entry_mut(out, parent)?; + } + out.insert(key_to_move.clone(), value); + + Ok(()) +} + +/// Split a dotted field path into its components, splitting on unescaped dots +/// and removing the escaping. A literal `.` or `\` in a key is escaped with `\`. +fn split_field(field: &str) -> Vec { + let bytes = field.as_bytes(); + let mut result = Vec::new(); + let mut prev_start = 0; + + for (i, &b) in bytes.iter().enumerate() { + if b != b'.' { + continue; + } + // Count the run of backslashes immediately preceding the dot. The dot is + // escaped iff that count is odd. + let mut backslashes = 0; + let mut j = i; + while j > 0 && bytes[j - 1] == b'\\' { + backslashes += 1; + j -= 1; + } + if backslashes % 2 == 0 { + result.push(unescape(&field[prev_start..i])); + prev_start = i + 1; + } + } + + result.push(unescape(&field[prev_start..])); + result +} + +/// Remove field-path escaping: `\\` and `\.` collapse to the second character; +/// any other `\x` is left as-is. +fn unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.peek() { + Some(&c @ ('\\' | '.')) => { + out.push(c); + chars.next(); + } + _ => out.push('\\'), + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::split_field; + + #[test] + fn test_split_field() { + // Ported from the Python `SplitFieldTestCase` that previously lived in + // tests/events/test_utils.py (removed alongside `_split_field`). + let cases: &[(&str, &[&str])] = &[ + // A field with no dots. + ("m", &["m"]), + // Simple dotted fields. + ("m.foo", &["m", "foo"]), + ("m.foo.bar", &["m", "foo", "bar"]), + // Backslash is used as an escape character. + (r"m\.foo", &["m.foo"]), + (r"m\\.foo", &["m\\", "foo"]), + (r"m\\\.foo", &[r"m\.foo"]), + (r"m\\\\.foo", &["m\\\\", "foo"]), + (r"m\foo", &[r"m\foo"]), + (r"m\\foo", &[r"m\foo"]), + (r"m\\\foo", &[r"m\\foo"]), + (r"m\\\\foo", &[r"m\\foo"]), + // Ensure that escapes at the end don't cause issues. + ("m.foo\\", &["m", "foo\\"]), + (r"m.foo\.", &["m", "foo."]), + (r"m.foo\\.", &["m", "foo\\", ""]), + (r"m.foo\\\.", &["m", r"foo\."]), + // Empty parts (corresponding to empty-string properties) are allowed. + (".m", &["", "m"]), + ("..m", &["", "", "m"]), + ("m.", &["m", ""]), + ("m..", &["m", "", ""]), + ("m..foo", &["m", "", "foo"]), + // Invalid escape sequences are left alone. + (r"\m", &[r"\m"]), + ]; + + for (input, expected) in cases { + let expected: Vec = expected.iter().map(|s| s.to_string()).collect(); + assert_eq!(split_field(input), expected, "split_field({input:?})"); + } + } +} diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 931c412325..8db0a338bd 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -291,6 +291,14 @@ impl Unsigned { } } +impl Unsigned { + /// Get the `age_ts` field, which is used to generate the `age` field when + /// serializing an event. + pub fn age_ts(&self) -> PyResult> { + Ok(self.py_read()?.persisted_fields.age_ts.clone()) + } +} + fn room_state_to_py<'py>( py: Python<'py>, state: &[serde_json::Value], diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs new file mode 100644 index 0000000000..d560423166 --- /dev/null +++ b/rust/src/handlers/mod.rs @@ -0,0 +1,95 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use pyo3::{ + prelude::*, + types::{PyAnyMethods, PyModule, PyModuleMethods}, + Bound, PyResult, Python, +}; + +use crate::config::SynapseHomeServerConfig; +use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; +use crate::storage::store::Store; + +pub mod versions; + +#[pyclass] +struct RustHandlers { + versions: Py, +} + +#[pymethods] +impl RustHandlers { + #[new] + #[pyo3(signature = (homeserver))] + pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { + let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; + + // The Twisted reactor, used both to drive our Tokio runtime and to + // marshal database work back onto the reactor thread. + let reactor: Py = homeserver.call_method0("get_reactor")?.unbind(); + + // hs.get_datastores().main.db_pool + let db_pool_py: Py = homeserver + .call_method0("get_datastores")? + .getattr("main")? + .getattr("db_pool")? + .unbind(); + let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); + + // Store is shared across all of the handlers so let's use an `Arc` + let store = Arc::new(Store { + db_pool: Box::new(db_pool), + }); + + let global_unstable_feature_map = Arc::new( + versions::synapse_config_to_global_unstable_feature_map(&config), + ); + + let versions = Py::new( + py, + versions::VersionsHandler { + global_unstable_feature_map: Arc::clone(&global_unstable_feature_map), + store: Arc::clone(&store), + reactor: reactor.clone_ref(py), + }, + )?; + + Ok(RustHandlers { versions }) + } + + #[getter] + fn versions(&self, py: Python<'_>) -> Py { + self.versions.clone_ref(py) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "handlers")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import push` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.handlers", child_module)?; + + Ok(()) +} diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs new file mode 100644 index 0000000000..25da9d23fc --- /dev/null +++ b/rust/src/handlers/versions.rs @@ -0,0 +1,336 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use pyo3::prelude::*; +use pythonize::{pythonize, PythonizeError}; +use serde::Serialize; + +use crate::config::{types::RoomCreationPreset, SynapseHomeServerConfig}; +use crate::deferred::create_deferred; +use crate::storage::store::{PerUserExperimentalFeature, Store}; + +/// `GET /_matrix/client/versions` response +#[derive(Serialize, Clone, Debug)] +struct VersionsResponse { + versions: Vec, + /// as per MSC1497 + unstable_features: UnstableFeatureMap, +} + +impl<'py> IntoPyObject<'py> for VersionsResponse { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'py>) -> Result { + pythonize(py, &self) + } +} + +#[pyclass] +pub struct VersionsHandler { + pub global_unstable_feature_map: Arc, + pub store: Arc, + /// The Twisted reactor, used to bridge our `async` response back into a + /// Twisted deferred that Python can `await`. + pub reactor: Py, +} + +#[pymethods] +impl VersionsHandler { + /// Assemble a `/versions` response, returning a Twisted deferred that + /// resolves to the response body (a dict). + #[pyo3(signature = (user_id=None))] + fn get_versions<'py>( + &self, + py: Python<'py>, + user_id: Option, + ) -> PyResult> { + let store = Arc::clone(&self.store); + let global_unstable_feature_map = Arc::clone(&self.global_unstable_feature_map); + + create_deferred(py, self.reactor.bind(py), async move { + build_versions_response(&store, &global_unstable_feature_map, user_id.as_deref()) + .await + .map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build /versions response: {err:#}" + )) + }) + }) + } +} + +/// Assemble a `/versions` response body. +/// +/// Args: +/// * store +/// * global_unstable_feature_map: The global values before any per-user overrides +/// * user_id: The user making the request +async fn build_versions_response( + store: &Store, + global_unstable_feature_map: &UnstableFeatureMap, + user_id: Option<&str>, +) -> Result { + let msc3881_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3881 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3881) + .await? + // Default to false if there is no entry for this user + .unwrap_or(false) + } + } + None => global_unstable_feature_map.msc3881, + }; + + let msc3575_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3575 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3575) + .await? + // Default to false if there is no entry for this user + .unwrap_or(false) + } + } + None => global_unstable_feature_map.msc3575, + }; + + let unstable_feature_map = UnstableFeatureMap { + msc3575: msc3575_enabled, + msc3881: msc3881_enabled, + ..*global_unstable_feature_map + }; + + Ok(VersionsResponse { + versions: Vec::from([ + // XXX: at some point we need to decide whether we need to include + // the previous version numbers, given we've defined r0.3.0 to be + // backwards compatible with r0.2.0. But need to check how + // conscientious we've been in compatibility, and decide whether the + // middle number is the major revision when at 0.X.Y (as opposed to + // X.Y.Z). And we need to decide whether it's fair to make clients + // parse the version string to figure out what's going on. + "r0.0.1".to_string(), + "r0.1.0".to_string(), + "r0.2.0".to_string(), + "r0.3.0".to_string(), + "r0.4.0".to_string(), + "r0.5.0".to_string(), + "r0.6.0".to_string(), + "r0.6.1".to_string(), + "v1.1".to_string(), + "v1.2".to_string(), + "v1.3".to_string(), + "v1.4".to_string(), + "v1.5".to_string(), + "v1.6".to_string(), + "v1.7".to_string(), + "v1.8".to_string(), + "v1.9".to_string(), + "v1.10".to_string(), + "v1.11".to_string(), + "v1.12".to_string(), + ]), + unstable_features: unstable_feature_map, + }) +} + +/// Experimental features the server supports +#[derive(Serialize, Debug, Clone)] +pub struct UnstableFeatureMap { + /// Implements support for label-based filtering as described in + /// MSC2326. + #[serde(rename = "org.matrix.label_based_filtering")] + msc2326: bool, + /// Implements support for cross signing as described in MSC1756 + #[serde(rename = "org.matrix.e2e_cross_signing")] + msc1756: bool, + /// Implements additional endpoints as described in MSC2432 + #[serde(rename = "org.matrix.msc2432")] + msc2432: bool, + /// Implements additional endpoints as described in MSC2666 + #[serde(rename = "uk.half-shot.msc2666.query_mutual_rooms.stable")] + msc2666: bool, + // Supports the busy presence state described in MSC3026. + #[serde(rename = "org.matrix.msc3026.busy_presence")] + msc3026: bool, + /// Supports receiving private read receipts as per MSC2285 + // TODO: Remove when MSC2285 becomes a part of the spec + #[serde(rename = "org.matrix.msc2285.stable")] + msc2285: bool, + /// Supports filtering of /publicRooms by room type as per MSC3827 + #[serde(rename = "org.matrix.msc3827.stable")] + msc3827: bool, + /// Adds support for thread relations, per MSC3440. + // TODO: remove when "v1.3" is added above + #[serde(rename = "org.matrix.msc3440.stable")] + msc3440: bool, + /// Support for thread read receipts & notification counts. + #[serde(rename = "org.matrix.msc3771")] + msc3771: bool, + #[serde(rename = "org.matrix.msc3773")] + msc3773: bool, + /// Allows moderators to fetch redacted event content as described in MSC2815 + #[serde(rename = "fi.mau.msc2815")] + msc2815: bool, + /// Adds a ping endpoint for appservices to check HS->AS connection + // TODO: remove when "v1.7" is added above + #[serde(rename = "fi.mau.msc2659.stable")] + msc2659: bool, + // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + #[serde(rename = "org.matrix.msc3882")] + msc3882: bool, + /// Adds support for remotely enabling/disabling pushers, as per MSC3881 + #[serde(rename = "org.matrix.msc3881")] + msc3881: bool, + /// Adds support for filtering /messages by event relation. + #[serde(rename = "org.matrix.msc3874")] + msc3874: bool, + // Adds support for relation-based redactions as per MSC3912. + #[serde(rename = "org.matrix.msc3912")] + msc3912: bool, + /// Whether recursively provide relations is supported. + // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + #[serde(rename = "org.matrix.msc3981")] + msc3981: bool, + /// Adds support for deleting account data. + #[serde(rename = "org.matrix.msc3391")] + msc3391: bool, + /// Allows clients to inhibit profile update propagation. + #[serde(rename = "org.matrix.msc4069")] + msc4069: bool, + // Allows clients to handle push for encrypted events. + #[serde(rename = "org.matrix.msc4028")] + msc4028: bool, + /// MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + #[serde(rename = "org.matrix.msc4108")] + msc4108: bool, + /// MSC4140: Delayed events + #[serde(rename = "org.matrix.msc4140")] + msc4140: bool, + /// Simplified sliding sync + #[serde(rename = "org.matrix.simplified_msc3575")] + msc3575: bool, + /// Arbitrary key-value profile fields. + #[serde(rename = "uk.tcpip.msc4133")] + msc4133: bool, + /// Arbitrary key-value profile fields (stable identifier) + #[serde(rename = "uk.tcpip.msc4133.stable")] + msc4133_stable: bool, + /// MSC4155: Invite filtering + #[serde(rename = "org.matrix.msc4155")] + msc4155: bool, + /// MSC4306: Support for thread subscriptions + #[serde(rename = "org.matrix.msc4306")] + msc4306: bool, + /// MSC4169: Backwards-compatible redaction sending using `/send` + #[serde(rename = "com.beeper.msc4169")] + msc4169: bool, + /// MSC4354: Sticky events + #[serde(rename = "org.matrix.msc4354")] + msc4354: bool, + /// MSC4380: Invite blocking + #[serde(rename = "org.matrix.msc4380.stable")] + msc4380: bool, + /// MSC4445: Sync timeline order + #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] + msc4445_initial_sync_timeline_topological_ordering: bool, + /// MSC4491: Invite reasons in room creation + #[serde(rename = "uk.timedout.msc4491.create_room_invite_reasons")] + msc4491_enabled: bool, + /// MSC4143: Matrix RTC transports (LiveKit backend) + #[serde(rename = "org.matrix.msc4143")] + msc4143_enabled: bool, + /// MSC4446: Allow moving the fully read marker backwards. + #[serde(rename = "com.beeper.msc4446")] + msc4446_enabled: bool, + + // Whether new rooms will be set to encrypted or not (based on presets). + #[serde(rename = "io.element.e2ee_forced.public")] + e2ee_forced_public: bool, + #[serde(rename = "io.element.e2ee_forced.private")] + e2ee_forced_private: bool, + #[serde(rename = "io.element.e2ee_forced.trusted_private")] + e2ee_forced_trusted_private: bool, +} + +/// Convert from [`SynapseHomeServerConfig`] to the global defaults for unstable features that the +/// server supports [`UnstableFeatureMap`] +pub fn synapse_config_to_global_unstable_feature_map( + config: &SynapseHomeServerConfig, +) -> UnstableFeatureMap { + UnstableFeatureMap { + msc2326: true, + msc1756: true, + msc2432: true, + msc2666: true, + msc3026: config.experimental.msc3026_enabled, + msc2285: true, + msc3827: true, + msc3440: true, + msc3771: true, + msc3773: config.experimental.msc3773_enabled, + msc2815: config.experimental.msc2815_enabled, + msc2659: true, + msc3882: config.auth.login_via_existing_enabled, + msc3881: config.experimental.msc3881_enabled, + msc3874: config.experimental.msc3874_enabled, + msc3912: config.experimental.msc3912_enabled, + msc3981: true, + msc3391: config.experimental.msc3391_enabled, + msc4069: config.experimental.msc4069_profile_inhibit_propagation, + msc4028: config.experimental.msc4028_push_encrypted_events, + msc4108: config.experimental.msc4108_enabled + || (config.experimental.msc4108_delegation_endpoint.is_some()), + msc4140: config.server.msc4140_enabled, + msc3575: config.experimental.msc3575_enabled, + msc4133: config.experimental.msc4133_enabled, + msc4133_stable: true, + msc4155: config.experimental.msc4155_enabled, + msc4306: config.experimental.msc4306_enabled, + msc4169: config.experimental.msc4169_enabled, + msc4354: config.experimental.msc4354_enabled, + msc4380: true, + msc4445_initial_sync_timeline_topological_ordering: true, + msc4491_enabled: config.experimental.msc4491_enabled, + msc4143_enabled: config.experimental.msc4143_enabled, + msc4446_enabled: config.experimental.msc4446_enabled, + e2ee_forced_public: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PublicChat), + e2ee_forced_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PrivateChat), + e2ee_forced_trusted_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::TrustedPrivateChat), + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index eb2d8cecdd..28783afbba 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,10 +6,12 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; +pub mod config; pub mod deferred; pub mod duration; pub mod errors; pub mod events; +pub mod handlers; pub mod http; pub mod http_client; pub mod identifier; @@ -20,6 +22,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod storage; pub mod tokio_runtime; pub mod types; @@ -70,6 +73,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; + handlers::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; diff --git a/rust/src/msc4388_rendezvous/mod.rs b/rust/src/msc4388_rendezvous/mod.rs index bc9463639f..2f1b004ec1 100644 --- a/rust/src/msc4388_rendezvous/mod.rs +++ b/rust/src/msc4388_rendezvous/mod.rs @@ -331,6 +331,14 @@ impl MSC4388RendezvousHandler { .ok_or_else(NotFoundError::new)?; if !session.sequence_token().eq(&sequence_token) { + // Allow clients to safely retry a PUT (e.g. after a network error) + // by accepting the previous sequence_token as long as the data + // being submitted matches what is currently stored. This makes + // PUTs idempotent without weakening the concurrent-write check. + if session.is_idempotent_retry(&sequence_token, &data) { + return Ok((200, session.put_response())); + } + return Err(SynapseError::new( StatusCode::CONFLICT, "sequence_token does not match".to_owned(), diff --git a/rust/src/msc4388_rendezvous/session.rs b/rust/src/msc4388_rendezvous/session.rs index 467d1b5baf..a4959a4f7f 100644 --- a/rust/src/msc4388_rendezvous/session.rs +++ b/rust/src/msc4388_rendezvous/session.rs @@ -25,6 +25,12 @@ use ulid::Ulid; pub struct Session { id: Ulid, hash: [u8; 32], + /// The hash from before the last `update`, if any. Used so that clients can + /// safely retry a PUT request (e.g. after a network error) without getting + /// a spurious 409 conflict: a PUT whose `sequence_token` matches this + /// previous hash and whose `data` matches the currently-stored data is + /// treated as an idempotent no-op. + previous_hash: Option<[u8; 32]>, data: String, last_modified: SystemTime, expires: SystemTime, @@ -86,6 +92,7 @@ impl Session { Self { id, hash, + previous_hash: None, data, expires: now + ttl, last_modified: now, @@ -99,11 +106,26 @@ impl Session { /// Update the session with new data and last modified time. pub fn update(&mut self, data: String, now: SystemTime) { + self.previous_hash = Some(self.hash); self.hash = Self::compute_hash(&data, now); self.data = data; self.last_modified = now; } + /// Returns true if a PUT with the given `sequence_token` and `data` should + /// be treated as an idempotent retry of the most recent update (i.e. the + /// token matches the hash from before the last update, and the data + /// already matches the currently-stored data). + pub fn is_idempotent_retry(&self, sequence_token: &str, data: &str) -> bool { + let Some(previous_hash) = self.previous_hash else { + return false; + }; + if data != self.data { + return false; + } + URL_SAFE_NO_PAD.encode(previous_hash) == sequence_token + } + /// Compute the hash of the data and timestamp. fn compute_hash(data: &str, now: SystemTime) -> [u8; 32] { let mut hasher = Sha256::new(); diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs new file mode 100644 index 0000000000..fd40d52f08 --- /dev/null +++ b/rust/src/storage/db/mod.rs @@ -0,0 +1,256 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::any::Any; +use std::future::Future; + +use futures::future::BoxFuture; +use futures::FutureExt; + +pub mod python_db_pool; + +/// A type-erased `run_interaction` callback. +/// +/// This is the dyn-compatible form of the `func` passed to +/// [`DatabasePoolExt::run_interaction`]. +/// +/// The ergonomic [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts +/// the result back to `R` for the caller. +/// +/// It may be invoked multiple times under certain failure modes (serialization +/// and deadlock errors), so it is `Fn` rather than `FnOnce`. +pub type ErasedInteraction = + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; + +/// The type-erased *result* of an [`ErasedInteraction`] +/// [`DatabasePool::run_interaction_erased`]. +pub type ErasedResult = anyhow::Result>; + +/// A database connection pool. +/// +/// Held behind a trait object (e.g. `Box`) as it can be backed by +/// either the Python-backed pool (in Synapse, see [`python_db_pool`]) or a Rust native +/// `tokio-postgres` pool (expected to be used in `synapse-rust-apps`). +/// +/// To keep the trait dyn-compatible, we have to specify a type-erased +/// [`run_interaction_erased`](Self::run_interaction_erased) version; callers should +/// prefer the ergonomic, generic [`run_interaction`](DatabasePoolExt::run_interaction). +/// +/// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. +#[async_trait::async_trait] +pub trait DatabasePool: Send + Sync { + /// Starts a transaction on the database and runs the given (type-erased) + /// `func`, returning its boxed result. + /// + /// Implementors implement this; callers should prefer + /// [`DatabasePoolExt::run_interaction`], which boxes up the result and + /// downcasts it back to the concrete type for you. + async fn run_interaction_erased( + &self, + name: &'static str, + func: ErasedInteraction, + ) -> ErasedResult; +} + +/// Ergonomic, strongly-typed access to a [`DatabasePool`]. +pub trait DatabasePoolExt: DatabasePool { + /// Starts a transaction on the database and runs the given function, + /// returning its result. + /// + /// `name` should be a descriptive identifier for logging/metrics + /// + /// `func` may be called multiple times under certain failure modes (like + /// serialization and deadlock errors), so it is `Fn` rather than `FnOnce`. + /// + /// `func` is async but you should only call `.await` on [`Transaction`] methods. + /// This is a minor cosmetic flaw but seems fine, as you don't want to be doing any + /// unnecessary waiting in your transaction anyway. + /// + /// Usage: + /// ```ignore + /// db_pool + /// .run_interaction(|txn| { + /// async move { + /// /* do stuff with txn */ + /// } + /// .boxed() + /// }) + /// ``` + // + // Ideally, this method signature would be slightly different to allow downstream + // usage to look like the following (simpler) but because we allow the work to + // happen on other threads, the `Future` needs to be `Send`; As of 2026-06-22, the + // `AsyncFn` trait has no stable way to express that "the future this async closure + // produces is `Send`". The intended fix is probably return-type-notation + // (https://github.com/rust-lang/rust/issues/109417). + // ``` + // db_pool.run_interaction("description", async move |txn| { + // /* do stuff with txn */ + // }) + // ``` + // + // Refs: + // - [RFC 3668: Async closures](https://github.com/rust-lang/rfcs/pull/3668) + // - [RFC 3654: Return Type Notation](https://github.com/rust-lang/rfcs/pull/3654) + // - [Tracking Issue for return type notation](https://github.com/rust-lang/rust/issues/109417) + fn run_interaction( + &self, + name: &'static str, + func: F, + ) -> impl Future> + Send + where + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + 'static, + { + // Erase the concrete return type `R` into `Box` so we can call + // through the dyn-compatible `run_interaction_erased`. + let erased: ErasedInteraction = Box::new(move |txn| { + let fut = func(txn); + async move { Ok(Box::new(fut.await?) as Box) }.boxed() + }); + + async move { + let boxed = self.run_interaction_erased(name, erased).await?; + boxed.downcast::().map(|b| *b).map_err(|_| { + anyhow::anyhow!( + "run_interaction return type mismatch (this is a Synapse programming error)" + ) + }) + } + } +} + +/// Blanket-implemented for every [`DatabasePool`] so +/// [`run_interaction`](DatabasePoolExt::run_interaction) is always available +impl DatabasePoolExt for T {} + +/// A transaction to interact with the database +/// +/// Based on the ergonomics of [`tokio_postgres::Transaction`] +#[async_trait::async_trait] +pub trait Transaction: Send { + /// Run a database query, returning a list of resulting rows. + /// + /// We expect the `sql` query should use `?` placeholders for the `args`. Downstream + /// implementations should string-replace `?` as necessary. + // + // `async` as this is representing a round-trip between the app and database + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; +} + +/// A single backend-agnostic value within a [`DbRow`]. +/// +/// Each pool maps the values its database driver hands back into this common +/// set, so callers can work with one representation regardless of engine. +#[derive(Debug, Clone, PartialEq)] +pub enum DbValue { + /// A SQL `NULL`. + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(String), +} + +/// A row of data returned from the database by a query. +/// +/// Each pool converts the cells its database driver hands back into the +/// engine-agnostic [`DbValue`] representation, so a row is simply a list of them. +/// Values are pulled out by their numeric index with [`DbRowExt::try_get`]. +pub type DbRow = Vec; + +/// Extension methods for reading typed values out of a [`DbRow`]. +/// +/// Based on [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) +/// converts the [`DbValue`] at a given index into the requested type via +/// [`FromDbValue`] (our analogue of `tokio-postgres`'s `FromSql`). +pub trait DbRowExt { + /// Deserializes a value from the row, specified by its numeric index, + /// returning an error if the index is out of bounds or the value cannot be + /// converted into `T`. + fn try_get(&self, index: usize) -> Result; +} + +impl DbRowExt for DbRow { + fn try_get(&self, index: usize) -> Result { + let value = self.get(index).cloned().ok_or_else(|| { + anyhow::anyhow!( + "tried to get column {index} but the row only has {} column(s)", + self.len() + ) + })?; + + T::from_value(value) + } +} + +/// Converts a backend-agnostic [`DbValue`] into a concrete Rust type, analogous to +/// `tokio-postgres`'s `FromSql`. +pub trait FromDbValue: Sized { + fn from_value(value: DbValue) -> Result; +} + +impl FromDbValue for bool { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Bool(b) => Ok(b), + // SQLite has no native boolean type and stores them as integers. + DbValue::Int(i) => match i { + 0 => Ok(false), + 1 => Ok(true), + _ => anyhow::bail!("cannot read DbValue::Int({i}) as bool"), + }, + other => anyhow::bail!("cannot read {other:?} as bool"), + } + } +} + +impl FromDbValue for i64 { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Int(i) => Ok(i), + other => anyhow::bail!("cannot read {other:?} as i64"), + } + } +} + +impl FromDbValue for f64 { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Float(f) => Ok(f), + other => anyhow::bail!("cannot read {other:?} as f64"), + } + } +} + +impl FromDbValue for String { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Text(s) => Ok(s), + other => anyhow::bail!("cannot read {other:?} as String"), + } + } +} + +impl FromDbValue for Option { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Null => Ok(None), + other => Ok(Some(T::from_value(other)?)), + } + } +} diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs new file mode 100644 index 0000000000..5e62b06656 --- /dev/null +++ b/rust/src/storage/db/python_db_pool.rs @@ -0,0 +1,371 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A database pool that calls into Python to re-use the same database pool used on the +//! Python side. This is desirable because we want to avoid having two separate database +//! pools (one for Rust, one for Python) to avoid database connection exhaustion +//! problems. This is a stepping stone until all of our database interactions are in +//! Rust. +//! +//! We have these main classes: +//! - Database pool [`PythonDatabasePoolWrapper`] (implements [`DatabasePool`]) which +//! allows you to start a... +//! - transaction [`LoggingTransactionWrapper`] (implements [`Transaction`]) and query +//! the database. + +use std::sync::{Arc, Mutex}; + +use anyhow::Context; +use futures::FutureExt; +use once_cell::sync::OnceCell; +use pyo3::{ + exceptions::{PyAssertionError, PyRuntimeError, PyTypeError}, + intern, + prelude::*, + types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, +}; + +use crate::deferred::run_python_awaitable; +use crate::storage::db::{ + DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, +}; + +/// A reference to the `synapse.storage.engines` module. +static STORAGE_ENGINES_MODULE: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.storage.engines` module. +fn storage_engines_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(STORAGE_ENGINES_MODULE + .get_or_try_init(|| py.import("synapse.storage.engines").map(Into::into))? + .bind(py)) +} + +static SQLITE3_ENGINE_CLASS: OnceCell> = OnceCell::new(); +static POSTGRES_ENGINE_CLASS: OnceCell> = OnceCell::new(); + +/// Access to the `Sqlite3Engine` class +fn sqlite3_engine_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(SQLITE3_ENGINE_CLASS + .get_or_try_init(|| { + storage_engines_module(py)? + .getattr("Sqlite3Engine") + .map(Into::into) + })? + .bind(py)) +} + +/// Access to the `PostgresEngine` class +fn postgres_engine_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(POSTGRES_ENGINE_CLASS + .get_or_try_init(|| { + storage_engines_module(py)? + .getattr("PostgresEngine") + .map(Into::into) + })? + .bind(py)) +} + +/// The database engines we support in the Python side of Synapse +#[derive(Copy, Clone, Debug)] +pub enum DatabaseEngine { + Sqlite, + Postgres, +} + +impl DatabaseEngine { + pub fn supports_using_any_list(&self) -> bool { + match self { + DatabaseEngine::Sqlite => false, + DatabaseEngine::Postgres => true, + } + } +} + +/// Wrapper for a `DatabasePool` from the Python side of Synapse. +pub struct PythonDatabasePoolWrapper { + /// The underlying Python `DatabasePool` + database_pool_py: Py, + + /// The Twisted reactor. We need this to marshal back onto the reactor thread + /// (via `callFromThread`) when starting transactions, since Twisted's thread + /// pool machinery must be driven from there. + reactor: Py, +} + +impl PythonDatabasePoolWrapper { + /// Build a wrapper around the Python `DatabasePool` (e.g. + /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. + pub fn new(database_pool_py: Py, reactor: Py) -> Self { + Self { + database_pool_py, + reactor, + } + } +} + +#[async_trait::async_trait] +impl DatabasePool for PythonDatabasePoolWrapper { + async fn run_interaction_erased( + &self, + name: &'static str, + func: ErasedInteraction, + ) -> ErasedResult { + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB thread and + // expects a synchronous return value. Since we can't round-trip an arbitrary + // Rust result back out through Python (remember, `func` returns an + // `ErasedResult`, not a `PyAny`), the callback stashes the result here and we + // pick it up once the deferred fires. + // + // Note the callback may run more than once (`runInteraction` retries on + // serialization/deadlock errors), so we only trust this slot once the + // deferred has fired, i.e. once the transaction has finally committed or + // failed. + let result_slot: Arc>> = Arc::new(Mutex::new(None)); + + // Build the callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`, plus owned handles we can move onto + // the reactor thread. We drive `func` to completion in the callback; the + // Python query path is synchronous under the hood, so it's safe to block + // this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { + let callback = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let txn_py = args.get_item(0)?; + let mut txn = txn_py.extract::()?; + + // Since we expect people to only call `.await` on + // [`Transaction`] related methods (mentioned in the + // [`Transaction`] docstring) AND because there is no actual + // async work to suspend on in the Python [`Transaction`] + // (resolves synchonously), we can get away with polling once as + // it should immediately resolve to [`Poll::Ready`]. Getting + // [`Poll::Pending`] would be considered a programming error. + // + // Alternatively, we could just use `futures::executor::block_on` + // which is probably cleaner but a single-shot poll is more + // enforcing of the concept we want to represent. + match func(&mut txn).now_or_never() { + Some(Ok(value)) => { + let mut callback_slot = callback_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; + *callback_slot = Some(Ok(value)); + Ok(py.None()) + } + Some(Err(err)) => { + // Re-raise into Python so `runInteraction` rolls the + // transaction back (and can apply its retry logic for + // serialization/deadlock errors). + let py_err = anyhow_to_pyerr(&err); + let mut callback_slot = callback_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; + *callback_slot = Some(Err(err)); + Err(py_err) + } + None => { + Err(PyAssertionError::new_err( + "The `run_interaction` transaction callback future returned `Poll::Pending`, \ + but we expect Synapse Python database work to resolve synchronously. \ + This is a Synapse programming error: genuine async work is \ + not supported here.", + )) + } + } + }, + )? + .unbind(); + + Ok(( + callback, + self.database_pool_py.clone_ref(py), + self.reactor.clone_ref(py), + )) + }) + .map_err(anyhow::Error::from)?; + + // Use `runInteraction` directly + let run_interaction_outcome = run_python_awaitable(reactor, move |py| { + database_pool_py + .bind(py) + .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + }) + .await; + + // Return the result we captured based on if `runInteraction` was successful + let captured_result = result_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `result_slot`: {:#}", err))? + .take(); + match run_interaction_outcome { + // Only return the `captured_result` if `runInteraction` succeeded. We don't + // want to accidentally return a successful result when the transaction + // actually failed to commit. + Ok(_) => match captured_result { + Some(result) => result, + // This is unexpected as we either expect `runInteraction` to have + // completed successfully and run the provided `callback` which runs the + // `func` and we capture a result or it fails. + None => Err(anyhow::anyhow!( + "Expected to capture result after running `runInteraction` and seeing it succeed (but saw nothing). \ + This is a Synapse programming error." + )), + }, + Err(py_err) => Err(anyhow::Error::from(py_err)).with_context(|| format!("run_interaction(name={}) failed", name)), + } + } +} + +/// Convert an [`anyhow::Error`] into a [`PyErr`] to re-raise into Python. +/// +/// If the error wraps an original Python exception (e.g. a database error +/// surfaced through [`Transaction::query`]), we re-raise *that* exception so +/// Synapse's transaction machinery can apply its retry logic +/// (serialization/deadlock detection) on the real error. +fn anyhow_to_pyerr(err: &anyhow::Error) -> PyErr { + if let Some(py_err) = err.downcast_ref::() { + return Python::attach(|py| py_err.clone_ref(py)); + } + PyRuntimeError::new_err(format!("{err:#}")) +} + +/// Given a Python `LoggingTransaction`, figures out the database engine that backs it +fn detect_engine(txn_py: &Bound<'_, PyAny>) -> PyResult { + let py = txn_py.py(); + let database_engine = txn_py.getattr(intern!(py, "database_engine"))?; + + // Compare against the actual engine classes imported from Python (the PyO3 + // equivalent of an `isinstance` check). + if database_engine.is_instance(postgres_engine_class(py)?)? { + Ok(DatabaseEngine::Postgres) + } else if database_engine.is_instance(sqlite3_engine_class(py)?)? { + Ok(DatabaseEngine::Sqlite) + } else { + Err(PyTypeError::new_err(format!( + "Unknown database engine {}. This is a Synapse programming error.", + database_engine.get_type().name()? + ))) + } +} + +/// Wrapper for a `LoggingTransaction` from the Python side of Synapse. +pub struct LoggingTransactionWrapper { + /// The underlying `LoggingTransaction` + /// + /// We purposely avoid `Bound<'py, PyAny>` so it can be stored and moved freely + /// across threads (as required by `Transaction` trait). + logging_transaction_py: Py, + + /// Disambiguate which underlying database engine we're working with + /// + /// Some features are only available on Postgres vs SQLite and the queries need to + /// be differentiated (for compatibility or performance reasons). + pub database_engine: DatabaseEngine, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for LoggingTransactionWrapper { + type Error = PyErr; + + /// Extract from a Python `LoggingTransaction` passed as an argument. + fn extract(logging_transaction_py: Borrowed<'a, 'py, PyAny>) -> PyResult { + let database_engine = detect_engine(&logging_transaction_py.to_owned())?; + Ok(Self { + logging_transaction_py: logging_transaction_py.to_owned().unbind(), + database_engine, + }) + } +} + +impl LoggingTransactionWrapper { + /// Calls the Python `LoggingTransaction.execute` function. + fn execute<'py>( + &mut self, + py: Python<'py>, + sql: &str, + args: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let execute_fn = self + .logging_transaction_py + .bind(py) + .getattr(intern!(py, "execute"))?; + execute_fn.call1((sql, args))?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl Transaction for LoggingTransactionWrapper { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + Python::attach(|py| -> PyResult> { + // Convert the Rust `&[&str]` of SQL parameters into a Python sequence + // so it can be passed through to the Python-side `execute`. + // + // We don't need to do anything to the SQL as the `?`-style arg placeholders + // already align with what `LoggingTransaction` expects. + let args = PyList::new(py, args)?; + self.execute(py, sql, args.as_any())?; + + // Pull the rows back out, converting each cell from its Python type + // into the engine-agnostic `DbValue` representation as we go. + let rows_py = self + .logging_transaction_py + .bind(py) + .call_method0(intern!(py, "fetchall"))?; + + let mut rows: Vec = Vec::new(); + for row_py in rows_py.try_iter()? { + let row_py = row_py?; + let mut row: DbRow = Vec::new(); + for cell in row_py.try_iter()? { + row.push(py_cell_to_value(&cell?)?); + } + rows.push(row); + } + + Ok(rows) + }) + .map_err(anyhow::Error::from) + } +} + +/// Convert a single cell from a Python row into a backend-agnostic [`DbValue`] by +/// inspecting its Python type (the pyo3 equivalent of `isinstance` checks). +fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { + // `None` maps to SQL `NULL`. + if cell.is_none() { + return Ok(DbValue::Null); + } + + // A `bool` *is* an `int` in SQLite, so ensure we try `bool` first. + if let Ok(b) = cell.cast::() { + Ok(DbValue::Bool(b.extract()?)) + } else if let Ok(i) = cell.cast::() { + Ok(DbValue::Int(i.extract()?)) + } else if let Ok(f) = cell.cast::() { + Ok(DbValue::Float(f.extract()?)) + } else if let Ok(s) = cell.cast::() { + Ok(DbValue::Text(s.to_string())) + } else { + Err(PyTypeError::new_err(format!( + "unsupported column type {} returned from the database", + cell.get_type().name()? + ))) + } +} diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs new file mode 100644 index 0000000000..735fcecb26 --- /dev/null +++ b/rust/src/storage/mod.rs @@ -0,0 +1,17 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +pub mod db; +pub mod store; diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs new file mode 100644 index 0000000000..b339d7748c --- /dev/null +++ b/rust/src/storage/store.rs @@ -0,0 +1,110 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use futures::FutureExt; +use serde::Serialize; + +use crate::storage::db::{DatabasePool, DatabasePoolExt, DbRowExt}; + +/// Currently supported per-user features +#[derive(Serialize, Debug)] +pub enum PerUserExperimentalFeature { + #[serde(rename = "msc3881")] + MSC3881, + #[serde(rename = "msc3575")] + MSC3575, + #[serde(rename = "msc4222")] + MSC4222, +} + +impl std::fmt::Display for PerUserExperimentalFeature { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + // Serialize so we can use the serde name of the variant as the source of truth + serde_json::to_string(self) + .unwrap_or_else(|err| format!( + "", + self, err + )) + // Remove the surrounding quotes from JSON serialization + .trim_matches('"') + ) + } +} + +pub struct Store { + pub db_pool: Box, +} + +impl Store { + /// Checks whether a given feature is enabled/disabled for this user + /// + /// If there is no entry, returns None + pub async fn is_feature_enabled_for_user( + &self, + user_id: &str, + feature: PerUserExperimentalFeature, + ) -> Result, anyhow::Error> { + // We need owned copies to move into the callback because it is `'static` (it + // may be moved to another thread). We use `Arc` rather than `String` so + // the per-call clone is just a cheap refcount bump rather than a fresh + // allocation. + let user_id: Arc = user_id.into(); + let feature: Arc = feature.to_string().into(); + + let is_feature_enabled_for_user = self + .db_pool + .run_interaction("is_feature_enabled_for_user", move |txn| { + let user_id = user_id.clone(); + let feature = feature.clone(); + async move { + let rows = txn + .query( + r#" + SELECT enabled + FROM per_user_experimental_features + WHERE user_id = ? AND feature = ? + "#, + &[user_id.as_ref(), feature.as_ref()], + ) + .await?; + + let enabled = match &rows[..] { + // No row for this user + [] => None, + // Otherwise, we should only find a single row for this (user, feature) + [row] => Some(row.try_get(0)?), + rows => { + anyhow::bail!( + "Unexpected number of rows returned (expected exactly 0 or 1, saw {}). \ + This probably means the SQL query probably doesn't match our expectations.", + rows.len(), + ); + } + }; + + Ok(enabled) + } + .boxed() + }) + .await?; + + Ok(is_feature_enabled_for_user) + } +} diff --git a/rust/src/types/mod.rs b/rust/src/types/mod.rs index ffb19a83a2..f23b44da8e 100644 --- a/rust/src/types/mod.rs +++ b/rust/src/types/mod.rs @@ -38,31 +38,31 @@ fn user_id_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// Represents the user making a request. #[pyclass(frozen, skip_from_py_object, get_all, eq)] -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Requester { /// The ID of the user making the request, in string form (see /// [`Self::user`] for accessing the parsed `UserID`). - user_id: String, + pub user_id: String, /// The ID of the access token used for this request, or None for /// appservices, guests, and tokens generated by the admin API - access_token_id: Option, + pub access_token_id: Option, /// True if the user making this request is a guest - is_guest: bool, + pub is_guest: bool, /// Any scopes associated with the access token used for this request, or an /// empty set if no token or a non-oauth token was used - scope: HashSet, + pub scope: HashSet, /// True if the user making this request is shadow banned - shadow_banned: bool, + pub shadow_banned: bool, /// The device_id which was set at authentication time, or None for /// appservices, guests, and tokens generated by the admin API - device_id: Option, + pub device_id: Option, /// The ID of the AS requesting on behalf of the user, or None. - app_service_id: Option, + pub app_service_id: Option, /// The entity that authenticated when making the request. /// /// This is different to the `user_id` when an admin user or the server is /// "puppeting" the user. - authenticated_entity: String, + pub authenticated_entity: String, } #[pymethods] diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 35a69f977c..78d664cf5f 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://famedly.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://famedly.github.io/synapse/schema/synapse/v1.156/synapse-config.schema.json +$id: https://famedly.github.io/synapse/schema/synapse/v1.157/synapse-config.schema.json type: object properties: famedly_maximum_refresh_token_lifetime: @@ -218,6 +218,46 @@ properties: `include_offline_users_on_sync` to `true` will always include offline users in the results. default: false + last_active_granularity: + $ref: "#/$defs/duration" + description: >- + How long after a user was last active that they are still shown as + "currently active" to other users. Larger values reduce the rate of + presence updates sent to other users and servers. + + + *Added in Synapse 1.156.0.* + default: 1m + examples: + - 5m + sync_online_timeout: + $ref: "#/$defs/duration" + description: >- + How long after a client's last sync request their presence is + switched to offline. Clients are expected to keep a sync request + open at (almost) all times while online, so this only needs to + cover the gap between two consecutive sync requests. Note that if + `rc_presence` is set to ratelimit how often syncs can affect + presence, this must be greater than the ratelimit's interval or + users will incorrectly be marked as offline in between syncs. + + + *Added in Synapse 1.156.0.* + default: 30s + examples: + - 1m + idle_timeout: + $ref: "#/$defs/duration" + description: >- + How long after a user was last active that their presence is + switched to "unavailable" (idle) while they remain connected. Must + be greater than `last_active_granularity`. + + + *Added in Synapse 1.156.0.* + default: 5m + examples: + - 10m examples: - enabled: false include_offline_users_on_sync: false @@ -4854,6 +4894,45 @@ properties: include_content: false group_unread_count_by_room: false jitter_delay: 10s + push_rules: + type: object + description: Options for push rules + properties: + limits: + type: object + description: Limits on the size of push rules that users can have + properties: + rule_count: + type: integer + default: 10000 + description: >- + This is the total number of push rules that each user can have. + Power users may expect to have one push rule per room. + rule_id_length: + type: integer + default: 300 + description: >- + This is the maximum length of a push rule ID, in bytes. + Push rule IDs need to be allowed to be at least as long + as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit. + rule_size: + type: integer + default: 1024 + description: >- + This is the maximum size of a push rule's body, in bytes. + + The exact mechanism for calculating this size is currently an implementation + detail, subject to change. + This limit should be treated as a coarse sanity limit rather than something + to fine-tune. + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit and a mechanism for calculating it. encryption_enabled_by_default_for_room_type: type: string description: >- @@ -5329,6 +5408,18 @@ properties: default: [] examples: - - "!foo:example.com" + exclude_rooms_from_presence: + type: array + description: >- + A list of rooms to exclude from presence updates. Presence will not be + routed between two users solely because they share one of these rooms. + Users who also share a non-excluded room continue to exchange presence as + normal. + items: + type: string + default: [] + examples: + - - "!foo:example.com" opentracing: type: object description: >- diff --git a/scripts-dev/release.py b/scripts-dev/release.py index ea4fb0f142..af4a199277 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -260,7 +260,18 @@ def _prepare() -> None: schema_file = "schema/synapse-config.schema.yaml" major_minor_version = ".".join(new_version.split(".")[:2]) url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" - subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Find/replace the `$id: ...` line in `schema/synapse-config.schema.yaml` with a new + # unique identifier for this release + # + # We use two `open(...)` blocks as it's easier to read/write then figure out the + # seek/truncate dance with one. + with open(schema_file) as f: + contents = f.read() + contents = re.sub( + r"^\$id: .*", f"$id: {url}", contents, count=1, flags=re.MULTILINE + ) + with open(schema_file, "w") as f: + f.write(contents) # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) diff --git a/synapse/__init__.py b/synapse/__init__.py index 3acfc1a0d7..a223066f04 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -49,7 +49,9 @@ from twisted.internet import asyncioreactor - asyncioreactor.install(asyncio.get_event_loop()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + asyncioreactor.install(loop) # Twisted and canonicaljson will fail to import when this file is executed to # get the __version__ during a fresh install. That's OK and subsequent calls to diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py deleted file mode 100644 index 3b37f39875..0000000000 --- a/synapse/api/auth/msc3861_delegated.py +++ /dev/null @@ -1,618 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2023 The Matrix.org Foundation. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -import logging -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable -from urllib.parse import urlencode - -from authlib.oauth2 import ClientAuth -from authlib.oauth2.auth import encode_client_secret_basic, encode_client_secret_post -from authlib.oauth2.rfc7523 import ClientSecretJWT, PrivateKeyJWT, private_key_jwt_sign -from authlib.oauth2.rfc7662 import IntrospectionToken -from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url - -from synapse.api.auth.base import BaseAuth -from synapse.api.errors import ( - AuthError, - HttpResponseException, - InvalidClientTokenError, - SynapseError, - UnrecognizedRequestError, -) -from synapse.http.site import SynapseRequest -from synapse.logging.opentracing import ( - active_span, - force_tracing, - inject_request_headers, - start_active_span, -) -from synapse.metrics import SERVER_NAME_LABEL -from synapse.synapse_rust.http_client import HttpClient -from synapse.types import Requester, UserID, create_requester -from synapse.util.caches.cached_call import RetryOnExceptionCachedCall -from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext -from synapse.util.duration import Duration -from synapse.util.json import json_decoder - -from . import introspection_response_timer - -if TYPE_CHECKING: - from synapse.rest.admin.experimental_features import ExperimentalFeature - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - -# Scope as defined by MSC2967 -# https://github.com/matrix-org/matrix-spec-proposals/pull/2967 -UNSTABLE_SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*" -UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" -STABLE_SCOPE_MATRIX_API = "urn:matrix:client:api:*" -STABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:client:device:" - -# Scope which allows access to the Synapse admin API -SCOPE_SYNAPSE_ADMIN = "urn:synapse:admin:*" - - -def scope_to_list(scope: str) -> list[str]: - """Convert a scope string to a list of scope tokens""" - return scope.strip().split(" ") - - -@dataclass -class IntrospectionResult: - _inner: IntrospectionToken - - # when we retrieved this token, - # in milliseconds since the Unix epoch - retrieved_at_ms: int - - def is_active(self, now_ms: int) -> bool: - if not self._inner.get("active"): - return False - - expires_in = self._inner.get("expires_in") - if expires_in is None: - return True - if not isinstance(expires_in, int): - raise InvalidClientTokenError("token `expires_in` is not an int") - - absolute_expiry_ms = expires_in * 1000 + self.retrieved_at_ms - return now_ms < absolute_expiry_ms - - def get_scope_list(self) -> list[str]: - value = self._inner.get("scope") - if not isinstance(value, str): - return [] - return scope_to_list(value) - - def get_sub(self) -> str | None: - value = self._inner.get("sub") - if not isinstance(value, str): - return None - return value - - def get_username(self) -> str | None: - value = self._inner.get("username") - if not isinstance(value, str): - return None - return value - - def get_name(self) -> str | None: - value = self._inner.get("name") - if not isinstance(value, str): - return None - return value - - def get_device_id(self) -> str | None: - value = self._inner.get("device_id") - if value is not None and not isinstance(value, str): - raise AuthError( - 500, - "Invalid device ID in introspection result", - ) - return value - - -class PrivateKeyJWTWithKid(PrivateKeyJWT): # type: ignore[misc] - """An implementation of the private_key_jwt client auth method that includes a kid header. - - This is needed because some providers (Keycloak) require the kid header to figure - out which key to use to verify the signature. - """ - - def sign(self, auth: Any, token_endpoint: str) -> bytes: - return private_key_jwt_sign( - auth.client_secret, - client_id=auth.client_id, - token_endpoint=token_endpoint, - claims=self.claims, - header={"kid": auth.client_secret["kid"]}, - ) - - -class MSC3861DelegatedAuth(BaseAuth): - AUTH_METHODS = { - "client_secret_post": encode_client_secret_post, - "client_secret_basic": encode_client_secret_basic, - "client_secret_jwt": ClientSecretJWT(), - "private_key_jwt": PrivateKeyJWTWithKid(), - } - - EXTERNAL_ID_PROVIDER = "oauth-delegated" - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self._config = hs.config.experimental.msc3861 - auth_method = MSC3861DelegatedAuth.AUTH_METHODS.get( - self._config.client_auth_method.value, None - ) - # Those assertions are already checked when parsing the config - assert self._config.enabled, "OAuth delegation is not enabled" - assert self._config.issuer, "No issuer provided" - assert self._config.client_id, "No client_id provided" - assert auth_method is not None, "Invalid client_auth_method provided" - - self.server_name = hs.hostname - self._clock = hs.get_clock() - self._http_client = hs.get_proxied_http_client() - self._hostname = hs.hostname - self._admin_token: Callable[[], str | None] = self._config.admin_token - self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users - - self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # # Token Introspection Cache - # This remembers what users/devices are represented by which access tokens, - # in order to reduce overall system load: - # - on Synapse (as requests are relatively expensive) - # - on the network - # - on MAS - # - # Since there is no invalidation mechanism currently, - # the entries expire after 2 minutes. - # This does mean tokens can be treated as valid by Synapse - # for longer than reality. - # - # Ideally, tokens should logically be invalidated in the following circumstances: - # - If a session logout happens. - # In this case, MAS will delete the device within Synapse - # anyway and this is good enough as an invalidation. - # - If the client refreshes their token in MAS. - # In this case, the device still exists and it's not the end of the world for - # the old access token to continue working for a short time. - self._introspection_cache: ResponseCache[str] = ResponseCache( - clock=self._clock, - name="token_introspection", - server_name=self.server_name, - timeout=Duration(minutes=2), - # don't log because the keys are access tokens - enable_logging=False, - ) - - self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( - self._load_metadata - ) - - if isinstance(auth_method, PrivateKeyJWTWithKid): - # Use the JWK as the client secret when using the private_key_jwt method - assert self._config.jwk, "No JWK provided" - self._client_auth = ClientAuth( - self._config.client_id, self._config.jwk, auth_method - ) - else: - # Else use the client secret - client_secret = self._config.client_secret() - assert client_secret, "No client_secret provided" - self._client_auth = ClientAuth( - self._config.client_id, client_secret, auth_method - ) - - async def _load_metadata(self) -> OpenIDProviderMetadata: - if self._config.issuer_metadata is not None: - return OpenIDProviderMetadata(**self._config.issuer_metadata) - url = get_well_known_url(self._config.issuer, external=True) - response = await self._http_client.get_json(url) - metadata = OpenIDProviderMetadata(**response) - # metadata.validate_introspection_endpoint() - return metadata - - async def issuer(self) -> str: - """ - Get the configured issuer - - This will use the issuer value set in the metadata, - falling back to the one set in the config if not set in the metadata - """ - metadata = await self._issuer_metadata.get() - return metadata.issuer or self._config.issuer - - async def account_management_url(self) -> str | None: - """ - Get the configured account management URL - - This will discover the account management URL from the issuer if it's not set in the config - """ - if self._config.account_management_url is not None: - return self._config.account_management_url - - try: - metadata = await self._issuer_metadata.get() - return metadata.get("account_management_uri", None) - # We don't want to raise here if we can't load the metadata - except Exception: - logger.warning("Failed to load metadata:", exc_info=True) - return None - - async def auth_metadata(self) -> dict[str, Any]: - """ - Returns the auth metadata dict - """ - return await self._issuer_metadata.get() - - async def _introspection_endpoint(self) -> str: - """ - Returns the introspection endpoint of the issuer - - It uses the config option if set, otherwise it will use OIDC discovery to get it - """ - if self._config.introspection_endpoint is not None: - return self._config.introspection_endpoint - - metadata = await self._issuer_metadata.get() - return metadata.get("introspection_endpoint") - - async def _introspect_token( - self, token: str, cache_context: ResponseCacheContext[str] - ) -> IntrospectionResult: - """ - Send a token to the introspection endpoint and returns the introspection response - - Parameters: - token: The token to introspect - - Raises: - HttpResponseException: If the introspection endpoint returns a non-2xx response - ValueError: If the introspection endpoint returns an invalid JSON response - JSONDecodeError: If the introspection endpoint returns a non-JSON response - Exception: If the HTTP request fails - - Returns: - The introspection response - """ - # By default, we shouldn't cache the result unless we know it's valid - cache_context.should_cache = False - introspection_endpoint = await self._introspection_endpoint() - raw_headers: dict[str, str] = { - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "application/json", - # Tell MAS that we support reading the device ID as an explicit - # value, not encoded in the scope. This is supported by MAS 0.15+ - "X-MAS-Supports-Device-Id": "1", - } - - args = {"token": token, "token_type_hint": "access_token"} - body = urlencode(args, True) - - # Fill the body/headers with credentials - uri, raw_headers, body = self._client_auth.prepare( - method="POST", uri=introspection_endpoint, headers=raw_headers, body=body - ) - - # Do the actual request - - logger.debug("Fetching token from MAS") - start_time = self._clock.time() - try: - with start_active_span("mas-introspect-token"): - inject_request_headers(raw_headers) - resp_body = await self._rust_http_client.post( - url=uri, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) - except HttpResponseException as e: - end_time = self._clock.time() - introspection_response_timer.labels( - code=e.code, **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - raise - except Exception: - end_time = self._clock.time() - introspection_response_timer.labels( - code="ERR", **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - raise - - logger.debug("Fetched token from MAS") - - end_time = self._clock.time() - introspection_response_timer.labels( - code=200, **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - - resp = json_decoder.decode(resp_body.decode("utf-8")) - - if not isinstance(resp, dict): - raise ValueError( - "The introspection endpoint returned an invalid JSON response." - ) - - # We had a valid response, so we can cache it - cache_context.should_cache = True - return IntrospectionResult( - IntrospectionToken(**resp), retrieved_at_ms=self._clock.time_msec() - ) - - async def is_server_admin(self, requester: Requester) -> bool: - return "urn:synapse:admin:*" in requester.scope - - def _is_access_token_the_admin_token(self, token: str) -> bool: - admin_token = self._admin_token() - if admin_token is None: - return False - return token == admin_token - - async def get_user_by_req( - self, - request: SynapseRequest, - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - """Get a registered user's ID. - - Args: - request: An HTTP request with an access_token query parameter. - allow_guest: If False, will raise an AuthError if the user making the - request is a guest. - allow_expired: If True, allow the request through even if the account - is expired, or session token lifetime has ended. Note that - /login will deliver access tokens regardless of expiration. - - Returns: - Resolves to the requester - Raises: - InvalidClientCredentialsError if no user by that token exists or the token - is invalid. - AuthError if access is denied for the user in the access token - """ - parent_span = active_span() - with start_active_span("get_user_by_req"): - requester = await self._wrapped_get_user_by_req( - request, allow_guest, allow_expired, allow_locked - ) - - if parent_span: - if requester.authenticated_entity in self._force_tracing_for_users: - # request tracing is enabled for this user, so we need to force it - # tracing on for the parent span (which will be the servlet span). - # - # It's too late for the get_user_by_req span to inherit the setting, - # so we also force it on for that. - force_tracing() - force_tracing(parent_span) - parent_span.set_tag( - "authenticated_entity", requester.authenticated_entity - ) - parent_span.set_tag("user_id", requester.user.to_string()) - if requester.device_id is not None: - parent_span.set_tag("device_id", requester.device_id) - if requester.app_service_id is not None: - parent_span.set_tag("appservice_id", requester.app_service_id) - return requester - - async def _wrapped_get_user_by_req( - self, - request: SynapseRequest, - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - access_token = self.get_access_token_from_request(request) - - requester = await self.get_appservice_user(request, access_token) - if not requester: - # TODO: we probably want to assert the allow_guest inside this call - # so that we don't provision the user if they don't have enough permission: - requester = await self.get_user_by_access_token(access_token, allow_expired) - - # Do not record requests from MAS using the virtual `__oidc_admin` user. - if not self._is_access_token_the_admin_token(access_token): - await self._record_request(request, requester) - - request.requester = requester - - return requester - - async def get_user_by_req_experimental_feature( - self, - request: SynapseRequest, - feature: "ExperimentalFeature", - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - try: - requester = await self.get_user_by_req( - request, - allow_guest=allow_guest, - allow_expired=allow_expired, - allow_locked=allow_locked, - ) - if await self.store.is_feature_enabled(requester.user.to_string(), feature): - return requester - - raise UnrecognizedRequestError(code=404) - except (AuthError, InvalidClientTokenError): - if feature.is_globally_enabled(self.hs.config): - # If its globally enabled then return the auth error - raise - - raise UnrecognizedRequestError(code=404) - - def is_request_using_the_admin_token(self, request: SynapseRequest) -> bool: - """ - Check if the request is using the admin token. - - Args: - request: The request to check. - - Returns: - True if the request is using the admin token, False otherwise. - """ - access_token = self.get_access_token_from_request(request) - return self._is_access_token_the_admin_token(access_token) - - async def get_user_by_access_token( - self, - token: str, - allow_expired: bool = False, - ) -> Requester: - if self._is_access_token_the_admin_token(token): - # XXX: This is a temporary solution so that the admin API can be called by - # the OIDC provider. This will be removed once we have OIDC client - # credentials grant support in matrix-authentication-service. - logger.info("Admin token used") - # XXX: that user doesn't exist and won't be provisioned. - # This is mostly fine for admin calls, but we should also think about doing - # requesters without a user_id. - admin_user = UserID("__oidc_admin", self._hostname) - return create_requester( - user_id=admin_user, - scope=["urn:synapse:admin:*"], - ) - - try: - introspection_result = await self._introspection_cache.wrap( - token, self._introspect_token, token, cache_context=True - ) - except Exception: - logger.exception("Failed to introspect token") - raise SynapseError(503, "Unable to introspect the access token") - - logger.debug("Introspection result: %r", introspection_result) - - # TODO: introspection verification should be more extensive, especially: - # - verify the audience - if not introspection_result.is_active(self._clock.time_msec()): - raise InvalidClientTokenError("Token is not active") - - # Let's look at the scope - scope: list[str] = introspection_result.get_scope_list() - - # Determine type of user based on presence of particular scopes - has_user_scope = ( - UNSTABLE_SCOPE_MATRIX_API in scope or STABLE_SCOPE_MATRIX_API in scope - ) - - if not has_user_scope: - raise InvalidClientTokenError("No scope in token granting user rights") - - # Match via the sub claim - sub = introspection_result.get_sub() - if sub is None: - raise InvalidClientTokenError( - "Invalid sub claim in the introspection result" - ) - - user_id_str = await self.store.get_user_by_external_id( - MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub - ) - if user_id_str is None: - # If we could not find a user via the external_id, it either does not exist, - # or the external_id was never recorded - - username = introspection_result.get_username() - if username is None: - raise AuthError( - 500, - "Invalid username claim in the introspection result", - ) - user_id = UserID(username, self._hostname) - - # Try to find a user from the username claim - user_info = await self.store.get_user_by_id(user_id=user_id.to_string()) - if user_info is None: - raise AuthError( - 500, - "User not found", - ) - - # And record the sub as external_id - await self.store.record_user_external_id( - MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub, user_id.to_string() - ) - else: - user_id = UserID.from_string(user_id_str) - - # MAS 0.15+ will give us the device ID as an explicit value for compatibility sessions - # If present, we get it from here, if not we get it in thee scope - device_id = introspection_result.get_device_id() - if device_id is None: - # Find device_ids in scope - # We only allow a single device_id in the scope, so we find them all in the - # scope list, and raise if there are more than one. The OIDC server should be - # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. - device_ids: set[str] = set() - for tok in scope: - if tok.startswith(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX): - device_ids.add(tok[len(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) - elif tok.startswith(STABLE_SCOPE_MATRIX_DEVICE_PREFIX): - device_ids.add(tok[len(STABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) - - if len(device_ids) > 1: - raise AuthError( - 500, - "Multiple device IDs in scope", - ) - - device_id = next(iter(device_ids), None) - - if device_id is not None: - # Sanity check the device_id - if len(device_id) > 255 or len(device_id) < 1: - raise AuthError( - 500, - "Invalid device ID in introspection result", - ) - - # Make sure the device exists - await self.store.get_device( - user_id=user_id.to_string(), device_id=device_id - ) - - # TODO: there is a few things missing in the requester here, which still need - # to be figured out, like: - # - impersonation, with the `authenticated_entity`, which is used for - # rate-limiting, MAU limits, etc. - # - shadow-banning, with the `shadow_banned` flag - # - a proper solution for appservices, which still needs to be figured out in - # the context of MSC3861 - return create_requester( - user_id=user_id, - device_id=device_id, - scope=scope, - ) diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 87918e15dc..1aed7f6de2 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -117,7 +117,7 @@ async def check_auth_blocking( # If the user is already part of the MAU cohort or a trial user if user_id: timestamp = await self.store.user_last_seen_monthly_active(user_id) - if timestamp: + if timestamp is not None: return is_trial = await self.store.is_trial_user(user_id) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0c35b4a7ba..3dfea09b84 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -823,20 +823,24 @@ def __init__(self, code: int, msg: str, response: bytes): super().__init__(code, msg) self.response = response - def to_synapse_error(self) -> SynapseError: - """Make a SynapseError based on an HTTPResponseException + def unsafe_to_verbatim_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. This is useful when a proxied request has failed, and we need to decide how to map the failure onto a matrix error to send back to the client. - An attempt is made to parse the body of the http response as a matrix + An attempt is made to parse the body of the HTTP response as a Matrix error. If that succeeds, the errcode and error message from the body - are used as the errcode and error message in the new synapse error. + are copied verbatim into the new Synapse error. Otherwise, the errcode is set to M_UNKNOWN, and the error message is set to the reason code from the HTTP response. + Safety: + This must ONLY be used on errors from TRUSTED sources, + such as other Synapse workers. + Returns: The error converted to a SynapseError. """ @@ -851,10 +855,68 @@ def to_synapse_error(self) -> SynapseError: j = {} errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg return ProxiedRequestError(self.code, errmsg, errcode, j) + def to_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. + + This is useful when a proxied request has failed, and we need to + decide how to map the failure onto a matrix error to send back to the + client. + + An attempt is made to parse the body of the HTTP response as a Matrix + error. If that succeeds, the errcode and error message from the body + are copied verbatim into the new Synapse error, unless it's of + a forbidden type. + + Otherwise, the errcode is set to M_UNKNOWN, and the error message is + set to the reason code from the HTTP response. + + Safety: + This is the correct method to use when forwarding errors + from upstream requests (e.g. federation, policy servers). + + FIXME: restrict forwarded errors further + + Returns: + The error converted to a SynapseError. + """ + # try to parse the body as json, to get better errcode/msg, but + # default to M_UNKNOWN with the HTTP status as the error text + try: + j = json_decoder.decode(self.response.decode("utf-8")) + except ValueError: + j = {} + + if not isinstance(j, dict): + j = {} + + status = self.code + errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN + errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg + + if errcode == Codes.UNKNOWN_TOKEN: + # We must not relay this error code back down to clients, + # because clients interpret this code to mean that they + # have been logged out. + # See: https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + errcode = Codes.UNKNOWN + + if status == HTTPStatus.UNAUTHORIZED: + status = HTTPStatus.BAD_REQUEST + + return ProxiedRequestError(status, errmsg, errcode, j) + class HomeServerNotSetupException(Exception): """ diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6303cde182..5f6fb4971c 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -40,7 +40,7 @@ TransactionUnusedFallbackKeys, ) from synapse.events import EventBase -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL @@ -560,7 +560,7 @@ async def _serialize( return await self._event_serializer.serialize_events( [FilteredEvent(event=e, membership=None) for e in events], time_now, - config=SerializeEventConfig( + config=await self._event_serializer.create_config( as_client_event=True, # If this is an invite or a knock membership event, then include # any stripped state alongside the event. We could narrow this diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 7c371d161c..f226afc2b4 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -38,6 +38,7 @@ from synapse.config import ( # noqa: F401 oidc, password_auth_providers, push, + push_rules, ratelimiting, redis, registration, @@ -103,6 +104,7 @@ class RootConfig: worker: workers.WorkerConfig authproviders: password_auth_providers.PasswordAuthProviderConfig push: push.PushConfig + push_rules: push_rules.PushRulesConfig spamchecker: spam_checker.SpamCheckerConfig room: room.RoomConfig userdirectory: user_directory.UserDirectoryConfig diff --git a/synapse/config/_util.py b/synapse/config/_util.py index e09c68ebd4..8e1dd52c29 100644 --- a/synapse/config/_util.py +++ b/synapse/config/_util.py @@ -18,12 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any, TypeVar +from typing import Annotated, Any, TypeVar import jsonschema -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, StrictInt, TypeAdapter, ValidationError +from pydantic_core.core_schema import int_schema -from synapse.config._base import ConfigError +from synapse.config._base import Config, ConfigError from synapse.types import JsonDict, StrSequence @@ -97,3 +98,11 @@ def parse_and_validate_mapping( except ValidationError as e: raise ConfigError(str(e)) from e return instances + + +ConfigByteSize = Annotated[ + StrictInt, BeforeValidator(Config.parse_size), int_schema(ge=0) +] +""" +A size in bytes. Pydantic-compatible wrapper for `Config.parse_size` +""" diff --git a/synapse/config/auth.py b/synapse/config/auth.py index 31b332dc09..35b730aed3 100644 --- a/synapse/config/auth.py +++ b/synapse/config/auth.py @@ -36,11 +36,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: if password_config is None: password_config = {} - auth_delegated = (config.get("experimental_features") or {}).get( - "msc3861", {} - ).get("enabled", False) or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) # The default value of password_config.enabled is True, unless auth is delegated passwords_enabled = password_config.get("enabled", not auth_delegated) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 2a40a59e1d..8e4f52aa9e 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -20,30 +20,16 @@ # # -import enum from functools import cache -from typing import TYPE_CHECKING, Any, Optional +from typing import Any import attr -import attr.validators from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config import ConfigError -from synapse.config._base import Config, RootConfig, read_file +from synapse.config._base import Config, read_file from synapse.types import JsonDict, StrSequence -# Determine whether authlib is installed. -try: - import authlib # noqa: F401 - - HAS_AUTHLIB = True -except ImportError: - HAS_AUTHLIB = False - -if TYPE_CHECKING: - # Only import this if we're type checking, as it might not be installed at runtime. - from authlib.jose.rfc7517 import JsonWebKey - @cache def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: @@ -51,308 +37,6 @@ def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: return read_file(file_path, config_path).strip() -class ClientAuthMethod(enum.Enum): - """List of supported client auth methods.""" - - CLIENT_SECRET_POST = "client_secret_post" - CLIENT_SECRET_BASIC = "client_secret_basic" - CLIENT_SECRET_JWT = "client_secret_jwt" - PRIVATE_KEY_JWT = "private_key_jwt" - - -def _parse_jwks(jwks: JsonDict | None) -> Optional["JsonWebKey"]: - """A helper function to parse a JWK dict into a JsonWebKey.""" - - if jwks is None: - return None - - from authlib.jose.rfc7517 import JsonWebKey - - return JsonWebKey.import_key(jwks) - - -def _check_client_secret( - instance: "MSC3861", _attribute: attr.Attribute, _value: str | None -) -> None: - if instance._client_secret and instance._client_secret_path: - raise ConfigError( - ( - "You have configured both " - "`experimental_features.msc3861.client_secret` and " - "`experimental_features.msc3861.client_secret_path`. " - "These are mutually incompatible." - ), - ("experimental", "msc3861", "client_secret"), - ) - # Check client secret can be retrieved - instance.client_secret() - - -def _check_admin_token( - instance: "MSC3861", _attribute: attr.Attribute, _value: str | None -) -> None: - if instance._admin_token and instance._admin_token_path: - raise ConfigError( - ( - "You have configured both " - "`experimental_features.msc3861.admin_token` and " - "`experimental_features.msc3861.admin_token_path`. " - "These are mutually incompatible." - ), - ("experimental", "msc3861", "admin_token"), - ) - # Check client secret can be retrieved - instance.admin_token() - - -@attr.s(slots=True, frozen=True) -class MSC3861: - """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC""" - - enabled: bool = attr.ib(default=False, validator=attr.validators.instance_of(bool)) - """Whether to enable MSC3861 auth delegation.""" - - @enabled.validator - def _check_enabled(self, attribute: attr.Attribute, value: bool) -> None: - # Only allow enabling MSC3861 if authlib is installed - if value and not HAS_AUTHLIB: - raise ConfigError( - "MSC3861 is enabled but authlib is not installed. " - "Please install authlib to use MSC3861.", - ("experimental", "msc3861", "enabled"), - ) - - issuer: str = attr.ib(default="", validator=attr.validators.instance_of(str)) - """The URL of the OIDC Provider.""" - - issuer_metadata: JsonDict | None = attr.ib(default=None) - """The issuer metadata to use, otherwise discovered from /.well-known/openid-configuration as per MSC2965.""" - - client_id: str = attr.ib( - default="", - validator=attr.validators.instance_of(str), - ) - """The client ID to use when calling the introspection endpoint.""" - - client_auth_method: ClientAuthMethod = attr.ib( - default=ClientAuthMethod.CLIENT_SECRET_POST, converter=ClientAuthMethod - ) - """The auth method used when calling the introspection endpoint.""" - - _client_secret: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_client_secret, - ], - ) - """ - The client secret to use when calling the introspection endpoint, - when using any of the client_secret_* client auth methods. - """ - - _client_secret_path: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_client_secret, - ], - ) - """ - Alternative to `client_secret`: allows the secret to be specified in an - external file. - """ - - jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks) - """ - The JWKS to use when calling the introspection endpoint, - when using the private_key_jwt client auth method. - """ - - @client_auth_method.validator - def _check_client_auth_method( - self, attribute: attr.Attribute, value: ClientAuthMethod - ) -> None: - # Check that the right client credentials are provided for the client auth method. - if not self.enabled: - return - - if value == ClientAuthMethod.PRIVATE_KEY_JWT and self.jwk is None: - raise ConfigError( - "A JWKS must be provided when using the private_key_jwt client auth method", - ("experimental", "msc3861", "client_auth_method"), - ) - - if ( - value - in ( - ClientAuthMethod.CLIENT_SECRET_POST, - ClientAuthMethod.CLIENT_SECRET_BASIC, - ClientAuthMethod.CLIENT_SECRET_JWT, - ) - and self.client_secret() is None - ): - raise ConfigError( - f"A client secret must be provided when using the {value} client auth method", - ("experimental", "msc3861", "client_auth_method"), - ) - - introspection_endpoint: str | None = attr.ib( - default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), - ) - """The URL of the introspection endpoint used to validate access tokens.""" - - account_management_url: str | None = attr.ib( - default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), - ) - """The URL of the My Account page on the OIDC Provider as per MSC2965.""" - - _admin_token: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_admin_token, - ], - ) - """ - A token that should be considered as an admin token. - This is used by the OIDC provider, to make admin calls to Synapse. - """ - - _admin_token_path: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_admin_token, - ], - ) - """ - Alternative to `admin_token`: allows the secret to be specified in an - external file. - """ - - def client_secret(self) -> str | None: - """Returns the secret given via `client_secret` or `client_secret_path`.""" - if self._client_secret_path: - return read_secret_from_file_once( - self._client_secret_path, - ("experimental_features", "msc3861", "client_secret_path"), - ) - return self._client_secret - - def admin_token(self) -> str | None: - """Returns the admin token given via `admin_token` or `admin_token_path`.""" - if self._admin_token_path: - return read_secret_from_file_once( - self._admin_token_path, - ("experimental_features", "msc3861", "admin_token_path"), - ) - return self._admin_token - - def check_config_conflicts( - self, root: RootConfig, allow_secrets_in_config: bool - ) -> None: - """Checks for any configuration conflicts with other parts of Synapse. - - Raises: - ConfigError: If there are any configuration conflicts. - """ - - if not self.enabled: - return - - if self._client_secret and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "client_secret"), - ) - - if self.jwk and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "jwk"), - ) - - if self._admin_token and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "admin_token"), - ) - - if ( - root.auth.password_enabled_for_reauth - or root.auth.password_enabled_for_login - ): - raise ConfigError( - "Password auth cannot be enabled when OAuth delegation is enabled", - ("password_config", "enabled"), - ) - - if root.registration.enable_registration: - raise ConfigError( - "Registration cannot be enabled when OAuth delegation is enabled", - ("enable_registration",), - ) - - # We only need to test the user consent version, as if it must be set if the user_consent section was present in the config - if root.consent.user_consent_version is not None: - raise ConfigError( - "User consent cannot be enabled when OAuth delegation is enabled", - ("user_consent",), - ) - - if ( - root.oidc.oidc_enabled - or root.saml2.saml2_enabled - or root.cas.cas_enabled - or root.jwt.jwt_enabled - ): - raise ConfigError("SSO cannot be enabled when OAuth delegation is enabled") - - if bool(root.authproviders.password_providers): - raise ConfigError( - "Password auth providers cannot be enabled when OAuth delegation is enabled" - ) - - if root.captcha.enable_registration_captcha: - raise ConfigError( - "CAPTCHA cannot be enabled when OAuth delegation is enabled", - ("captcha", "enable_registration_captcha"), - ) - - if root.auth.login_via_existing_enabled: - raise ConfigError( - "Login via existing session cannot be enabled when OAuth delegation is enabled", - ("login_via_existing_session", "enabled"), - ) - - if root.registration.refresh_token_lifetime: - raise ConfigError( - "refresh_token_lifetime cannot be set when OAuth delegation is enabled", - ("refresh_token_lifetime",), - ) - - if root.registration.nonrefreshable_access_token_lifetime: - raise ConfigError( - "nonrefreshable_access_token_lifetime cannot be set when OAuth delegation is enabled", - ("nonrefreshable_access_token_lifetime",), - ) - - if root.registration.session_lifetime: - raise ConfigError( - "session_lifetime cannot be set when OAuth delegation is enabled", - ("session_lifetime",), - ) - - if root.registration.enable_3pid_changes: - raise ConfigError( - "enable_3pid_changes cannot be enabled when OAuth delegation is enabled", - ("enable_3pid_changes",), - ) - - @attr.s(auto_attribs=True, frozen=True, slots=True) class MSC3866Config: """Configuration for MSC3866 (mandating approval for new users)""" @@ -493,18 +177,14 @@ def read_config( # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) - # MSC3861: Matrix architecture change to delegate authentication via OIDC - try: - self.msc3861 = MSC3861(**experimental.get("msc3861", {})) - except ValueError as exc: + # MSC3861 was replaced by the stable Matrix Authentication Service integration. + msc3861_config = experimental.get("msc3861", {}) + if msc3861_config: # non-empty dict raise ConfigError( - "Invalid MSC3861 configuration", ("experimental", "msc3861") - ) from exc - - # Check that none of the other config options conflict with MSC3861 when enabled - self.msc3861.check_config_conflicts( - self.root, allow_secrets_in_config=allow_secrets_in_config - ) + "experimental_features.msc3861 was removed. " + "Use the matrix_authentication_service configuration instead.", + ("experimental", "msc3861"), + ) self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False @@ -526,15 +206,15 @@ def read_config( # See https://github.com/element-hq/synapse/issues/19524 self.msc4370_enabled = experimental.get("msc4370_enabled", False) - auth_delegated = self.msc3861.enabled or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) if ( self.msc4108_enabled or self.msc4108_delegation_endpoint is not None ) and not auth_delegated: raise ConfigError( - "MSC4108 requires MSC3861 or matrix_authentication_service to be enabled", + "MSC4108 requires matrix_authentication_service to be enabled", ("experimental", "msc4108_delegation_endpoint"), ) @@ -610,6 +290,10 @@ def read_config( # (and MSC4308: Thread Subscriptions extension to Sliding Sync) self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) + # MSC4446: Allow moving the fully read marker backwards. + # Tracked in: https://github.com/element-hq/synapse/issues/19940 + self.msc4446_enabled: bool = experimental.get("msc4446_enabled", False) + # MSC4354: Sticky Events # Tracked in: https://github.com/element-hq/synapse/issues/19409 # Note that sticky events persisted before this feature is enabled will not be diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 94ebe583a4..e2ae182fc3 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,6 +19,8 @@ # # +from synapse.config.push_rules import PushRulesConfig + from ._base import ConfigError, RootConfig from .account_validity import AccountValidityConfig from .api import ApiConfig @@ -102,6 +104,7 @@ class HomeServerConfig(RootConfig): EmailConfig, PasswordAuthProviderConfig, PushConfig, + PushRulesConfig, SpamCheckerConfig, RoomConfig, UserDirectoryConfig, diff --git a/synapse/config/mas.py b/synapse/config/mas.py index 6973e9ae58..a26ad1a13f 100644 --- a/synapse/config/mas.py +++ b/synapse/config/mas.py @@ -102,13 +102,6 @@ def check_config_conflicts( if not self.enabled: return - if root.experimental.msc3861.enabled: - raise ConfigError( - "Experimental MSC3861 was replaced by Matrix Authentication Service." - "Please disable MSC3861 or disable Matrix Authentication Service.", - ("experimental", "msc3861"), - ) - if ( root.auth.password_enabled_for_reauth or root.auth.password_enabled_for_login diff --git a/synapse/config/push_rules.py b/synapse/config/push_rules.py new file mode 100644 index 0000000000..deb7e8dde1 --- /dev/null +++ b/synapse/config/push_rules.py @@ -0,0 +1,63 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Annotated, Any + +from pydantic import ( + Field, + StrictInt, + ValidationError, +) + +from synapse.config._util import ConfigByteSize +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel + +from ._base import Config, ConfigError + + +class PushRulesLimitsConfig(ParseModel): + # Chosen arbitrarily, but with the rough rationale that a user + # might have on the order of 10k rooms and want to set a push rule override for each one. + rule_count: Annotated[StrictInt, Field(ge=0)] = 10_000 + + # Chosen arbitrarily, but with the rationale that room IDs are allowed to be up to 255 bytes + # and they are often used in rule IDs. + rule_id_length: Annotated[StrictInt, Field(ge=1)] = 300 + + # Chosen arbitrarily, but with the rationale that real-world push rules don't get + # nearly this big in practice. + # Even 512 bytes would probably have been fine, but we should leave space for the use cases + # of push rules to grow in the future. + rule_size: Annotated[ConfigByteSize, Field(ge=1)] = 1024 + + +class PushRulesConfigModel(ParseModel): + limits: PushRulesLimitsConfig = Field(default_factory=PushRulesLimitsConfig) + + +class PushRulesConfig(Config): + section = "push_rules" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + raw_config = config.get("push_rules", {}) + + try: + parsed = PushRulesConfigModel(**raw_config) + except ValidationError as e: + raise ConfigError( + f"Could not validate configuration: {e}", + path=("push_rules",), + ) from e + + self.limits = parsed.limits diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 437e2b1afe..3699936ae4 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -148,13 +148,11 @@ def read_config( self.enable_set_displayname = config.get("enable_set_displayname", True) self.enable_set_avatar_url = config.get("enable_set_avatar_url", True) - auth_delegated = (config.get("experimental_features") or {}).get( - "msc3861", {} - ).get("enabled", False) or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) - # The default value of enable_3pid_changes is True, unless msc3861 is enabled. + # The default value of enable_3pid_changes is True, unless auth is delegated. self.enable_3pid_changes = config.get("enable_3pid_changes", not auth_delegated) self.disable_msisdn_registration = config.get( diff --git a/synapse/config/room.py b/synapse/config/room.py index e698c7bafd..6a1f4d1eb8 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -48,23 +48,23 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: RoomDefaultEncryptionTypes.OFF, ) if encryption_for_room_type == RoomDefaultEncryptionTypes.ALL: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, RoomCreationPreset.PUBLIC_CHAT, - ] + } elif encryption_for_room_type == RoomDefaultEncryptionTypes.INVITE: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, - ] + } elif ( encryption_for_room_type == RoomDefaultEncryptionTypes.OFF or encryption_for_room_type is False ): # PyYAML translates "off" into False if it's unquoted, so we also need to # check for encryption_for_room_type being False. - self.encryption_enabled_by_default_for_room_presets = [] + self.encryption_enabled_by_default_for_room_presets = set() else: raise ConfigError( "Invalid value for encryption_enabled_by_default_for_room_type" diff --git a/synapse/config/server.py b/synapse/config/server.py index 3b57531fa4..4dafeff0b7 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -37,6 +37,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.types import JsonDict, StrSequence +from synapse.util.duration import Duration from synapse.util.module_loader import load_module from synapse.util.stringutils import parse_and_validate_server_name @@ -177,6 +178,19 @@ def generate_ip_set( DEFAULT_ROOM_VERSION = "10" +# Defaults for the presence state machine timers, in milliseconds. Overridden +# by the corresponding options in the `presence` config section. +# +# How long after a user was last active that they are still considered +# "currently_active". +DEFAULT_LAST_ACTIVE_GRANULARITY = 60 * 1000 +# How long to wait until a new /events or /sync request before assuming the +# client has gone. +DEFAULT_SYNC_ONLINE_TIMEOUT = 30 * 1000 +# How long to wait before marking the user as idle. Compared against last +# active. +DEFAULT_IDLE_TIMER = 5 * 60 * 1000 + ROOM_COMPLEXITY_TOO_GREAT = ( "Your homeserver is unable to join rooms this large or complex. " "Please speak to your server administrator, or upgrade your instance " @@ -505,6 +519,32 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "include_offline_users_on_sync", False ) + # Timers controlling the presence state machine. + self.presence_last_active_granularity = self.parse_duration( + presence_config.get( + "last_active_granularity", DEFAULT_LAST_ACTIVE_GRANULARITY + ) + ) + self.presence_sync_online_timeout = self.parse_duration( + presence_config.get("sync_online_timeout", DEFAULT_SYNC_ONLINE_TIMEOUT) + ) + self.presence_idle_timeout = self.parse_duration( + presence_config.get("idle_timeout", DEFAULT_IDLE_TIMER) + ) + if self.presence_last_active_granularity <= 0: + raise ConfigError( + "'presence.last_active_granularity' must be a positive duration" + ) + if self.presence_sync_online_timeout <= 0: + raise ConfigError( + "'presence.sync_online_timeout' must be a positive duration" + ) + if self.presence_idle_timeout <= self.presence_last_active_granularity: + raise ConfigError( + "'presence.idle_timeout' must be greater than " + "'presence.last_active_granularity'" + ) + # Custom presence router module # This is the legacy way of configuring it (the config should now be put in the modules section) self.presence_router_module_class = None @@ -896,6 +936,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("exclude_rooms_from_sync") or [] ) + self.rooms_to_exclude_from_presence: list[str] = ( + config.get("exclude_rooms_from_presence") or [] + ) + delete_stale_devices_after: str | None = ( config.get("delete_stale_devices_after") or None ) @@ -910,13 +954,34 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # The maximum allowed delay duration for delayed events (MSC4140). max_event_delay_duration = config.get("max_event_delay_duration") if max_event_delay_duration is not None: - self.max_event_delay_ms: int | None = self.parse_duration( - max_event_delay_duration - ) - if self.max_event_delay_ms <= 0: - raise ConfigError("max_event_delay_duration must be a positive value") + max_event_delay_ms = self.parse_duration(max_event_delay_duration) + if max_event_delay_ms <= 0: + raise ConfigError( + "'max_event_delay_duration' must be a positive value if set", + ("max_event_delay_duration",), + ) + self.max_event_delay_duration = Duration(milliseconds=max_event_delay_ms) else: - self.max_event_delay_ms = None + self.max_event_delay_duration = Duration() + + # The maximum number of delayed events a user may have scheduled at a time. + # (Defined here despite being experimental to be near the other MSC4140 config) + experimental = config.get("experimental_features") or {} + self.max_delayed_events_per_user: int = experimental.get( + "msc4140_max_delayed_events_per_user", 100 + ) + if ( + not isinstance(self.max_delayed_events_per_user, int) + or self.max_delayed_events_per_user < 0 + ): + raise ConfigError( + "'msc4140_max_delayed_events_per_user' must be a non-negative integer", + ("experimental", "msc4140_max_delayed_events_per_user"), + ) + + self.msc4140_enabled = bool( + self.max_delayed_events_per_user and self.max_event_delay_duration + ) def has_tls_listener(self) -> bool: return any(listener.is_tls() for listener in self.listeners) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 8ce795052a..55891f9b15 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -20,7 +20,6 @@ # # import collections.abc -import re from typing import ( TYPE_CHECKING, Any, @@ -28,7 +27,6 @@ Callable, Collection, Mapping, - Match, MutableMapping, ) @@ -40,27 +38,47 @@ CANONICALJSON_MIN_INT, MAX_PDU_SIZE, EventTypes, - EventUnsignedContentFields, - RelationTypes, ) from synapse.api.errors import Codes, SynapseError from synapse.logging.opentracing import SynapseTags, set_tag, trace -from synapse.synapse_rust.events import Unsigned, redact_event -from synapse.types import JsonDict, Requester +from synapse.synapse_rust.events import ( + EventFormat, + SerializeEventConfig, + Unsigned, + format_event_for_client_v1, + format_event_for_client_v2, + format_event_for_client_v2_without_room_id, + format_event_raw, + redact_event, + serialize_events, +) +from synapse.synapse_rust.types import Requester +from synapse.types import JsonDict from . import EventBase, StrippedStateEvent +# These are imported only to re-export them (callers import them from this +# module); listing them in __all__ stops the unused-import lint flagging them +# and re-exports them for `import *`. +# +# The `format_event_*` functions are a backwards compatibility hack: they have +# never been part of the module API and modules shouldn't be pulling them in, +# but some in the wild import them from here anyway. They may be removed in +# the future; nothing in Synapse itself should use them. +__all__ = [ + "EventFormat", + "SerializeEventConfig", + "format_event_for_client_v1", + "format_event_for_client_v2", + "format_event_for_client_v2_without_room_id", + "format_event_raw", +] + if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations from synapse.server import HomeServer -# Split strings on "." but not "\." (or "\\\."). -SPLIT_FIELD_REGEX = re.compile(r"\\*\.") -# Find escaped characters, e.g. those with a \ in front of them. -ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)") - - # Module API callback that allows adding fields to the unsigned section of # events that are sent to clients. ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK = Callable[ @@ -90,177 +108,6 @@ def clone_event(event: EventBase) -> EventBase: return event.deep_copy() -def _copy_field(src: JsonDict, dst: JsonDict, field: list[str]) -> None: - """Copy the field in 'src' to 'dst'. - - For example, if src={"foo":{"bar":5}} and dst={}, and field=["foo","bar"] - then dst={"foo":{"bar":5}}. - - Args: - src: The dict to read from. - dst: The dict to modify. - field: List of keys to drill down to in 'src'. - """ - if len(field) == 0: # this should be impossible - return - if len(field) == 1: # common case e.g. 'origin_server_ts' - if field[0] in src: - dst[field[0]] = src[field[0]] - return - - # Else is a nested field e.g. 'content.body' - # Pop the last field as that's the key to move across and we need the - # parent dict in order to access the data. Drill down to the right dict. - key_to_move = field.pop(-1) - sub_dict = src - for sub_field in field: # e.g. sub_field => "content" - if sub_field in sub_dict and isinstance( - sub_dict[sub_field], collections.abc.Mapping - ): - sub_dict = sub_dict[sub_field] - else: - return - - if key_to_move not in sub_dict: - return - - # Insert the key into the output dictionary, creating nested objects - # as required. We couldn't do this any earlier or else we'd need to delete - # the empty objects if the key didn't exist. - sub_out_dict = dst - for sub_field in field: - sub_out_dict = sub_out_dict.setdefault(sub_field, {}) - sub_out_dict[key_to_move] = sub_dict[key_to_move] - - -def _escape_slash(m: Match[str]) -> str: - """ - Replacement function; replace a backslash-backslash or backslash-dot with the - second character. Leaves any other string alone. - """ - if m.group(1) in ("\\", "."): - return m.group(1) - return m.group(0) - - -def _split_field(field: str) -> list[str]: - """ - Splits strings on unescaped dots and removes escaping. - - Args: - field: A string representing a path to a field. - - Returns: - A list of nested fields to traverse. - """ - - # Convert the field and remove escaping: - # - # 1. "content.body.thing\.with\.dots" - # 2. ["content", "body", "thing\.with\.dots"] - # 3. ["content", "body", "thing.with.dots"] - - # Find all dots (and their preceding backslashes). If the dot is unescaped - # then emit a new field part. - result = [] - prev_start = 0 - for match in SPLIT_FIELD_REGEX.finditer(field): - # If the match is an *even* number of characters than the dot was escaped. - if len(match.group()) % 2 == 0: - continue - - # Add a new part (up to the dot, exclusive) after escaping. - result.append( - ESCAPE_SEQUENCE_PATTERN.sub( - _escape_slash, field[prev_start : match.end() - 1] - ) - ) - prev_start = match.end() - - # Add any part of the field after the last unescaped dot. (Note that if the - # character is a dot this correctly adds a blank string.) - result.append(re.sub(r"\\(.)", _escape_slash, field[prev_start:])) - - return result - - -def only_fields(dictionary: JsonDict, fields: list[str]) -> JsonDict: - """Return a new dict with only the fields in 'dictionary' which are present - in 'fields'. - - If there are no event fields specified then all fields are included. - The entries may include '.' characters to indicate sub-fields. - So ['content.body'] will include the 'body' field of the 'content' object. - A literal '.' or '\' character in a field name may be escaped using a '\'. - - Args: - dictionary: The dictionary to read from. - fields: A list of fields to copy over. Only shallow refs are - taken. - Returns: - A new dictionary with only the given fields. If fields was empty, - the same dictionary is returned. - """ - if len(fields) == 0: - return dictionary - - # for each field, convert it: - # ["content.body.thing\.with\.dots"] => [["content", "body", "thing\.with\.dots"]] - split_fields = [_split_field(f) for f in fields] - - output: JsonDict = {} - for field_array in split_fields: - _copy_field(dictionary, output, field_array) - return output - - -def format_event_raw(d: JsonDict) -> JsonDict: - return d - - -def format_event_for_client_v1(d: JsonDict) -> JsonDict: - d = format_event_for_client_v2(d) - - sender = d.get("sender") - if sender is not None: - d["user_id"] = sender - - copy_keys = ( - "age", - "redacted_because", - "replaces_state", - "prev_content", - "invite_room_state", - "knock_room_state", - ) - for key in copy_keys: - if key in d["unsigned"]: - d[key] = d["unsigned"][key] - - return d - - -def format_event_for_client_v2(d: JsonDict) -> JsonDict: - drop_keys = ( - "auth_events", - "prev_events", - "hashes", - "signatures", - "depth", - "origin", - "prev_state", - ) - for key in drop_keys: - d.pop(key, None) - return d - - -def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: - d = format_event_for_client_v2(d) - d.pop("room_id", None) - return d - - @attr.s(slots=True, frozen=True, auto_attribs=True) class FilteredEvent: """An event annotated with per-user data for client serialization. @@ -305,180 +152,6 @@ def admin_override(cls, event: "EventBase") -> "FilteredEvent": return cls(event=event, membership=None) -@attr.s(slots=True, frozen=True, auto_attribs=True) -class SerializeEventConfig: - as_client_event: bool = True - # Function to convert from federation format to client format - event_format: Callable[[JsonDict], JsonDict] = format_event_for_client_v1 - # The entity that requested the event. This is used to determine whether to include - # the transaction_id and delay_id in the unsigned section of the event. - requester: Requester | None = None - # List of event fields to include. If empty, all fields will be returned. - only_event_fields: list[str] | None = attr.ib(default=None) - # Some events can have stripped room state stored in the `unsigned` field. - # This is required for invite and knock functionality. If this option is - # False, that state will be removed from the event before it is returned. - # Otherwise, it will be kept. - include_stripped_room_state: bool = False - # When True, sets unsigned fields to help clients identify events which - # only server admins can see through other configuration. For example, - # whether an event was soft failed by the server. - include_admin_metadata: bool = False - # Whether MSC4354 (sticky events) is enabled. When True, the sticky TTL - # will be computed and included in the unsigned section of sticky events. - msc4354_enabled: bool = False - - @only_event_fields.validator - def _validate_only_event_fields( - self, attribute: attr.Attribute, value: Any - ) -> None: - if value is None: - return - - if not isinstance(value, list) or not all(isinstance(f, str) for f in value): - raise TypeError("only_event_fields must be a list of strings") - - -_DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig() - - -def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfig: - # Set the options which are only available to server admins, - # and copy the rest. - return attr.evolve(existing, include_admin_metadata=True) - - -def _serialize_event( - e: JsonDict | EventBase, - time_now_ms: int, - *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, - membership: str | None = None, -) -> JsonDict: - """Serialize event for clients - - Args: - e - time_now_ms - config: Event serialization config - membership: The requesting user's membership at the time of the event, - to be injected into unsigned.membership (MSC4115). - - Returns: - The serialized event dictionary. - """ - - # FIXME(erikj): To handle the case of presence events and the like - if not isinstance(e, EventBase): - return e - - time_now_ms = int(time_now_ms) - - # Should this strip out None's? - d = dict(e.get_dict().items()) - - d["event_id"] = e.event_id - - if "age_ts" in d["unsigned"]: - d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - del d["unsigned"]["age_ts"] - - # If we have applicable fields saved in the internal_metadata, include them in the - # unsigned section of the event if the event was sent by the same session (or when - # appropriate, just the same sender) as the one requesting the event. - if config.requester is not None and config.requester.user.to_string() == e.sender: - txn_id: str | None = getattr(e.internal_metadata, "txn_id", None) - if txn_id is not None: - # Some events do not have the device ID stored in the internal metadata, - # this includes old events as well as those created by appservice, guests, - # or with tokens minted with the admin API. For those events, fallback - # to using the access token instead. - event_device_id: str | None = getattr( - e.internal_metadata, "device_id", None - ) - if event_device_id is not None: - if event_device_id == config.requester.device_id: - d["unsigned"]["transaction_id"] = txn_id - - else: - # Fallback behaviour: only include the transaction ID if the event - # was sent from the same access token. - # - # For regular users, the access token ID can be used to determine this. - # This includes access tokens minted with the admin API. - # - # For guests and appservice users, we can't check the access token ID - # so assume it is the same session. - event_token_id: int | None = getattr( - e.internal_metadata, "token_id", None - ) - if ( - ( - event_token_id is not None - and config.requester.access_token_id is not None - and event_token_id == config.requester.access_token_id - ) - or config.requester.is_guest - or config.requester.app_service_id - ): - d["unsigned"]["transaction_id"] = txn_id - - delay_id: str | None = getattr(e.internal_metadata, "delay_id", None) - if delay_id is not None: - d["unsigned"]["org.matrix.msc4140.delay_id"] = delay_id - - # invite_room_state and knock_room_state are a list of stripped room state events - # that are meant to provide metadata about a room to an invitee/knocker. They are - # intended to only be included in specific circumstances, such as down sync, and - # should not be included in any other case. - if not config.include_stripped_room_state: - d["unsigned"].pop("invite_room_state", None) - d["unsigned"].pop("knock_room_state", None) - - if config.as_client_event: - d = config.event_format(d) - - # Ensure the room_id field is set for create events in MSC4291 rooms - if e.type == EventTypes.Create and e.room_version.msc4291_room_ids_as_hashes: - d["room_id"] = e.room_id - - # If the event is a redaction, the field with the redacted event ID appears - # in a different location depending on the room version. e.redacts handles - # fetching from the proper location; copy it to the other location for forwards- - # and backwards-compatibility with clients. - if e.type == EventTypes.Redaction and e.redacts is not None: - if e.room_version.updated_redaction_rules: - d["redacts"] = e.redacts - else: - d["content"] = dict(d["content"]) - d["content"]["redacts"] = e.redacts - - if config.include_admin_metadata: - if e.internal_metadata.is_soft_failed(): - d["unsigned"]["io.element.synapse.soft_failed"] = True - if e.internal_metadata.policy_server_spammy: - d["unsigned"]["io.element.synapse.policy_server_spammy"] = True - - if config.msc4354_enabled: - sticky_duration = e.sticky_duration() - if sticky_duration: - expires_at = ( - # min() ensures that the origin server can't lie about the time and - # send the event 'in the future', as that would allow them to exceed - # the 1 hour limit on stickiness duration. - min(e.origin_server_ts, time_now_ms) + sticky_duration.as_millis() - ) - if expires_at > time_now_ms: - d["unsigned"][EventUnsignedContentFields.STICKY_TTL] = ( - expires_at - time_now_ms - ) - - if membership is not None: - d["unsigned"][EventUnsignedContentFields.MEMBERSHIP] = membership - - return d - - class EventClientSerializer: """Serializes events that are to be sent to clients. @@ -495,12 +168,65 @@ def __init__(self, hs: "HomeServer") -> None: ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] + async def create_config( + self, + *, + as_client_event: bool = True, + event_format: EventFormat = EventFormat.ClientV1, + requester: Requester | None = None, + event_field_allowlist: list[str] | None = None, + include_stripped_room_state: bool = False, + include_admin_metadata: bool | None = None, + ) -> SerializeEventConfig: + """ + Create a new SerializeEventConfig for the given parameters. + + Helper method that sets the `include_admin_metadata` field based on + whether the requester is a server admin if it is not explicitly + provided. Also sets the `msc4354_enabled` field based on the homeserver + config. + + Args: + as_client_event: Whether to serialize the events as client events. + event_format: The format to serialize events in. requester: The user + requesting the events, if any. Used to determine + whether to include admin-only metadata in the serialized events. + event_field_allowlist: A list of event fields to include in the + serialized events. + include_stripped_room_state: Whether to include stripped room state + in the serialized events. + include_admin_metadata: Whether to include admin-only metadata in + the serialized events. If None, this will be determined based on + whether the requester is a server admin. + Returns: + A SerializeEventConfig instance. + """ + + # If include_admin_metadata is None, determine whether to include + # admin-only metadata based on the requester. + if include_admin_metadata is None: + # Check if the requester is a server admin. + if requester is not None and await self._auth.is_server_admin(requester): + include_admin_metadata = True + else: + include_admin_metadata = False + + return SerializeEventConfig( + as_client_event=as_client_event, + event_format=event_format, + requester=requester, + event_field_allowlist=event_field_allowlist, + include_stripped_room_state=include_stripped_room_state, + include_admin_metadata=include_admin_metadata, + msc4354_enabled=self._config.experimental.msc4354_enabled, + ) + async def serialize_event( self, event: JsonDict | FilteredEvent, time_now: int, *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + config: SerializeEventConfig | None = None, bundle_aggregations: dict[str, "BundledAggregations"] | None = None, redaction_map: Mapping[str, "EventBase"] | None = None, ) -> JsonDict: @@ -518,158 +244,111 @@ async def serialize_event( Returns: The serialized event """ - # To handle the case of presence events and the like + # FIXME: Ideally we would only call `serialize_event` with + # `FilteredEvent`s. Currently though some of the old `/events` code paths + # pass through presence events and the like. if not isinstance(event, FilteredEvent): return event - # Force-enable server admin metadata because the only time an event with - # relevant metadata will be when the admin requested it via their admin - # client config account data. Also, it's "just" some `unsigned` fields, so - # shouldn't cause much in terms of problems to downstream consumers. - if config.requester is not None and await self._auth.is_server_admin( - config.requester - ): - config = make_config_for_admin(config) - - if self._config.experimental.msc4354_enabled: - config = attr.evolve(config, msc4354_enabled=True) + if config is None: + # Generate default config if none was provided. + config = await self.create_config() - serialized_event = _serialize_event( - event.event, time_now, config=config, membership=event.membership + # Perform all the async DB/IO work up front, then run the synchronous + # serialization core. + redaction_map, unsigned_additions = await self._prepare_serialization( + [event], bundle_aggregations, redaction_map ) - # If the event was redacted, fetch the redaction event from the database - # and include it in the serialized event's unsigned section. - redacted_by: str | None = event.event.internal_metadata.redacted_by - if redacted_by is not None: - serialized_event.setdefault("unsigned", {})["redacted_by"] = redacted_by - if redaction_map is not None: - redaction_event: EventBase | None = redaction_map.get(redacted_by) - else: - redaction_event = await self._store.get_event( - redacted_by, - allow_none=True, - ) - if redaction_event is not None: - serialized_redaction = _serialize_event( - redaction_event, time_now, config=config - ) - serialized_event.setdefault("unsigned", {})["redacted_because"] = ( - serialized_redaction - ) - # format_event_for_client_v1 copies redacted_because to the - # top level, but since we add it after that runs, do it here. - if ( - config.as_client_event - and config.event_format is format_event_for_client_v1 - ): - serialized_event["redacted_because"] = serialized_redaction - - new_unsigned = {} - for callback in self._add_extra_fields_to_unsigned_client_event_callbacks: - u = await callback(event.event) - new_unsigned.update(u) - - if new_unsigned: - # We do the `update` this way round so that modules can't clobber - # existing fields. - new_unsigned.update(serialized_event["unsigned"]) - serialized_event["unsigned"] = new_unsigned - - # Only include fields that the client has requested. - # - # Note: we always return bundled aggregations, though it is unclear why. - only_event_fields = config.only_event_fields - if only_event_fields: - serialized_event = only_fields(serialized_event, only_event_fields) - - # Check if there are any bundled aggregations to include with the event. - if bundle_aggregations: - if event.event.event_id in bundle_aggregations: - await self._inject_bundled_aggregations( - event.event, - time_now, - config, - bundle_aggregations, - serialized_event, - ) - - return serialized_event - - async def _inject_bundled_aggregations( + return serialize_events( + [(event.event, event.membership)], + time_now, + config, + bundle_aggregations=bundle_aggregations, + redaction_map=redaction_map, + unsigned_additions=unsigned_additions, + )[0] + + async def _prepare_serialization( self, - event: EventBase, - time_now: int, - config: SerializeEventConfig, - bundled_aggregations: dict[str, "BundledAggregations"], - serialized_event: JsonDict, - ) -> None: - """Potentially injects bundled aggregations into the unsigned portion of the serialized event. + events: Collection[FilteredEvent], + bundle_aggregations: dict[str, "BundledAggregations"] | None, + redaction_map: Mapping[str, "EventBase"] | None = None, + ) -> tuple[dict[str, "EventBase"], dict[str, JsonDict]]: + """Perform all the async DB/IO work needed to serialize `events`. + + Does two things: + 1. Fetches any redaction events needed to serialize `events` (and any + bundled events) and returns a map from redaction event_id to event. + 2. Runs the module callbacks for each event to build up the additional + `unsigned` fields they contribute. Args: - event: The event being serialized. - time_now: The current time in milliseconds - config: Event serialization config - bundled_aggregations: Bundled aggregations to be injected. - A map from event_id to aggregation data. Must contain at least an - entry for `event`. + events: The events that will be serialized. + bundle_aggregations: A map from event_id to the aggregations to be + bundled into the event. Used to discover the sub-events (edits + and thread latest events) that will also be serialized. + redaction_map: An optional caller-supplied map from redaction + event_id to the redaction event. Any redactions already present + here are not re-fetched, and these entries take precedence over + anything we fetch ourselves. - While serializing the bundled aggregations this map may be searched - again for additional events in a recursive manner. - serialized_event: The serialized event which may be modified. + Returns: + A tuple of: + - a map from redaction event_id to the redaction event, + - a map from event_id to the extra `unsigned` fields contributed + by the registered module callbacks. """ - # We have already checked that aggregations exist for this event. - event_aggregations = bundled_aggregations[event.event_id] - - # The JSON dictionary to be added under the unsigned property of the event - # being serialized. - serialized_aggregations = {} + # First we collect all events that get included in the serialization of + # `events`, including the events themselves and any bundled events (edits + # and thread latest events, which are themselves serialized). + collected = {e.event.event_id: e.event for e in events} + if bundle_aggregations is not None: + for aggregation in bundle_aggregations.values(): + if aggregation.replace: + collected[aggregation.replace.event_id] = aggregation.replace + if aggregation.thread: + latest_event = aggregation.thread.latest_event + collected[latest_event.event_id] = latest_event + + # Next, check the redaction status of all events, and fetch the + # redactions if needed. + redaction_map = redaction_map or {} + + redaction_ids_to_fetch = { + redacted_by + for collected_event in collected.values() + if (redacted_by := collected_event.internal_metadata.redacted_by) + is not None + and redacted_by not in redaction_map + } + + if redaction_ids_to_fetch: + fetched_redaction_map = await self._store.get_events(redaction_ids_to_fetch) + else: + fetched_redaction_map = {} - if event_aggregations.references: - serialized_aggregations[RelationTypes.REFERENCE] = ( - event_aggregations.references - ) + # Ensure the returned redaction map includes any caller-supplied + # redactions + fetched_redaction_map.update(redaction_map) - if event_aggregations.replace: - # Include information about it in the relations dict. - # - # Matrix spec v1.5 (https://spec.matrix.org/v1.5/client-server-api/#server-side-aggregation-of-mreplace-relationships) - # said that we should only include the `event_id`, `origin_server_ts` and - # `sender` of the edit; however MSC3925 proposes extending it to the whole - # of the edit, which is what we do here. - serialized_aggregations[RelationTypes.REPLACE] = await self.serialize_event( - FilteredEvent(event=event_aggregations.replace, membership=None), - time_now, - config=config, - ) + # Run the module callbacks for each event (once per event_id, since + # `collected` is already de-duplicated) to build up the additional + # `unsigned` fields they contribute. + unsigned_additions: dict[str, JsonDict] = {} + if self._add_extra_fields_to_unsigned_client_event_callbacks: + for collected_event in collected.values(): + new_unsigned: JsonDict = {} + for ( + callback + ) in self._add_extra_fields_to_unsigned_client_event_callbacks: + new_unsigned.update(await callback(collected_event)) - # Include any threaded replies to this event. - if event_aggregations.thread: - thread = event_aggregations.thread - - serialized_latest_event = await self.serialize_event( - FilteredEvent(event=thread.latest_event, membership=None), - time_now, - config=config, - bundle_aggregations=bundled_aggregations, - ) + if new_unsigned: + unsigned_additions[collected_event.event_id] = new_unsigned - thread_summary = { - "latest_event": serialized_latest_event, - "count": thread.count, - "current_user_participated": thread.current_user_participated, - } - serialized_aggregations[RelationTypes.THREAD] = thread_summary - - # Include the bundled aggregations in the event. - if serialized_aggregations: - # There is likely already an "unsigned" field, but a filter might - # have stripped it off (via the event_fields option). The server is - # allowed to return additional fields, so add it back. - serialized_event.setdefault("unsigned", {}).setdefault( - "m.relations", {} - ).update(serialized_aggregations) + return fetched_redaction_map, unsigned_additions @trace async def serialize_events( @@ -677,7 +356,7 @@ async def serialize_events( events: Collection[JsonDict | FilteredEvent], time_now: int, *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + config: SerializeEventConfig | None = None, bundle_aggregations: dict[str, "BundledAggregations"] | None = None, ) -> list[JsonDict]: """Serializes multiple events. @@ -698,26 +377,37 @@ async def serialize_events( str(len(events)), ) - # Batch-fetch all redaction events in one go rather than one per event. - redaction_ids: set[str] = set() - for e in events: - base = e.event if isinstance(e, FilteredEvent) else e - if isinstance(base, EventBase): - redacted_by = base.internal_metadata.redacted_by - if redacted_by is not None: - redaction_ids.add(redacted_by) - redaction_map = ( - await self._store.get_events(redaction_ids) if redaction_ids else {} + if config is None: + # Generate default config if none was provided. + config = await self.create_config() + + filtered_events = [e for e in events if isinstance(e, FilteredEvent)] + + # Perform all the async DB/IO work up front, then run the synchronous + # serialization core for the whole batch in one go. + redaction_map, unsigned_additions = await self._prepare_serialization( + filtered_events, bundle_aggregations ) - return [ - await self.serialize_event( - event, + serialized = iter( + serialize_events( + [(e.event, e.membership) for e in filtered_events], time_now, - config=config, + config, bundle_aggregations=bundle_aggregations, redaction_map=redaction_map, + unsigned_additions=unsigned_additions, ) + ) + + # Stitch the serialized events back in, passing through anything that + # wasn't a FilteredEvent (e.g. presence events) unchanged. + # + # FIXME: Ideally we would only call `serialize_events` with + # `FilteredEvent`s. Currently though some of the old `/events` code paths + # pass through presence events and the like. + return [ + event if not isinstance(event, FilteredEvent) else next(serialized) for event in events ] diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1fe58fae58..69c4187a2f 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -265,6 +265,7 @@ async def on_timestamp_to_event_request( body including `event_id`. """ async with self._server_linearizer.queue((origin, room_id)): + await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -1132,7 +1133,7 @@ async def on_event_auth( await self.check_server_matches_acl(origin_host, room_id) time_now = self._clock.time_msec() - auth_pdus = await self.handler.on_event_auth(event_id) + auth_pdus = await self.handler.on_event_auth(event_id, room_id) res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} return 200, res diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index ada799d96a..d9bce78d4d 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -363,7 +363,9 @@ async def start_redact_events( requester: JsonMapping, use_admin: bool, reason: str | None, - limit: int | None, + before_ts: int | None = None, + after_ts: int | None = None, + limit: int | None = None, ) -> str: """ Start a task redacting the events of the given user in the given rooms @@ -374,6 +376,8 @@ async def start_redact_events( requester: the user requesting the events use_admin: whether to use the admin account to issue the redactions reason: reason for requesting the redaction, ie spam, etc + before_ts: only redact events that happened before this time + after_ts: only redact events that happened after this time limit: limit on the number of events in each room to redact Returns: @@ -402,6 +406,8 @@ async def start_redact_events( "user_id": user_id, "use_admin": use_admin, "reason": reason, + "before_ts": before_ts, + "after_ts": after_ts, "limit": limit, }, ) @@ -417,8 +423,8 @@ async def _redact_all_events( self, task: ScheduledTask ) -> tuple[TaskStatus, Mapping[str, Any] | None, str | None]: """ - Task to redact all of a users events in the given rooms, tracking which, if any, events - whose redaction failed + Task to redact all of a users events in the given rooms in the given time period, + tracking which, if any, events whose redaction failed """ assert task.params is not None @@ -446,6 +452,8 @@ async def _redact_all_events( authenticated_entity=admin.user.to_string(), ) + before_ts = task.params.get("before_ts") + after_ts = task.params.get("after_ts") reason = task.params.get("reason") limit = task.params.get("limit") assert limit is not None @@ -460,6 +468,8 @@ async def _redact_all_events( room, limit, ["m.room.member", "m.room.message", "m.room.encrypted"], + before_ts, + after_ts, ) if not event_ids: # nothing to redact in this room diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 36b2f63e41..68b8aa71f1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -313,7 +313,10 @@ def notify_interested_services_ephemeral( StreamKeyType.PRESENCE, StreamKeyType.TO_DEVICE, ) - and service.supports_ephemeral + # Honour both the stable `receive_ephemeral` registration flag and the + # legacy `de.sorunome.msc2409.push_ephemeral` one, matching the + # transaction body built in `ApplicationServiceApi.push_bulk`. + and (service.supports_ephemeral or service.supports_unstable_ephemeral) ) or ( stream_key == StreamKeyType.DEVICE_LIST diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e2c41fc168..eab599b2c9 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -277,9 +277,7 @@ def __init__(self, hs: "HomeServer"): # response. self._extra_attributes: dict[str, SsoLoginExtraAttributes] = {} - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def validate_user_via_ui_auth( self, @@ -332,7 +330,8 @@ async def validate_user_via_ui_auth( """ if self._auth_delegation_enabled: raise SynapseError( - HTTPStatus.INTERNAL_SERVER_ERROR, "UIA shouldn't be used with MSC3861" + HTTPStatus.INTERNAL_SERVER_ERROR, + "UIA shouldn't be used when auth is delegated", ) if not requester.access_token_id: diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 538bdaaaf8..9ec00d55ad 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -349,6 +349,9 @@ async def activate_account(self, user_id: str) -> None: # Ensure the user is not marked as erased. await self.store.mark_user_not_erased(user_id) + # The profile row is deleted on erasure, so recreate it if missing. + await self.store.create_profile(user) + # Mark the user as active. await self.store.set_user_deactivated_status(user_id, False) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index f016d95e31..13d6a54de2 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -13,12 +13,13 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes, StickyEvent, StickyEventField -from synapse.api.errors import ShadowBanError, SynapseError +from synapse.api.errors import Codes, ShadowBanError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.http.site import SynapseRequest @@ -330,7 +331,7 @@ async def add( state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, ) -> str: """ @@ -344,20 +345,37 @@ async def add( origin_server_ts: The custom timestamp to send the event with. If None, the timestamp will be the actual time when the event is sent. content: The content of the event to be sent. - delay: How long (in milliseconds) to wait before automatically sending the event. + delay: How long to wait before automatically sending the event. sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). The event will be attempted to be reliably delivered to clients and remote servers during its sticky period. Returns: The ID of the added delayed event. Raises: - SynapseError: if the delayed event fails validation checks. + SynapseError: if the delayed event fails validation checks, or + if the requested delay is longer than allowed, or + if sending delayed events has been disallowed entirely. """ # Use standard request limiter for scheduling new delayed events. # TODO: Instead apply ratelimiting based on the scheduled send time. # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) + if not self._config.server.msc4140_enabled: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "Sending delayed events has been disallowed", + Codes.FORBIDDEN, + ) + if delay > self._config.server.max_event_delay_duration: + requested_delay = delay.as_millis() + max_delay = self._config.server.max_event_delay_duration.as_millis() + raise SynapseError( + HTTPStatus.FORBIDDEN, + f"The requested delay ({requested_delay}ms) exceeds the allowed maximum ({max_delay}ms)", + Codes.FORBIDDEN, + ) + self._event_creation_handler.validator.validate_builder( self._event_creation_handler.event_builder_factory.for_room_version( await self._store.get_room_version(room_id), @@ -384,6 +402,7 @@ async def add( content=content, delay=delay, sticky_duration_ms=sticky_duration_ms, + limit=self._config.server.max_delayed_events_per_user, ) if self._repl_client is not None: diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 3be65d91f3..6a64c1ccf7 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -23,6 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any +import attr from canonicaljson import encode_canonical_json from synapse.api.constants import ( @@ -53,6 +54,13 @@ logger = logging.getLogger(__name__) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class DehydratedEvents: + events: list[JsonDict] + stream_id: str + limited: bool + + class DeviceMessageHandler: def __init__(self, hs: "HomeServer"): """ @@ -113,6 +121,7 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None: origin, sender_user_id, ) + return message_type = content["type"] message_id = content["message_id"] for user_id, by_device in content["messages"].items(): @@ -350,7 +359,7 @@ async def get_events_for_dehydrated_device( device_id: str, since_token: str | None, limit: int, - ) -> JsonDict: + ) -> DehydratedEvents: """Fetches up to `limit` events sent to `device_id` starting from `since_token` and returns the new since token. If there are no more messages, returns an empty array. @@ -361,8 +370,9 @@ async def get_events_for_dehydrated_device( since_token: stream id to start from when fetching messages limit: the number of messages to fetch Returns: - A dict containing the to-device messages, as well as a token that the client - can provide in the next call to fetch the next batch of messages + A DehydratedEvents containing the to-device `events` and `stream_id` token that the + client can provide in the next call to fetch the next batch of messages. If there are + more messages which will arrive in the next batch, `limited` is True, otherwise False. """ user_id = requester.user.to_string() @@ -426,10 +436,11 @@ async def get_events_for_dehydrated_device( user_id, ) - return { - "events": messages, - "next_batch": f"d{stream_id}", - } + return DehydratedEvents( + events=messages, + stream_id=f"d{stream_id}", + limited=(stream_id != to_token), + ) def split_device_messages_into_edus( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 2518716bc7..56ded1634e 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -25,7 +25,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.handlers.presence import format_user_presence_state from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig @@ -129,7 +129,7 @@ async def get_stream( chunks = await self._event_serializer.serialize_events( events, time_now, - config=SerializeEventConfig( + config=await self._event_serializer.create_config( as_client_event=as_client_event, requester=requester ), ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9b96a193b4..cdf17bab35 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -575,8 +575,8 @@ async def send_invite(self, target_host: str, event: EventBase) -> EventBase: return pdu - async def on_event_auth(self, event_id: str) -> list[EventBase]: - event = await self.store.get_event(event_id) + async def on_event_auth(self, event_id: str, room_id: str) -> list[EventBase]: + event = await self.store.get_event(event_id, check_room_id=room_id) auth = await self.store.get_auth_chain( event.room_id, list(event.auth_event_ids()), include_given=True ) @@ -791,10 +791,55 @@ async def do_invite_join( if not predecessor or not isinstance(predecessor.get("room_id"), str): return event.event_id, max_stream_id old_room_id = predecessor["room_id"] - logger.debug( - "Found predecessor for %s during remote join: %s", room_id, old_room_id + + # We can't take the new room's word for it. + # Check to see that the predecessor room consents to the + # room upgrade. + if not await self._event_auth_handler.is_host_in_room( + room_id=old_room_id, host=self.hs.hostname + ): + logger.info( + "Ignoring unverified predecessor for %s during remote join: %s (not in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + tombstone = await self._state_storage_controller.get_current_state_event( + old_room_id, + event_type=EventTypes.Tombstone, + state_key="", + ) + + if tombstone is None: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (no tombstone in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + intended_successor_room = tombstone.content.get( + EventContentFields.TOMBSTONE_SUCCESSOR_ROOM, None ) + if not isinstance(intended_successor_room, str): + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (tombstone is invalid)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + if intended_successor_room != room_id: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: predecessor defined as %s (the old room ID) but the old room's tombstone points to %r which doesn't match", + room_id, + old_room_id, + intended_successor_room, + ) + return event.event_id, max_stream_id + # We retrieve the room member handler here as to not cause a cyclic dependency member_handler = self.hs.get_room_member_handler() await member_handler.transfer_room_state_on_room_upgrade( diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 591a0aefd3..56f4d86d41 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,7 +30,7 @@ Membership, ) from synapse.api.errors import SynapseError -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.events.validator import EventValidator from synapse.handlers.presence import format_user_presence_state from synapse.handlers.receipts import ReceiptEventSource @@ -169,7 +169,9 @@ async def _snapshot_all_rooms( public_room_ids = await self.store.get_public_room_ids() - serializer_options = SerializeEventConfig(as_client_event=as_client_event) + serializer_options = await self._event_serializer.create_config( + as_client_event=as_client_event + ) async def handle_room(event: RoomsForUser) -> None: d: JsonDict = { @@ -395,7 +397,9 @@ async def _room_initial_sync_parted( end_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, stream_token) time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) return { "membership": membership, @@ -436,7 +440,9 @@ async def _room_initial_sync_joined( # TODO: These concurrently time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) # Don't bundle aggregations as this is a deprecated API. state = await self._event_serializer.serialize_events( [FilteredEvent.state(e) for e in current_state.values()], diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b8f10e0192..33fa5e8c92 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -64,7 +64,6 @@ ) from synapse.events.utils import ( FilteredEvent, - SerializeEventConfig, maybe_upsert_event_field, ) from synapse.events.validator import EventValidator @@ -269,7 +268,7 @@ async def get_state_events( events = await self._event_serializer.serialize_events( [FilteredEvent.state(e) for e in room_state.values()], self.clock.time_msec(), - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return events diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 4c3adca46e..55dd5ffb59 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -122,6 +122,7 @@ ) from synapse.util.async_helpers import Linearizer from synapse.util.duration import Duration +from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -179,19 +180,18 @@ labelnames=[SERVER_NAME_LABEL], ) -# If a user was last active in the last LAST_ACTIVE_GRANULARITY, consider them -# "currently_active" -LAST_ACTIVE_GRANULARITY = 60 * 1000 +# Note: the timers deciding when a user goes idle or offline and how long +# they count as "currently_active" are configurable, via the +# `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options +# in the `presence` config section (see +# `synapse.config.server.DEFAULT_LAST_ACTIVE_GRANULARITY` and friends for the +# defaults). -# How long to wait until a new /events or /sync request before assuming -# the client has gone. -SYNC_ONLINE_TIMEOUT = 30 * 1000 -# Busy status waits longer, but does eventually go offline. +# How long to wait until a device with busy status stops syncing before it +# goes offline. Busy status waits longer than the (configurable) sync online +# timeout, but does eventually go offline. BUSY_ONLINE_TIMEOUT = 60 * 60 * 1000 -# How long to wait before marking the user as idle. Compared against last active -IDLE_TIMER = 5 * 60 * 1000 - # How often we expect remote servers to resend us presence. FEDERATION_TIMEOUT = 30 * 60 * 1000 @@ -206,8 +206,6 @@ # syncing. UPDATE_SYNCING_USERS = Duration(seconds=10) -assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER - class BasePresenceHandler(abc.ABC): """Parts of the PresenceHandler that are shared between workers and presence @@ -225,6 +223,19 @@ def __init__(self, hs: "HomeServer"): self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence + # Rooms which, on their own, should not cause presence to be routed + # between their members. See `exclude_rooms_from_presence` in the config. + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) + + # The (configurable) presence state machine timers. + self._last_active_granularity = ( + hs.config.server.presence_last_active_granularity + ) + self._sync_online_timeout = hs.config.server.presence_sync_online_timeout + self._idle_timer = hs.config.server.presence_idle_timeout + self._federation = None if hs.should_send_federation(): self._federation = hs.get_federation_sender() @@ -431,6 +442,7 @@ async def maybe_send_presence_to_interested_destinations( self.store, self.presence_router, states, + self._rooms_to_exclude_from_presence, ) for destinations, host_states in hosts_to_states: @@ -526,10 +538,34 @@ def __init__(self, hs: "HomeServer"): # syncing but we haven't notified the presence writer of that yet self._user_devices_going_offline: dict[tuple[str, str | None], int] = {} + # How often to relay an unchanged sync-driven presence state to the + # presence writer. The relayed updates are what feed the writer's device + # last_sync_ts/last_active_ts timers, so this must sit comfortably below + # the timers it feeds — the (configurable) sync online timeout and + # last-active granularity — or users would flap offline / lose + # "currently active" between relays. We use 5/6 of the tighter of the + # two, i.e. the historic 25s at the default 30s sync online timeout. + self._sync_presence_relay_interval = ( + min(self._sync_online_timeout, self._last_active_granularity) * 5 // 6 + ) + + # (user_id, device_id) -> (state, last_sent_ms) of the most recent + # sync-driven presence update we proxied to the presence writer. Used + # to suppress the per-sync-request set_state/bump calls, which are + # no-ops on the writer at finer granularity than its timers: while + # the state is unchanged there is no point relaying more than one + # update per relay interval. Entries older than the window are swept by + # `_sweep_last_sent_presence`. + self._last_sent_presence: dict[tuple[str, str | None], tuple[str, int]] = {} + self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS) + if self._track_presence: + self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS) + self.clock.looping_call( + self._sweep_last_sent_presence, Duration(minutes=30) + ) hs.register_async_shutdown_handler( phase="before", @@ -572,6 +608,8 @@ def mark_as_going_offline(self, user_id: str, device_id: str | None) -> None: sending a stopped syncing immediately followed by a started syncing notification to the presence writer """ + if not self._track_presence: + return self._user_devices_going_offline[(user_id, device_id)] = self.clock.time_msec() def send_stop_syncing(self) -> None: @@ -585,6 +623,22 @@ def send_stop_syncing(self) -> None: if now - last_sync_ms > UPDATE_SYNCING_USERS.as_millis(): self._user_devices_going_offline.pop((user_id, device_id), None) self.send_user_sync(user_id, device_id, False, last_sync_ms) + # Once the writer knows the device stopped syncing it may time + # the user out, so if the device comes back we must relay its + # state again rather than suppress it as a repeat. + self._last_sent_presence.pop((user_id, device_id), None) + + def _sweep_last_sent_presence(self) -> None: + """Drop expired presence-throttling entries. + + Entries should be dropped in `send_stop_syncing`, but we add a safety + net here to ensure that the dict deesn't grow unbounded. + """ + now = self.clock.time_msec() + + for key, (_, last_sent_ms) in list(self._last_sent_presence.items()): + if now - last_sent_ms >= self._sync_presence_relay_interval: + self._last_sent_presence.pop(key, None) async def user_syncing( self, @@ -602,7 +656,9 @@ async def user_syncing( return _NullContextManager() # Note that this causes last_active_ts to be incremented which is not - # what the spec wants. + # what the spec wants. (This call is throttled in `set_state`: while + # the state is unchanged, only one update per relay interval is relayed + # to the presence writer.) await self.set_state( UserID.from_string(user_id), device_id, @@ -640,7 +696,12 @@ def _user_syncing() -> Generator[None, None, None]: async def notify_from_replication( self, states: list[UserPresenceState], stream_id: int ) -> None: - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -685,7 +746,11 @@ async def process_replication_rows( self.user_to_current_state[new_state.user_id] = new_state is_mine = self.is_mine_id(new_state.user_id) if not old_state or should_notify( - old_state, new_state, is_mine, self.server_name + old_state, + new_state, + is_mine, + self.server_name, + last_active_granularity=self._last_active_granularity, ): state_to_notify.append(new_state) @@ -735,6 +800,28 @@ async def set_state( if not self._track_presence: return + now = self.clock.time_msec() + if is_sync and not force_notify: + # Sync-driven updates arrive on every /sync request, which is far + # finer-grained than any of the writer's presence timers need: + # while the state is unchanged, relaying one update per relay + # interval is enough to keep them fed. State changes always go + # through immediately. + last_sent = self._last_sent_presence.get((user_id, device_id)) + if last_sent is not None: + last_presence, last_sent_ms = last_sent + if ( + presence == last_presence + and now - last_sent_ms < self._sync_presence_relay_interval + ): + return + self._last_sent_presence[(user_id, device_id)] = (presence, now) + else: + # An explicit (non-sync) update doesn't refresh the writer's + # last_sync_ts, so it must not count as a recent relay: drop any + # entry so the next sync-driven update goes through. + self._last_sent_presence.pop((user_id, device_id), None) + # Proxy request to instance that writes presence await self._set_state_client( instance_name=self._presence_writer_instance, @@ -755,8 +842,25 @@ async def bump_presence_active_time( if not self._track_presence: return - # Proxy request to instance that writes presence user_id = user.to_string() + + # A bump's only effects on the writer are updating last_active_ts and + # flipping an idle device back online. Going idle takes far longer + # than the relay window, so if we relayed an *online* update within + # the window the user cannot have gone idle since, and this bump is a + # no-op: skip it. Bumps after any other state (or an unknown one) go + # through immediately, as they may un-idle the device. + now = self.clock.time_msec() + last_sent = self._last_sent_presence.get((user_id, device_id)) + if ( + last_sent is not None + and last_sent[0] == PresenceState.ONLINE + and now - last_sent[1] < self._sync_presence_relay_interval + ): + return + self._last_sent_presence[(user_id, device_id)] = (PresenceState.ONLINE, now) + + # Proxy request to instance that writes presence await self._bump_active_client( instance_name=self._presence_writer_instance, user_id=user_id, @@ -791,7 +895,8 @@ def __init__(self, hs: "HomeServer"): if self._track_presence: for state in self.user_to_current_state.values(): # Create a psuedo-device to properly handle time outs. This will - # be overridden by any "real" devices within SYNC_ONLINE_TIMEOUT. + # be overridden by any "real" devices within the sync online + # timeout. pseudo_device_id = None self._user_to_device_to_current_state[state.user_id] = { pseudo_device_id: UserDevicePresenceState( @@ -804,12 +909,14 @@ def __init__(self, hs: "HomeServer"): } self.wheel_timer.insert( - now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER + now=now, + obj=state.user_id, + then=state.last_active_ts + self._idle_timer, ) self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=state.last_user_sync_ts + self._sync_online_timeout, ) if self.is_mine_id(state.user_id): self.wheel_timer.insert( @@ -878,6 +985,14 @@ def __init__(self, hs: "HomeServer"): Duration(minutes=1), ) + if not self._presence_enabled and self.user_to_current_state: + # Presence is disabled but the database still contains non-offline + # presence states, i.e. presence used to be enabled. Nothing writes + # to the presence stream while presence is disabled, so without + # intervention clients would show the stale states forever. Send + # out one final round of updates marking everyone as offline. + self.clock.call_when_running(self._mark_stale_presence_as_offline) + presence_wheel_timer_size_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), hook=lambda: {(self.server_name,): len(self.wheel_timer)}, @@ -935,6 +1050,36 @@ async def _persist_unpersisted_changes(self) -> None: [self.user_to_current_state[user_id] for user_id in unpersisted] ) + @wrap_as_background_process("PresenceHandler._mark_stale_presence_as_offline") + async def _mark_stale_presence_as_offline(self) -> None: + """One-off job, run at startup when presence is disabled, that marks + any non-offline presence states left over from when presence was + enabled as offline, and streams the changes out to clients. + """ + states = [ + state.copy_and_replace( + state=PresenceState.OFFLINE, + status_msg=None, + currently_active=False, + ) + for state in self.user_to_current_state.values() + if state.state != PresenceState.OFFLINE + ] + if not states: + return + + logger.info( + "Presence is disabled: marking %d stale presence states as offline", + len(states), + ) + + self.user_to_current_state.update({state.user_id: state for state in states}) + + # There may be a lot of stale states (e.g. everyone that was online + # when presence was disabled), so persist them in batches. + for batch in batch_iter(states, 500): + await self._persist_and_notify(list(batch)) + async def _update_states( self, new_states: Iterable[UserPresenceState], @@ -996,6 +1141,9 @@ async def _update_states( # When overriding disabled presence, don't kick off all the # wheel timers. persist=not self._track_presence, + idle_timer=self._idle_timer, + sync_online_timeout=self._sync_online_timeout, + last_active_granularity=self._last_active_granularity, ) if force_notify: @@ -1044,6 +1192,7 @@ async def _update_states( self.store, self.presence_router, list(to_federation_ping.values()), + self._rooms_to_exclude_from_presence, ) for destinations, states in hosts_to_states: @@ -1108,6 +1257,9 @@ async def _handle_timeouts(self) -> None: syncing_user_devices=syncing_user_devices, user_to_devices=self._user_to_device_to_current_state, now=now, + idle_timer=self._idle_timer, + sync_online_timeout=self._sync_online_timeout, + last_active_granularity=self._last_active_granularity, ) return await self._update_states(changes) @@ -1314,7 +1466,12 @@ async def _persist_and_notify(self, states: list[UserPresenceState]) -> None: """ stream_id, max_token = await self.store.update_presence(states) - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1461,7 +1618,10 @@ async def is_visible(self, observed_user: UserID, observer_user: UserID) -> bool observed_user.to_string() ) - if observer_room_ids & observed_room_ids: + shared_room_ids = ( + observer_room_ids & observed_room_ids + ) - self._rooms_to_exclude_from_presence + if shared_room_ids: return True return False @@ -1572,6 +1732,12 @@ async def _handle_state_delta(self, room_id: str, deltas: list[StateDelta]) -> N to be handled. """ + # Excluded rooms should not, on their own, share presence between their + # members. This method is entirely per-room presence fan-out, so skip + # excluded rooms wholesale. + if room_id in self._rooms_to_exclude_from_presence: + return + # Sets of newly joined users. Note that if the local server is # joining a remote room for the first time we'll see both the joining # user and all remote users as newly joined. @@ -1695,6 +1861,8 @@ def should_notify( new_state: UserPresenceState, is_mine: bool, our_server_name: str, + *, + last_active_granularity: int, ) -> bool: """Decides if a presence state change should be sent to interested parties.""" user_location = "remote" @@ -1741,7 +1909,7 @@ def should_notify( if ( new_state.last_active_ts - old_state.last_active_ts - > LAST_ACTIVE_GRANULARITY + > last_active_granularity ): # Only notify about last active bumps if we're not currently active if not new_state.currently_active: @@ -1752,7 +1920,7 @@ def should_notify( ).inc() return True - elif new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: + elif new_state.last_active_ts - old_state.last_active_ts > last_active_granularity: # Always notify for a transition where last active gets bumped. notify_reason_counter.labels( locality=user_location, @@ -1827,6 +1995,9 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) async def get_new_events( self, @@ -1941,9 +2112,31 @@ async def get_new_events( **{SERVER_NAME_LABEL: self.server_name}, ).inc() - sharing_users = await self.store.do_users_share_a_room( - user_id, updated_users - ) + # An updated user is interesting if they share a + # (non-excluded) room with the syncing user. We check by + # intersecting the cached per-user room sets rather than via + # `do_users_share_a_room`: its per-pair cache has a + # quadratic working set and is cleared wholesale on every + # membership change, so on busy servers every check missed + # into SQL. + # + # For every presence update we need to run this code for + # every user that is currently syncing. The + # `get_rooms_for_user` will therefore be computed only once + # for each updated user regardless of the number of syncing + # users. + # + # The syncing user's rooms will also be cached as its needed + # during sync processing anyway. + my_rooms = await self.store.get_rooms_for_user(user_id) + if self._rooms_to_exclude_from_presence: + my_rooms = my_rooms - self._rooms_to_exclude_from_presence + rooms_by_user = await self.store.get_rooms_for_users(updated_users) + sharing_users = { + updated_user + for updated_user, rooms in rooms_by_user.items() + if not my_rooms.isdisjoint(rooms) + } interested_and_updated_users = ( sharing_users.union(additional_users_interested_in) @@ -1958,7 +2151,9 @@ async def get_new_events( ).inc() users_interested_in = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) users_interested_in.update(additional_users_interested_in) @@ -1971,7 +2166,9 @@ async def get_new_events( # No from_key has been specified. Return the presence for all users # this user is interested in interested_and_updated_users = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) interested_and_updated_users.update(additional_users_interested_in) @@ -2075,6 +2272,10 @@ def handle_timeouts( syncing_user_devices: AbstractSet[tuple[str, str | None]], user_to_devices: dict[str, dict[str | None, UserDevicePresenceState]], now: int, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> list[UserPresenceState]: """Checks the presence of users that have timed out and updates as appropriate. @@ -2085,6 +2286,11 @@ def handle_timeouts( syncing_user_devices: A set of (user ID, device ID) tuples with active syncs.. user_to_devices: A map of user ID to device ID to UserDevicePresenceState. now: Current time in ms. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: List of UserPresenceState updates @@ -2101,6 +2307,9 @@ def handle_timeouts( syncing_user_devices, user_to_devices.get(user_id, {}), now, + idle_timer=idle_timer, + sync_online_timeout=sync_online_timeout, + last_active_granularity=last_active_granularity, ) if new_state: changes[state.user_id] = new_state @@ -2114,6 +2323,10 @@ def handle_timeout( syncing_device_ids: AbstractSet[tuple[str, str | None]], user_devices: dict[str | None, UserDevicePresenceState], now: int, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> UserPresenceState | None: """Checks the presence of the user to see if any of the timers have elapsed @@ -2123,6 +2336,11 @@ def handle_timeout( syncing_user_devices: A set of (user ID, device ID) tuples with active syncs.. user_devices: A map of device ID to UserDevicePresenceState. now: Current time in ms. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: A UserPresenceState update or None if no update. @@ -2140,7 +2358,7 @@ def handle_timeout( offline_devices = [] for device_id, device_state in user_devices.items(): if device_state.state == PresenceState.ONLINE: - if now - device_state.last_active_ts > IDLE_TIMER: + if now - device_state.last_active_ts > idle_timer: # Currently online, but last activity ages ago so auto # idle device_state.state = PresenceState.UNAVAILABLE @@ -2161,7 +2379,7 @@ def handle_timeout( online_timeout = ( BUSY_ONLINE_TIMEOUT if device_state.state == PresenceState.BUSY - else SYNC_ONLINE_TIMEOUT + else sync_online_timeout ) if now - sync_or_active > online_timeout: # Mark the device as going offline. @@ -2180,7 +2398,7 @@ def handle_timeout( state = state.copy_and_replace(state=new_presence) changed = True - if now - state.last_active_ts > LAST_ACTIVE_GRANULARITY: + if now - state.last_active_ts > last_active_granularity: # So that we send down a notification that we've # stopped updating. changed = True @@ -2209,6 +2427,10 @@ def handle_update( wheel_timer: WheelTimer, now: int, persist: bool, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> tuple[UserPresenceState, bool, bool]: """Given a presence update: 1. Add any appropriate timers. @@ -2223,6 +2445,11 @@ def handle_update( now: Time now in ms persist: True if this state should persist until another update occurs. Skips insertion into wheel timers. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: 3-tuple: `(new_state, persist_and_notify, federation_ping)` where: @@ -2242,17 +2469,17 @@ def handle_update( # Idle timer if not persist: wheel_timer.insert( - now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER + now=now, obj=user_id, then=new_state.last_active_ts + idle_timer ) - active = now - new_state.last_active_ts < LAST_ACTIVE_GRANULARITY + active = now - new_state.last_active_ts < last_active_granularity new_state = new_state.copy_and_replace(currently_active=active) if active and not persist: wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_active_ts + last_active_granularity, ) if new_state.state != PresenceState.OFFLINE: @@ -2261,7 +2488,7 @@ def handle_update( wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_user_sync_ts + sync_online_timeout, ) last_federate = new_state.last_federation_update_ts @@ -2287,7 +2514,13 @@ def handle_update( ) # Check whether the change was something worth notifying about - if should_notify(prev_state, new_state, is_mine, our_server_name): + if should_notify( + prev_state, + new_state, + is_mine, + our_server_name, + last_active_granularity=last_active_granularity, + ): new_state = new_state.copy_and_replace(last_federation_update_ts=now) persist_and_notify = True @@ -2335,7 +2568,10 @@ def _combine_device_states( async def get_interested_parties( - store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState] + store: DataStore, + presence_router: PresenceRouter, + states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> tuple[dict[str, list[UserPresenceState]], dict[str, list[UserPresenceState]]]: """Given a list of states return which entities (rooms, users) are interested in the given states. @@ -2344,6 +2580,8 @@ async def get_interested_parties( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed between their members. Returns: A 2-tuple of `(room_ids_to_states, users_to_states)`, @@ -2354,6 +2592,8 @@ async def get_interested_parties( for state in states: room_ids = await store.get_rooms_for_user(state.user_id) for room_id in room_ids: + if room_id in excluded_rooms: + continue room_ids_to_states.setdefault(room_id, []).append(state) # Always notify self @@ -2374,6 +2614,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> list[tuple[StrCollection, Collection[UserPresenceState]]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -2384,6 +2625,8 @@ async def get_interested_remotes( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed to their remote members. Returns: A map from destinations to presence states to send to that destination. @@ -2397,6 +2640,8 @@ async def get_interested_remotes( room_ids = await store.get_rooms_for_user(state.user_id) hosts: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue room_hosts = await store.get_current_hosts_in_room(room_id) hosts.update(room_hosts) hosts_and_states.append((hosts, [state])) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb..3f3b9e6d8b 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -41,7 +41,11 @@ def __init__(self, hs: "HomeServer"): ) async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + allow_backward: bool = False, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -59,7 +63,7 @@ async def received_client_read_marker( # Get event ordering, this also ensures we know about the event event_ordering = await self.store.get_event_ordering(event_id, room_id) - if existing_read_marker: + if existing_read_marker and not allow_backward: try: old_event_ordering = await self.store.get_event_ordering( existing_read_marker["event_id"], room_id diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index ee4f8d672e..a8db082feb 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -28,16 +28,22 @@ Sequence, ) -import attr - from synapse.api.constants import Direction, EventTypes, RelationTypes from synapse.api.errors import SynapseError from synapse.events import EventBase, relation_from_event -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import trace from synapse.storage.databases.main.relations import ThreadsNextBatch, _RelatedEvent from synapse.streams.config import PaginationConfig + +# `BundledAggregations` and `ThreadAggregation` are implemented in Rust; they +# are re-exported here so existing call sites can keep importing them from +# `synapse.handlers.relations`. +from synapse.synapse_rust.events import ( # noqa: F401 + BundledAggregations, + ThreadAggregation, +) from synapse.types import JsonDict, Requester, UserID from synapse.util.async_helpers import gather_results from synapse.visibility import filter_and_transform_events_for_client @@ -56,32 +62,6 @@ class ThreadsListInclude(str, enum.Enum): participated = "participated" -@attr.s(slots=True, frozen=True, auto_attribs=True) -class _ThreadAggregation: - # The latest event in the thread. - latest_event: EventBase - # The total number of events in the thread. - count: int - # True if the current user has sent an event to the thread. - current_user_participated: bool - - -@attr.s(slots=True, auto_attribs=True) -class BundledAggregations: - """ - The bundled aggregations for an event. - - Some values require additional processing during serialization. - """ - - references: JsonDict | None = None - replace: EventBase | None = None - thread: _ThreadAggregation | None = None - - def __bool__(self) -> bool: - return bool(self.references or self.replace or self.thread) - - class RelationsHandler: def __init__(self, hs: "HomeServer"): self._main_store = hs.get_datastores().main @@ -170,7 +150,9 @@ async def get_relations( ) now = self._clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) return_value: JsonDict = { "chunk": await self._event_serializer.serialize_events( filtered_events, @@ -310,7 +292,7 @@ async def _get_threads_for_events( relations_by_id: dict[str, str], user_id: str, ignored_users: frozenset[str], - ) -> dict[str, _ThreadAggregation]: + ) -> dict[str, ThreadAggregation]: """Get the bundled aggregations for threads for the requested events. Args: @@ -421,7 +403,7 @@ async def _get_threads_for_events( continue latest_thread_event = event.event - results[event_id] = _ThreadAggregation( + results[event_id] = ThreadAggregation( latest_event=latest_thread_event, count=thread_count, # If there's a thread summary it must also exist in the @@ -478,8 +460,12 @@ async def get_bundled_aggregations( # The event should get bundled aggregations. events_by_id[event.event_id] = event - # event ID -> bundled aggregation in non-serialized form. - results: dict[str, BundledAggregations] = {} + # `BundledAggregations` is immutable, so we collect each kind of + # aggregation into its own map keyed by event ID and assemble the + # results once everything has been fetched. + thread_by_id: dict[str, ThreadAggregation] = {} + references_by_id: dict[str, JsonDict] = {} + replace_by_id: dict[str, EventBase] = {} # Fetch any ignored users of the requesting user. ignored_users = await self._main_store.ignored_users(user_id) @@ -495,7 +481,7 @@ async def get_bundled_aggregations( ignored_users, ) for event_id, thread in threads.items(): - results.setdefault(event_id, BundledAggregations()).thread = thread + thread_by_id[event_id] = thread # If the latest event in a thread is not already being fetched, # add it. This ensures that the bundled aggregations for the @@ -516,7 +502,7 @@ async def _fetch_references() -> None: ) for event_id, references in references_by_event_id.items(): if references: - results.setdefault(event_id, BundledAggregations()).references = { + references_by_id[event_id] = { "chunk": [{"event_id": ev.event_id} for ev in references] } @@ -535,7 +521,13 @@ async def _fetch_edits() -> None: ] ) for event_id, edit in edits.items(): - results.setdefault(event_id, BundledAggregations()).replace = edit + # `get_applicable_edits` returns `None` for events with no + # applicable edit. Skip those rather than recording an entry: a + # `None` replace contributes nothing during serialization, so + # the old code's empty `BundledAggregations` for such events was + # inert anyway. + if edit is not None: + replace_by_id[event_id] = edit # Parallelize the calls for annotations, references, and edits since they # are unrelated. @@ -548,7 +540,17 @@ async def _fetch_edits() -> None: ) ) - return results + # Assemble one (immutable) bundled aggregation per event that has any. + return { + event_id: BundledAggregations( + references=references_by_id.get(event_id), + replace=replace_by_id.get(event_id), + thread=thread_by_id.get(event_id), + ) + for event_id in thread_by_id.keys() + | references_by_id.keys() + | replace_by_id.keys() + } async def get_threads( self, diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 30e072d011..eb0492ff59 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -29,7 +29,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.filtering import Filter -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.visibility import filter_and_transform_events_for_client @@ -377,7 +377,9 @@ async def _search( # blocking calls after this. Otherwise, the 'age' will be wrong. time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) for context in contexts.values(): context["events_before"] = await self._event_serializer.serialize_events( diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 10ca3ddea0..afb93e42b4 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -57,6 +57,7 @@ StrCollection, StreamKeyType, StreamToken, + StrictJsonMapping, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.handlers.sliding_sync import ( @@ -953,7 +954,10 @@ async def get_room_sync_data( ) name_event = name_states.get((EventTypes.Name, "")) if name_event is not None: - room_name = name_event.content.get("name") + name_event_content: StrictJsonMapping = name_event.content + unchecked_room_name = name_event_content.get("name") + if isinstance(unchecked_room_name, str): + room_name = unchecked_room_name # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and @@ -1356,18 +1360,28 @@ async def get_room_sync_data( room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: - room_avatar = avatar_event.content.get("url") + room_avatar_content: StrictJsonMapping = avatar_event.content + unchecked_room_avatar = room_avatar_content.get("url") + if isinstance(unchecked_room_avatar, str): + room_avatar = unchecked_room_avatar # Assemble heroes: extract the info from the state we just fetched heroes: list[SlidingSyncResult.RoomResult.StrippedHero] = [] for hero_user_id in hero_user_ids: member_event = hero_membership_state.get((EventTypes.Member, hero_user_id)) if member_event is not None: + member_event_content: StrictJsonMapping = member_event.content + unchecked_display_name = member_event_content.get("displayname") + unchecked_avatar_url = member_event_content.get("avatar_url") heroes.append( SlidingSyncResult.RoomResult.StrippedHero( user_id=hero_user_id, - display_name=member_event.content.get("displayname"), - avatar_url=member_event.content.get("avatar_url"), + display_name=unchecked_display_name + if isinstance(unchecked_display_name, str) + else None, + avatar_url=unchecked_avatar_url + if isinstance(unchecked_avatar_url, str) + else None, ) ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9ecfe0da0f..a05d6c6e59 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1759,9 +1759,19 @@ async def generate_sync_result( await self._generate_sync_entry_for_account_data(sync_result_builder) # Presence data is included if the server has it enabled and not filtered out. - include_presence_data = bool( - self.hs_config.server.presence_enabled - and not sync_config.filter_collection.blocks_all_presence() + presence_enabled = bool(self.hs_config.server.presence_enabled) + if not presence_enabled and since_token is not None: + # Even with presence disabled we send down any presence updates the + # client hasn't yet seen, so that the "mark everyone as offline" + # updates written when presence was disabled reach clients that + # would otherwise show the old presence states forever. The stream + # doesn't advance while presence is disabled, so once clients have + # caught up this check stops any further presence work. + presence_enabled = ( + since_token.presence_key < sync_result_builder.now_token.presence_key + ) + include_presence_data = ( + presence_enabled and not sync_config.filter_collection.blocks_all_presence() ) # Device list updates are sent if a since token is provided. include_device_list_updates = bool(since_token and since_token.device_list_key) diff --git a/synapse/http/client.py b/synapse/http/client.py index 05c5f13a87..78f03ae58a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -87,8 +87,7 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence from synapse.util.async_helpers import timeout_deferred -from synapse.util.clock import Clock -from synapse.util.duration import Duration +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from synapse.util.json import json_decoder if TYPE_CHECKING: @@ -163,11 +162,6 @@ def _is_ip_blocked( return False -# The delay used by the scheduler to schedule tasks "as soon as possible", while -# still allowing other tasks to run between runs. -_EPSILON = Duration(microseconds=1) - - def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. @@ -176,7 +170,7 @@ def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCa def _scheduler(x: Callable[[], object]) -> IDelayedCall: return clock.call_later( - _EPSILON, + CLOCK_SCHEDULE_EPSILON, x, ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 2c235e04f4..a0ae20be16 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ Any, Awaitable, Callable, + Final, Iterable, Iterator, Pattern, @@ -673,8 +674,33 @@ def render(self, request: "SynapseRequest") -> int: # or the response bytes as a return value. return NOT_DONE_YET - def getChild(self, name: str, request: Request) -> resource.Resource: - return self + def getChild(self, path: str, request: Request) -> resource.Resource: + # The child of a catch-all unrecognised request handler + # is itself another unrecognised request handler. + # We can return any UnrecognizedRequestResource that doesn't + # have children. + assert len(_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE.children) == 0 + return _BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE + + +class _LeafUnrecognisedRequestResource(UnrecognizedRequestResource): + """ + UnrecognizedRequestResource, but with the added caveat that it can't have any children. + This makes it safe for it to return itself as a dynamic child. + + Constructed as a singleton; use `_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE` + """ + + def putChild(self, path: bytes, child: IResource) -> None: + raise RuntimeError("_LeafUnrecognisedRequestResource does not accept children") + + +_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE: Final[_LeafUnrecognisedRequestResource] = ( + _LeafUnrecognisedRequestResource() +) +""" +An UnrecognizedRequestResource that is guaranteed not to have children. +""" class RootRedirect(resource.Resource): diff --git a/synapse/http/site.py b/synapse/http/site.py index 9b7fd5c936..4b841a21e2 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -278,7 +278,7 @@ def requestReceived(self, command: bytes, path: bytes, version: bytes) -> None: # See: https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2 if command == b"POST": ctype = self.requestHeaders.getRawHeaders(b"content-type") - if ctype and b"multipart/form-data" in ctype[0]: + if ctype and b"multipart/form-data" in ctype[0].lower(): logger.warning( "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", self.client, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 2aed0608ff..22470172e5 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -355,9 +355,7 @@ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: self._device_handler = hs.get_device_handler() self.custom_template_dir = hs.config.server.custom_template_directory self._callbacks = hs.get_module_api_callbacks() - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled self._event_serializer = hs.get_event_client_serializer() try: diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 87d6e80898..686c2351d2 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -344,7 +344,12 @@ async def send_request( code=e.code, **{SERVER_NAME_LABEL: server_name}, ).inc() - raise e.to_synapse_error() + # This error is coming from another worker, so we trust it to be safe + # to relay to clients directly. + # In fact, we rely relaying verbatim at the very least to tell + # clients when they are rate-limited, + # but most likely other things too. + raise e.unsafe_to_verbatim_synapse_error() except Exception as e: _outgoing_request_counter.labels( name=cls.NAME, diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 8702fe056f..a7d3482a66 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -282,17 +282,11 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: # Admin servlets below may not work on workers. if hs.config.worker.worker_app is not None: - # Some admin servlets can be mounted on workers when MSC3861 is enabled. - # Note that this is only for MSC3861 mode, as modern MAS using the - # matrix_authentication_service integration uses the dedicated MAS API. - if hs.config.experimental.msc3861.enabled: - register_servlets_for_msc3861_delegation(hs, http_server) - else: - UserRestServletV2Get(hs).register(http_server) + UserRestServletV2Get(hs).register(http_server) return - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled register_servlets_for_client_rest_resource(hs, http_server) BlockRoomRestServlet(hs).register(http_server) @@ -364,7 +358,7 @@ def register_servlets_for_client_rest_resource( hs: "HomeServer", http_server: HttpServer ) -> None: """Register only the servlets which need to be exposed on /_matrix/client/xxx""" - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled WhoisRestServlet(hs).register(http_server) PurgeHistoryStatusRestServlet(hs).register(http_server) @@ -388,16 +382,3 @@ def register_servlets_for_client_rest_resource( # don't add more things here: new servlets should only be exposed on # /_synapse/admin so should not go here. Instead register them in register_servlets. - - -def register_servlets_for_msc3861_delegation( - hs: "HomeServer", http_server: HttpServer -) -> None: - """Register servlets needed by MAS when MSC3861 is enabled""" - assert hs.config.experimental.msc3861.enabled - - UserRestServletV2(hs).register(http_server) - UsernameAvailableRestServlet(hs).register(http_server) - UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) - DeviceRestServlet(hs).register(http_server) - DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/events.py b/synapse/rest/admin/events.py index 1c311b0471..7dbd7f5d2b 100644 --- a/synapse/rest/admin/events.py +++ b/synapse/rest/admin/events.py @@ -3,9 +3,8 @@ from synapse.api.errors import NotFoundError from synapse.events.utils import ( + EventFormat, FilteredEvent, - SerializeEventConfig, - format_event_raw, ) from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -57,11 +56,11 @@ async def on_GET( if event is None: raise NotFoundError("Event not found") - config = SerializeEventConfig( + config = await self._event_serializer.create_config( as_client_event=False, - event_format=format_event_raw, + event_format=EventFormat.Raw, requester=requester, - only_event_fields=None, + event_field_allowlist=None, include_stripped_room_state=True, include_admin_metadata=True, ) diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index abdb937793..c91c5b6a49 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -62,7 +62,7 @@ class ExperimentalFeaturesRestServlet(RestServlet): for a given user """ - PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)") + PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index f6693e0923..e47b6e9efe 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -1028,7 +1028,7 @@ async def on_GET( ): as_client_event = False - serialize_options = SerializeEventConfig( + serialize_options = await self._event_serializer.create_config( as_client_event=as_client_event, requester=requester ) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 8265c2d789..53d1f2d366 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -109,9 +109,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() self._msc3866_enabled = hs.config.experimental.msc3866.enabled - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) @@ -1373,7 +1371,7 @@ async def on_DELETE( class AccountDataRestServlet(RestServlet): """Retrieve the given user's account data""" - PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata") + PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1411,7 +1409,7 @@ class UserReplaceMasterCrossSigningKeyRestServlet(RestServlet): """ PATTERNS = admin_patterns( - "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia" + "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia$" ) REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes @@ -1445,7 +1443,7 @@ class UserByExternalId(RestServlet): """Find a user based on an external ID from an auth provider""" PATTERNS = admin_patterns( - "/auth_providers/(?P[^/]*)/users/(?P[^/]*)" + "/auth_providers/(?P[^/]*)/users/(?P[^/]*)$" ) def __init__(self, hs: "HomeServer"): @@ -1471,7 +1469,7 @@ async def on_GET( class UserByThreePid(RestServlet): """Find a user based on 3PID of a particular medium""" - PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)") + PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1495,12 +1493,17 @@ async def on_GET( class RedactUser(RestServlet): """ - Redact all the events of a given user in the given rooms or if empty dict is provided - then all events in all rooms user is member of. Kicks off a background process and - returns an id that can be used to check on the progress of the redaction progress. + Redact all the events of a given user in the given rooms in the given time period. + Kicks off a background process and returns an id that can be used to check on the + progress of the redaction progress. + If empty rooms dict is provided then all events in all rooms user is member of will + be affected. + Parameters before_ts and after_ts are millisecond timestamps. + If both are omitted, then messages will be redacted regardless the time they were sent. + If only one parameter is sent, then all messages before or after given time will be redacted. """ - PATTERNS = admin_patterns("/user/(?P[^/]*)/redact") + PATTERNS = admin_patterns("/user/(?P[^/]*)/redact$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1512,6 +1515,8 @@ class PostBody(RequestBodyModel): reason: StrictStr | None = None limit: StrictInt | None = None use_admin: StrictBool | None = None + before_ts: StrictInt | None = None + after_ts: StrictInt | None = None async def on_POST( self, request: SynapseRequest, user_id: str @@ -1543,8 +1548,18 @@ async def on_POST( if not use_admin: use_admin = False + before_ts = body.before_ts + after_ts = body.after_ts + redact_id = await self.admin_handler.start_redact_events( - user_id, rooms, requester.serialize(), use_admin, body.reason, limit + user_id, + rooms, + requester.serialize(), + use_admin, + body.reason, + before_ts, + after_ts, + limit, ) return HTTPStatus.OK, {"redact_id": redact_id} @@ -1599,7 +1614,7 @@ class UserInvitesCount(RestServlet): Return the count of invites that the user has sent after the given timestamp """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1624,7 +1639,7 @@ class UserJoinedRoomCount(RestServlet): if they have subsequently left/been banned from those rooms. """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index d1e404f0dc..3b01e40121 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -619,7 +619,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # ThreePidBindRestServelet.PostBody with an `alias_generator` to handle # `threePidCreds` versus `three_pid_creds`. async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: - if self.hs.config.mas.enabled or self.hs.config.experimental.msc3861.enabled: + if self.hs.config.mas.enabled: raise NotFoundError(errcode=Codes.UNRECOGNIZED) if not self.hs.config.registration.enable_3pid_changes: @@ -911,7 +911,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled ThreepidRestServlet(hs).register(http_server) WhoamiRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index b18232fc56..cccc0473e8 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -61,7 +61,7 @@ class AccountDataServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/account_data/(?P[^/]*)" + "/user/(?P[^/]*)/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -136,7 +136,7 @@ class UnstableAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) @@ -174,7 +174,7 @@ class RoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)" + "/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -271,7 +271,7 @@ class UnstableRoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index 2c6ad5bcf0..4cbf6935c5 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -46,7 +46,7 @@ class AppservicePingRestServlet(RestServlet): PATTERNS = client_patterns( - "/appservice/(?P[^/]*)/ping", + "/appservice/(?P[^/]*)/ping$", releases=("v1",), ) diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index 566c9c98c5..215779a355 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -47,7 +47,7 @@ class AuthRestServlet(RestServlet): Current use is for web fallback auth. """ - PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web") + PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web$") def __init__(self, hs: "HomeServer"): super().__init__() @@ -97,20 +97,6 @@ async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: url.encode(), ) - elif self.hs.config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self.auth, MSC3861DelegatedAuth) - - base = await self.auth.account_management_url() - if base is not None: - url = f"{base}?action=org.matrix.cross_signing_reset" - else: - url = await self.auth.issuer() - return respond_with_redirect(request, url.encode()) - if stagetype == LoginType.RECAPTCHA: html = self.recaptcha_template.render( session=session, diff --git a/synapse/rest/client/auth_metadata.py b/synapse/rest/client/auth_metadata.py index 062b8ed13e..42decfdd6a 100644 --- a/synapse/rest/client/auth_metadata.py +++ b/synapse/rest/client/auth_metadata.py @@ -13,7 +13,6 @@ # limitations under the License. import logging import typing -from typing import cast from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import Codes, SynapseError @@ -51,7 +50,7 @@ def __init__(self, hs: "HomeServer"): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # This endpoint is unauthenticated and the response only depends on # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth/MSC3861DelegatedAuth.issuer() are already caching the + # MasDelegatedAuth.issuer() is already caching the # response in memory anyway. Ideally we would follow any Cache-Control directive # given by MAS, but this is fine for now. # @@ -72,14 +71,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: assert isinstance(self._auth, MasDelegatedAuth) return 200, {"issuer": await self._auth.issuer()} - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self._auth, MSC3861DelegatedAuth) - return 200, {"issuer": await self._auth.issuer()} - else: # Wouldn't expect this to be reached: the servelet shouldn't have been # registered. Still, fail gracefully if we are registered for some reason. @@ -115,7 +106,7 @@ def __init__(self, hs: "HomeServer"): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # This endpoint is unauthenticated and the response only depends on # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth/MSC3861DelegatedAuth.issuer() are already caching the + # MasDelegatedAuth.issuer() is already caching the # response in memory anyway. Ideally we would follow any Cache-Control directive # given by MAS, but this is fine for now. # @@ -136,14 +127,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: assert isinstance(self._auth, MasDelegatedAuth) return 200, await self._auth.auth_metadata() - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - auth = cast(MSC3861DelegatedAuth, self._auth) - return 200, await auth.auth_metadata() - else: # Wouldn't expect this to be reached: the servlet shouldn't have been # registered. Still, fail gracefully if we are registered for some reason. @@ -155,6 +138,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: AuthIssuerServlet(hs).register(http_server) AuthMetadataServlet(hs).register(http_server) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 2be5f5849d..4ddaaeda74 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "capabilities" ]["m.profile_fields"] + response["capabilities"]["org.matrix.msc4140.delayed_events"] = { + "max_delay_ms": self.config.server.max_event_delay_duration.as_millis(), + "max_scheduled": self.config.server.max_delayed_events_per_user, + } + if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { "enabled": self.config.room.forget_on_leave, diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 0231ed374d..115ad543fc 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -33,6 +33,7 @@ RestServlet, parse_and_validate_json_object_from_request, parse_integer, + parse_string, ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns, interactive_auth_handler @@ -76,7 +77,7 @@ class DeleteDevicesRestServlet(RestServlet): key which lists the device_ids to delete. Requires user interactive auth. """ - PATTERNS = client_patterns("/delete_devices") + PATTERNS = client_patterns("/delete_devices$") def __init__(self, hs: "HomeServer"): super().__init__() @@ -136,9 +137,7 @@ def __init__(self, hs: "HomeServer"): handler = hs.get_device_handler() self.device_handler = handler self.auth_handler = hs.get_auth_handler() - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def on_GET( self, request: SynapseRequest, device_id: str @@ -179,7 +178,7 @@ async def on_DELETE( if requester.app_service_id: # MSC4190 allows appservices to delete devices through this endpoint without UIA - # It's also allowed with MSC3861 enabled + # It's also allowed when auth is delegated pass else: @@ -249,17 +248,59 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main + async def on_GET( + self, request: SynapseRequest, device_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + since_token = parse_string(request, "from") + limit = parse_integer(request, "limit", 100) + + msgs = await self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=device_id, + since_token=since_token, + limit=limit, + ) + + if msgs.limited: + msgs_json = { + "events": msgs.events, + "next_batch": msgs.stream_id, + } + else: + msgs_json = { + "events": msgs.events, + } + + return 200, msgs_json + class PostBody(RequestBodyModel): + """ + This is deprecated: you should use GET instead. + + The POST version is provided temporarily for backwards compatibility + with a previous unstable draft of MSC3814. + """ + next_batch: StrictStr | None = None async def on_POST( self, request: SynapseRequest, device_id: str ) -> tuple[int, JsonDict]: + """ + This is deprecated: you should use GET instead. + + The POST version is provided temporarily for backwards compatibility + with a previous unstable draft of MSC3814. + """ + requester = await self.auth.get_user_by_req(request) next_batch = parse_and_validate_json_object_from_request( request, self.PostBody ).next_batch + limit = parse_integer(request, "limit", 100) msgs = await self.message_handler.get_events_for_dehydrated_device( @@ -269,7 +310,14 @@ async def on_POST( limit=limit, ) - return 200, msgs + # For backwards compatibility, we always provide next_batch from the + # POST API. + msgs_json = { + "events": msgs.events, + "next_batch": msgs.stream_id, + } + + return 200, msgs_json class DehydratedDeviceV2Servlet(RestServlet): @@ -432,7 +480,7 @@ async def on_PUT(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled if not auth_delegated: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/events.py b/synapse/rest/client/events.py index de73c96fd0..f5b894038e 100644 --- a/synapse/rest/client/events.py +++ b/synapse/rest/client/events.py @@ -25,7 +25,6 @@ from typing import TYPE_CHECKING from synapse.api.errors import SynapseError -from synapse.events.utils import SerializeEventConfig from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_string from synapse.http.site import SynapseRequest @@ -104,7 +103,7 @@ async def on_GET( result = await self._event_serializer.serialize_event( event, self.clock.time_msec(), - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return 200, result else: diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py index cfe82e1473..6718daaf23 100644 --- a/synapse/rest/client/filter.py +++ b/synapse/rest/client/filter.py @@ -37,7 +37,7 @@ class GetFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): @@ -76,7 +76,7 @@ async def on_GET( class CreateFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 463c87d92b..7ffde7ac04 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -536,7 +536,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # If yes, then we need to authenticate the change. # MSC4190 can skip UIA for replacing cross-signing keys as well. if is_cross_signing_setup and not requester.app_service_id: - # With MSC3861, UIA is not possible. Instead, the auth service has to + # With auth delegation, UIA is not possible. Instead, the auth service has to # explicitly mark the master key as replaceable. if self.hs.config.mas.enabled: if not master_key_updatable_without_uia: @@ -569,47 +569,8 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: }, ) - elif self.hs.config.experimental.msc3861.enabled: - if not master_key_updatable_without_uia: - # If MSC3861 is enabled, we can assume self.auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self.auth, MSC3861DelegatedAuth) - - uri = await self.auth.account_management_url() - if uri is not None: - url = f"{uri}?action=org.matrix.cross_signing_reset" - else: - url = await self.auth.issuer() - - # We use a dummy session ID as this isn't really a UIA flow, but we - # reuse the same API shape for better client compatibility. - raise InteractiveAuthIncompleteError( - "dummy", - { - "session": "dummy", - "flows": [ - {"stages": ["m.oauth"]}, - # The unstable name from MSC4312 should be supported until enough clients have adopted the stable (`m.oauth`) name: - {"stages": ["org.matrix.cross_signing_reset"]}, - ], - "params": { - "m.oauth": { - "url": url, - }, - "org.matrix.cross_signing_reset": { - "url": url, - }, - }, - "msg": "To reset your end-to-end encryption cross-signing " - f"identity, you first need to approve it at {url} and " - "then try again.", - }, - ) - else: - # Without MSC3861, we require UIA. + # Without auth delegation, we require UIA. await self.auth_handler.validate_user_via_ui_auth( requester, request, diff --git a/synapse/rest/client/knock.py b/synapse/rest/client/knock.py index cd3afda11e..cce856f0ef 100644 --- a/synapse/rest/client/knock.py +++ b/synapse/rest/client/knock.py @@ -46,7 +46,7 @@ class KnockRoomAliasServlet(RestServlet): POST /knock/{roomIdOrAlias} """ - PATTERNS = client_patterns("/knock/(?P[^/]*)") + PATTERNS = client_patterns("/knock/(?P[^/]*)$") CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index d5af132a8e..8df18d8e55 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -726,7 +726,7 @@ async def on_GET(self, request: SynapseRequest, idp_id: str | None = None) -> No class CasTicketServlet(RestServlet): - PATTERNS = client_patterns("/login/cas/ticket", v1=True) + PATTERNS = client_patterns("/login/cas/ticket$", v1=True) def __init__(self, hs: "HomeServer"): super().__init__() @@ -751,7 +751,7 @@ async def on_GET(self, request: SynapseRequest) -> None: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: return LoginRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/logout.py b/synapse/rest/client/logout.py index 7228092073..3363423b8d 100644 --- a/synapse/rest/client/logout.py +++ b/synapse/rest/client/logout.py @@ -92,7 +92,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: return LogoutRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index f80a43b297..ae3893d296 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -24,9 +24,8 @@ from synapse.api.constants import ReceiptTypes from synapse.events.utils import ( + EventFormat, FilteredEvent, - SerializeEventConfig, - format_event_for_client_v2_without_room_id, ) from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_integer, parse_string @@ -98,8 +97,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: next_token = None - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self._event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) now = self.clock.time_msec() diff --git a/synapse/rest/client/openid.py b/synapse/rest/client/openid.py index e624a48ce7..b6e2c078d8 100644 --- a/synapse/rest/client/openid.py +++ b/synapse/rest/client/openid.py @@ -67,7 +67,7 @@ class IdTokenServlet(RestServlet): } """ - PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token") + PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token$") EXPIRES_MS = 3600 * 1000 diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index de3ffdaa0b..4565280113 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -40,7 +40,7 @@ class PresenceStatusRestServlet(RestServlet): - PATTERNS = client_patterns("/presence/(?P[^/]*)/status", v1=True) + PATTERNS = client_patterns("/presence/(?P[^/]*)/status$", v1=True) CATEGORY = "Presence requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index c2ec5b3611..360a83f39c 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -58,7 +58,7 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: class ProfileRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) + PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): @@ -92,13 +92,13 @@ async def on_GET( class ProfileFieldRestServlet(RestServlet): PATTERNS = [ *client_patterns( - "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + "/profile/(?P[^/]*)/(?Pdisplayname)$", v1=True ), *client_patterns( - "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + "/profile/(?P[^/]*)/(?Pavatar_url)$", v1=True ), re.compile( - r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)$", ), ] @@ -112,7 +112,7 @@ def __init__(self, hs: "HomeServer"): if hs.config.experimental.msc4133_enabled: self.PATTERNS.append( re.compile( - r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)$" ) ) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 874e7487bf..8e0f2a2e7a 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from synapse.api.constants import ReceiptTypes +from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -66,6 +67,21 @@ async def on_POST( body = parse_json_object_from_request(request) unrecognized_types = set(body.keys()) - self._known_receipt_types + + if self.config.experimental.msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + 400, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + # Prevent considering the `allow_backward` field as a receipt type. + unrecognized_types -= {"com.beeper.allow_backward"} + else: + allow_backward = False + if unrecognized_types: # It's fine if there are unrecognized receipt types, but let's log # it to help debug clients that have typoed the receipt type. @@ -86,6 +102,7 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index d3a43537bb..949a1e64ad 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -50,6 +51,7 @@ def __init__(self, hs: "HomeServer"): self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() self._main_store = hs.get_datastores().main + self._msc4446_enabled = hs.config.experimental.msc4446_enabled self._known_receipt_types = { ReceiptTypes.READ, @@ -73,6 +75,25 @@ async def on_POST( body = parse_json_object_from_request(request) + if self._msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + if allow_backward and receipt_type != ReceiptTypes.FULLY_READ: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward is only allowed to be true for " + f"{ReceiptTypes.FULLY_READ}.", + Codes.INVALID_PARAM, + ) + else: + allow_backward = False + # Pull the thread ID, if one exists. thread_id = None if "thread_id" in body: @@ -108,6 +129,7 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 73832ba7a8..8355d0c571 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -338,7 +338,7 @@ async def on_GET(self, request: Request, medium: str) -> None: class UsernameAvailabilityRestServlet(RestServlet): - PATTERNS = client_patterns("/register/available") + PATTERNS = client_patterns("/register/available$") def __init__(self, hs: "HomeServer"): super().__init__() @@ -401,7 +401,7 @@ class RegistrationTokenValidityRestServlet(RestServlet): """ PATTERNS = client_patterns( - f"/register/{LoginType.REGISTRATION_TOKEN}/validity", + f"/register/{LoginType.REGISTRATION_TOKEN}/validity$", releases=("v1",), ) CATEGORY = "Registration/login requests" @@ -907,7 +907,7 @@ async def _do_guest_registration( class RegisterAppServiceOnlyRestServlet(RestServlet): """An alternative registration API endpoint that only allows ASes to register - This replaces the regular /register endpoint if MSC3861. There are two notable + This replaces the regular /register endpoint if auth is delegated to MAS. There are two notable differences with the regular /register endpoint: - It only allows the `m.login.application_service` login type - It does not create a device or access token for the just-registered user @@ -1068,7 +1068,7 @@ def _calculate_registration_flows( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: RegisterAppServiceOnlyRestServlet(hs).register(http_server) return diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index c913bc6970..daf220545e 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -94,7 +94,7 @@ async def on_GET( class ThreadsServlet(RestServlet): - PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads"),) + PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads$"),) CATEGORY = "Client API requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 6e00197b6d..c3bb82e15c 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -53,9 +53,9 @@ from synapse.api.filtering import Filter from synapse.events.utils import ( EventClientSerializer, + EventFormat, FilteredEvent, SerializeEventConfig, - format_event_for_client_v2, ) from synapse.handlers.pagination import GetMessagesResult from synapse.http.server import HttpServer @@ -84,6 +84,7 @@ from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.stringutils import parse_and_validate_server_name @@ -215,7 +216,6 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.clock = hs.get_clock() self._event_serializer = hs.get_event_client_serializer() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._msc4354_enabled = hs.config.experimental.msc4354_enabled @@ -289,8 +289,8 @@ async def on_GET( event = await self._event_serializer.serialize_event( FilteredEvent.state(data), self.clock.time_msec(), - config=SerializeEventConfig( - event_format=format_event_for_client_v2, + config=await self._event_serializer.create_config( + event_format=EventFormat.ClientV2, requester=requester, ), ) @@ -343,7 +343,7 @@ async def on_PUT( if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -416,7 +416,6 @@ def __init__(self, hs: "HomeServer"): self.event_creation_handler = hs.get_event_creation_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._msc4354_enabled = hs.config.experimental.msc4354_enabled def register(self, http_server: HttpServer) -> None: @@ -442,7 +441,7 @@ async def _do( if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -515,47 +514,20 @@ async def on_PUT( ) -def _parse_request_delay( - request: SynapseRequest, - max_delay: int | None, -) -> int | None: +def _parse_request_for_delayed_event_delay(request: SynapseRequest) -> Duration | None: """Parses from the request string the delay parameter for delayed event requests, and checks it for correctness. Args: request: the twisted HTTP request. - max_delay: the maximum allowed value of the delay parameter, - or None if no delay parameter is allowed. Returns: The value of the requested delay, or None if it was absent. Raises: - SynapseError: if the delay parameter is present and forbidden, - or if it exceeds the maximum allowed value. + SynapseError: if the delay parameter is present and invalid. """ - delay = parse_integer(request, "org.matrix.msc4140.delay") - if delay is None: - return None - if max_delay is None: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Delayed events are not supported on this server", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_UNSUPPORTED", - }, - ) - if delay > max_delay: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "The requested delay exceeds the allowed maximum.", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": max_delay, - }, - ) - return delay + delay_ms = parse_integer(request, "org.matrix.msc4140.delay") + return Duration(milliseconds=delay_ms) if delay_ms is not None else None # TODO: Needs unit testing for room ID + alias joins @@ -925,7 +897,7 @@ async def on_GET( ): as_client_event = False - serialize_options = SerializeEventConfig( + serialize_options = await self.event_serializer.create_config( as_client_event=as_client_event, requester=requester ) @@ -1114,7 +1086,7 @@ async def on_GET( event, self.clock.time_msec(), bundle_aggregations=aggregations, - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return 200, event_dict @@ -1154,7 +1126,9 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) time_now = self.clock.time_msec() - serializer_options = SerializeEventConfig(requester=requester) + serializer_options = await self._event_serializer.create_config( + requester=requester + ) results = { "events_before": await self._event_serializer.serialize_events( event_context.events_before, @@ -1531,7 +1505,7 @@ class RoomAliasListServlet(RestServlet): PATTERNS = [ re.compile( r"^/_matrix/client/unstable/org\.matrix\.msc2432" - r"/rooms/(?P[^/]*)/aliases" + r"/rooms/(?P[^/]*)/aliases$" ), ] + list(client_patterns("/rooms/(?P[^/]*)/aliases$", unstable=False)) CATEGORY = "Client API requests" diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 702ddcd6ca..962317dedb 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -30,10 +30,9 @@ from synapse.api.presence import UserPresenceState from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( + EventFormat, FilteredEvent, SerializeEventConfig, - format_event_for_client_v2_without_room_id, - format_event_raw, ) from synapse.handlers.presence import format_user_presence_state from synapse.handlers.sliding_sync import SlidingSyncConfig, SlidingSyncResult @@ -304,18 +303,18 @@ async def encode_response( ) -> JsonDict: logger.debug("Formatting events in sync response") if filter.event_format == "client": - event_formatter = format_event_for_client_v2_without_room_id + event_formatter = EventFormat.ClientV2WithoutRoomId elif filter.event_format == "federation": - event_formatter = format_event_raw + event_formatter = EventFormat.Raw else: raise Exception("Unknown event format %s" % (filter.event_format,)) - serialize_options = SerializeEventConfig( + serialize_options = await self._event_serializer.create_config( event_format=event_formatter, requester=requester, - only_event_fields=filter.event_fields, + event_field_allowlist=filter.event_fields, ) - stripped_serialize_options = SerializeEventConfig( + stripped_serialize_options = await self._event_serializer.create_config( event_format=event_formatter, requester=requester, include_stripped_room_state=True, @@ -931,8 +930,8 @@ async def encode_rooms( ) -> JsonDict: time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self.event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) @@ -1158,8 +1157,8 @@ async def _serialise_sticky_events( time_now = self.clock.time_msec() # Same as SSS timelines. # - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self.event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index 5699ff35c7..f7272754e6 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -73,7 +73,7 @@ class TagServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)" + "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)$" ) CATEGORY = "Account data requests" diff --git a/synapse/rest/client/thirdparty.py b/synapse/rest/client/thirdparty.py index c17335eb48..b4be564eb0 100644 --- a/synapse/rest/client/thirdparty.py +++ b/synapse/rest/client/thirdparty.py @@ -37,7 +37,7 @@ class ThirdPartyProtocolsServlet(RestServlet): - PATTERNS = client_patterns("/thirdparty/protocols") + PATTERNS = client_patterns("/thirdparty/protocols$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/tokenrefresh.py b/synapse/rest/client/tokenrefresh.py index 2b4f7f8953..873c6f3e4e 100644 --- a/synapse/rest/client/tokenrefresh.py +++ b/synapse/rest/client/tokenrefresh.py @@ -39,7 +39,7 @@ class TokenRefreshRestServlet(RestServlet): token. """ - PATTERNS = client_patterns("/tokenrefresh") + PATTERNS = client_patterns("/tokenrefresh$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 809e920a2b..9312a616da 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -25,11 +25,9 @@ import re from typing import TYPE_CHECKING -from synapse.api.constants import RoomCreationPreset from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest -from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.types import JsonDict if TYPE_CHECKING: @@ -47,25 +45,10 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config self.auth = hs.get_auth() self.store = hs.get_datastores().main - - # Calculate these once since they shouldn't change after start-up. - self.e2ee_forced_public = ( - RoomCreationPreset.PUBLIC_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_private = ( - RoomCreationPreset.PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_trusted_private = ( - RoomCreationPreset.TRUSTED_PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) + self.rust_handlers = hs.get_rust_handlers() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - msc3881_enabled = self.config.experimental.msc3881_enabled - msc3575_enabled = self.config.experimental.msc3575_enabled - + user_id = None if self.auth.has_access_token(request): requester = await self.auth.get_user_by_req( request, @@ -74,13 +57,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: allow_expired=True, ) user_id = requester.user.to_string() - - msc3881_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3881 - ) - msc3575_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3575 - ) else: # Allow caching of unauthenticated responses, as they only depend # on server configuration which rarely changes. @@ -102,117 +78,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # authenticated responses are not served from cache. request.setHeader(b"Vary", b"Authorization") + versions_response_body = await self.rust_handlers.versions.get_versions(user_id) + return ( 200, - { - "versions": [ - # XXX: at some point we need to decide whether we need to include - # the previous version numbers, given we've defined r0.3.0 to be - # backwards compatible with r0.2.0. But need to check how - # conscientious we've been in compatibility, and decide whether the - # middle number is the major revision when at 0.X.Y (as opposed to - # X.Y.Z). And we need to decide whether it's fair to make clients - # parse the version string to figure out what's going on. - "r0.0.1", - "r0.1.0", - "r0.2.0", - "r0.3.0", - "r0.4.0", - "r0.5.0", - "r0.6.0", - "r0.6.1", - "v1.1", - "v1.2", - "v1.3", - "v1.4", - "v1.5", - "v1.6", - "v1.7", - "v1.8", - "v1.9", - "v1.10", - "v1.11", - "v1.12", - ], - # as per MSC1497: - "unstable_features": { - # Implements support for label-based filtering as described in - # MSC2326. - "org.matrix.label_based_filtering": True, - # Implements support for cross signing as described in MSC1756 - "org.matrix.e2e_cross_signing": True, - # Implements additional endpoints as described in MSC2432 - "org.matrix.msc2432": True, - # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666.query_mutual_rooms.stable": True, - # Whether new rooms will be set to encrypted or not (based on presets). - "io.element.e2ee_forced.public": self.e2ee_forced_public, - "io.element.e2ee_forced.private": self.e2ee_forced_private, - "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, - # Supports the busy presence state described in MSC3026. - "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, - # Supports receiving private read receipts as per MSC2285 - "org.matrix.msc2285.stable": True, # TODO: Remove when MSC2285 becomes a part of the spec - # Supports filtering of /publicRooms by room type as per MSC3827 - "org.matrix.msc3827.stable": True, - # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above - # Support for thread read receipts & notification counts. - "org.matrix.msc3771": True, - "org.matrix.msc3773": self.config.experimental.msc3773_enabled, - # Allows moderators to fetch redacted event content as described in MSC2815 - "fi.mau.msc2815": self.config.experimental.msc2815_enabled, - # Adds a ping endpoint for appservices to check HS->AS connection - "fi.mau.msc2659.stable": True, # TODO: remove when "v1.7" is added above - # TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - "org.matrix.msc3882": self.config.auth.login_via_existing_enabled, - # Adds support for remotely enabling/disabling pushers, as per MSC3881 - "org.matrix.msc3881": msc3881_enabled, - # Adds support for filtering /messages by event relation. - "org.matrix.msc3874": self.config.experimental.msc3874_enabled, - # Adds support for relation-based redactions as per MSC3912. - "org.matrix.msc3912": self.config.experimental.msc3912_enabled, - # Whether recursively provide relations is supported. - # TODO This is no longer needed once unstable MSC3981 does not need to be supported. - "org.matrix.msc3981": True, - # Adds support for deleting account data. - "org.matrix.msc3391": self.config.experimental.msc3391_enabled, - # Allows clients to inhibit profile update propagation. - "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation, - # Allows clients to handle push for encrypted events. - "org.matrix.msc4028": self.config.experimental.msc4028_push_encrypted_events, - # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - "org.matrix.msc4108": ( - self.config.experimental.msc4108_enabled - or ( - self.config.experimental.msc4108_delegation_endpoint - is not None - ) - ), - # MSC4140: Delayed events - "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), - # MSC4143: Matrix RTC transports (LiveKit backend) - "org.matrix.msc4143": self.config.experimental.msc4143_enabled, - # Simplified sliding sync - "org.matrix.simplified_msc3575": msc3575_enabled, - # Arbitrary key-value profile fields. - "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, - "uk.tcpip.msc4133.stable": True, - # MSC4155: Invite filtering - "org.matrix.msc4155": self.config.experimental.msc4155_enabled, - # MSC4306: Support for thread subscriptions - "org.matrix.msc4306": self.config.experimental.msc4306_enabled, - # MSC4169: Backwards-compatible redaction sending using `/send` - "com.beeper.msc4169": self.config.experimental.msc4169_enabled, - # MSC4354: Sticky events - "org.matrix.msc4354": self.config.experimental.msc4354_enabled, - # MSC4380: Invite blocking - "org.matrix.msc4380.stable": True, - # MSC4445: Sync timeline order - "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, - "uk.timedout.msc4491.create_room_invite_reasons": self.config.experimental.msc4491_enabled, - }, - }, + versions_response_body, ) diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 1b6b001b45..c962d5f624 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -37,7 +37,7 @@ class CreateResource(RestServlet): - PATTERNS = [re.compile("/_matrix/media/v1/create")] + PATTERNS = [re.compile("/_matrix/media/v1/create$")] def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 665ce77dd7..e04b84ecac 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -58,11 +58,6 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc if hs.config.mas.enabled: resources["/_synapse/mas"] = MasResource(hs) - elif hs.config.experimental.msc3861.enabled: - from synapse.rest.synapse.client.jwks import JwksResource - - resources["/_synapse/jwks"] = JwksResource(hs) - resources["/_synapse/mas"] = MasResource(hs) # provider-specific SSO bits. Only load these if they are enabled, since they # rely on optional dependencies. diff --git a/synapse/rest/synapse/client/jwks.py b/synapse/rest/synapse/client/jwks.py deleted file mode 100644 index 15ff6f47c1..0000000000 --- a/synapse/rest/synapse/client/jwks.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2022 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -import logging -from typing import TYPE_CHECKING - -from synapse.http.server import DirectServeJsonResource -from synapse.http.site import SynapseRequest -from synapse.types import JsonDict - -if TYPE_CHECKING: - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - - -class JwksResource(DirectServeJsonResource): - def __init__(self, hs: "HomeServer"): - super().__init__(clock=hs.get_clock(), extract_context=True) - - # Parameters that are allowed to be exposed in the public key. - # This is done manually, because authlib's private to public key conversion - # is unreliable depending on the version. Instead, we just serialize the private - # key and only keep the public parameters. - # List from https://www.iana.org/assignments/jose/jose.xhtml#web-key-parameters - public_parameters = { - "kty", - "use", - "key_ops", - "alg", - "kid", - "x5u", - "x5c", - "x5t", - "x5t#S256", - "crv", - "x", - "y", - "n", - "e", - "ext", - } - - key = hs.config.experimental.msc3861.jwk - - if key is not None: - private_key = key.as_dict() - public_key = { - k: v for k, v in private_key.items() if k in public_parameters - } - keys = [public_key] - else: - keys = [] - - self.res = { - "keys": keys, - } - - async def _async_render_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - return 200, self.res diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py index 7346198b75..71457a22b5 100644 --- a/synapse/rest/synapse/mas/_base.py +++ b/synapse/rest/synapse/mas/_base.py @@ -29,18 +29,9 @@ class MasBaseResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): auth = hs.get_auth() - if hs.config.mas.enabled: - assert isinstance(auth, MasDelegatedAuth) + assert isinstance(auth, MasDelegatedAuth) - self._is_request_from_mas = auth.is_request_using_the_shared_secret - else: - # Importing this module requires authlib, which is an optional - # dependency but required if msc3861 is enabled - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(auth, MSC3861DelegatedAuth) - - self._is_request_from_mas = auth.is_request_using_the_admin_token + self._is_request_from_mas = auth.is_request_using_the_shared_secret DirectServeJsonResource.__init__(self, extract_context=True) self.store = cast("GenericWorkerStore", hs.get_datastores().main) diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index 801d474ecc..3193761ad0 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -61,22 +61,6 @@ async def get_well_known(self) -> JsonDict | None: "account": await self._auth.account_management_url(), } - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self._auth, MSC3861DelegatedAuth) - - result["org.matrix.msc2965.authentication"] = { - "issuer": await self._auth.issuer(), - } - account_management_url = await self._auth.account_management_url() - if account_management_url is not None: - result["org.matrix.msc2965.authentication"]["account"] = ( - account_management_url - ) - if self._config.server.extra_well_known_client_content: for ( key, diff --git a/synapse/server.py b/synapse/server.py index 8bf19f11b5..b756223e54 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -174,6 +174,7 @@ from synapse.storage import Databases from synapse.storage.controllers import StorageControllers from synapse.streams.events import EventSources +from synapse.synapse_rust.handlers import RustHandlers from synapse.synapse_rust.msc4388_rendezvous import MSC4388RendezvousHandler from synapse.synapse_rust.rendezvous import RendezvousHandler from synapse.types import DomainSpecificString, ISynapseReactor @@ -741,10 +742,6 @@ def get_replication_notifier(self) -> ReplicationNotifier: def get_auth(self) -> Auth: if self.config.mas.enabled: return MasDelegatedAuth(self) - if self.config.experimental.msc3861.enabled: - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - return MSC3861DelegatedAuth(self) return InternalAuth(self) @cache_in_self @@ -963,6 +960,10 @@ def get_send_email_handler(self) -> SendEmailHandler: def get_set_password_handler(self) -> SetPasswordHandler: return SetPasswordHandler(self) + @cache_in_self + def get_rust_handlers(self) -> RustHandlers: + return RustHandlers(self) + @cache_in_self def get_event_sources(self) -> EventSources: return EventSources(self) diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 6c2bf90b37..1e2db20b44 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -84,7 +84,7 @@ def __init__( ) self.exclusive_user_regex = _make_exclusive_regex(self.services_cache) # When OAuth is enabled, force all appservices to enable MSC4190 too. - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: for appservice in self.services_cache: appservice.msc4190_device_management = True diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1727f589e2..bb512611e4 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -17,7 +17,7 @@ import attr -from synapse.api.errors import NotFoundError +from synapse.api.errors import LimitExceededError, NotFoundError from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( DatabasePool, @@ -28,6 +28,7 @@ from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomID from synapse.util import stringutils +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -122,20 +123,84 @@ async def add_delayed_event( state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, + limit: int, ) -> tuple[DelayID, Timestamp]: """ Inserts a new delayed event in the DB. + Args: + user_localpart: The localpart of the requester of the delayed event, who will be its owner. + device_id: The device ID of the requester. + creation_ts: The timestamp of when the request to add the delayed event was made. + room_id: The ID of the room where the event should be sent to. + event_type: The type of event to be sent. + state_key: The state key of the event to be sent, or None if it is not a state event. + origin_server_ts: The custom timestamp to send the event with. + If None, the timestamp will be the actual time when the event is sent. + content: The content of the event to be sent. + delay: How long to wait before automatically sending the event. + sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). + The event will be attempted to be reliably delivered to clients and remote servers + during its sticky period. + limit: The maximum number of delayed events the DB may store for the given requester. + Must be greater than 0. Returns: The generated ID assigned to the added delayed event, and the send time of the next delayed event to be sent, which is either the event just added or one added earlier. + + Raises: + LimitExceededError: if the DB has reached the limit of + how many delayed events it may store for the given requester. + AssertionError: if the limit is not greater than 0. """ + assert limit > 0, "limit must be greater than 0" + delay_id = _generate_delay_id() - send_ts = Timestamp(creation_ts + delay) + delay_ms = delay.as_millis() + send_ts = creation_ts + delay_ms def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: + num_existing: int = self.db_pool.simple_select_one_onecol_txn( + txn, + table="delayed_events", + keyvalues={"user_localpart": user_localpart}, + retcol="COUNT(*)", + ) + if num_existing >= limit: + # Find the send_ts threshold that will bring the queue back under the limit. + # When the amount of existing delayed events has reached the limit, + # this will be the send time of the next delayed event to be sent. + # When the amount has exceeded the limit (e.g., due to config changes), + # this will be the send time of the delayed event that will be sent + # once all earlier events that exceed the limit have been sent. + # + # FIXME: Remove "AS subquery" after dropping support for PostgreSQL <16 + txn.execute( + """ + SELECT MAX(send_ts) FROM ( + SELECT * FROM delayed_events + WHERE user_localpart = ? + ORDER BY send_ts ASC + LIMIT ? + ) AS subquery + """, + ( + user_localpart, + num_existing - limit + 1, + ), + ) + row = txn.fetchone() + assert row + retry_after_ms = row[0] - self.clock.time_msec() + err = LimitExceededError( + limiter_name="add_delayed_event", + retry_after_ms=retry_after_ms if retry_after_ms > 0 else None, + ) + err.msg = "The maximum number of delayed events has been reached." + raise err + self.db_pool.simple_insert_txn( txn, table="delayed_events", @@ -143,7 +208,7 @@ def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: "delay_id": delay_id, "user_localpart": user_localpart, "device_id": device_id, - "delay": delay, + "delay": delay_ms, "send_ts": send_ts, "room_id": room_id, "event_type": event_type, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 8670d68f38..0e4c8ac491 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -2019,7 +2019,10 @@ def _update_remote_device_list_cache_txn( txn, table="device_lists_remote_extremeties", keyvalues={"user_id": user_id}, - values={"stream_id": stream_id}, + # `stream_id` is a TEXT column, so store it as a string (this method + # takes an int) rather than relying on the driver to coerce it. + # (Ideally we'd fix the schema, but that is non-trivial) + values={"stream_id": str(stream_id)}, ) async def add_device_change_to_streams( @@ -2558,9 +2561,14 @@ def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: # We default to 0 here as that is less than all possible stream IDs. min_stream_id = 0 - def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: - nonlocal min_stream_id - + def prune_device_lists_changes_in_room_txn( + txn: LoggingTransaction, min_stream_id: int + ) -> tuple[int, int]: + """ + Returns tuple of: + - number of rows deleted + - new `min_stream_id` for the next iteration + """ delete_sql = """ DELETE FROM device_lists_changes_in_room WHERE stream_id IN ( @@ -2593,13 +2601,14 @@ def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: updatevalues={"stream_id": min_stream_id}, ) - return num_deleted + return num_deleted, min_stream_id progress_num_rows_deleted = 0 while True: - batch_deleted = await self.db_pool.runInteraction( + batch_deleted, min_stream_id = await self.db_pool.runInteraction( "prune_device_lists_changes_in_room", prune_device_lists_changes_in_room_txn, + min_stream_id, ) finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index d84c58dcf8..4cb55f8c46 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1983,6 +1983,13 @@ async def get_missing_events( latest_events: list[str], limit: int, ) -> list[EventBase]: + """ + Walk backwards in the DAG of events, + starting at `latest_events` and stopping at `earliest_events` (or when having reached `limit` events). + + This function will check that `latest_events` and `earliest_events` are in the correct + room (`room_id`), appropriately ignoring any that aren't. + """ ids = await self.db_pool.runInteraction( "get_missing_events", self._get_missing_events, @@ -2001,20 +2008,49 @@ def _get_missing_events( latest_events: list[str], limit: int, ) -> list[str]: + # It's OK that this has not been filtered by correct-room, + # because we will only compare based on event ID from the events + # we happen to run into. seen_events = set(earliest_events) - front = set(latest_events) - seen_events - event_results: list[str] = [] - query = ( - "SELECT prev_event_id FROM event_edges " - "WHERE event_id = ? AND NOT is_state " - "LIMIT ?" + # Pre-filter the `latest_events` to only include those + # that are in this room (and that we know about) + # This makes events in the wrong room get treated the same as unknown events. + events_clause, events_args = make_in_list_sql_clause( + self.database_engine, + "event_id", + # Don't waste time looking at events that the requester told us + # they already know about. + # (They probably shouldn't send this in the first place) + set(latest_events) - seen_events, ) + txn.execute( + f""" + SELECT event_id + FROM events + WHERE {events_clause} AND room_id = ? + """, + (*events_args, room_id), + ) + # Start walking back from the legitimate and known `latest_events` + front = {latest_event_id for (latest_event_id,) in txn} + + event_results: list[str] = [] while front and len(event_results) < limit: new_front = set() for event_id in front: - txn.execute(query, (event_id, limit - len(event_results))) + txn.execute( + """ + SELECT ee.prev_event_id FROM event_edges AS ee + JOIN events ON events.event_id = ee.prev_event_id + WHERE ee.event_id = ? + AND events.room_id = ? + AND NOT ee.is_state + LIMIT ? + """, + (event_id, room_id, limit - len(event_results)), + ) new_results = {t[0] for t in txn} - seen_events new_front |= new_results diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 84b38f4bf2..d92bbeeae3 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -3513,7 +3513,10 @@ def _store_rejections_txn( values={ "event_id": event_id, "reason": reason, - "last_check": self._clock.time_msec(), + # `last_check` is a TEXT column, so store the timestamp as a + # string rather than relying on the driver to coerce an int. + # (Ideally we'd fix the schema, but that is non-trivial) + "last_check": str(self._clock.time_msec()), }, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 65b7d251d9..27dab290b3 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -2702,7 +2702,10 @@ def mark_event_rejected_txn( keyvalues={"event_id": event_id}, values={ "reason": rejection_reason, - "last_check": self.clock.time_msec(), + # `last_check` is a TEXT column, so store the timestamp as a + # string rather than relying on the driver to coerce an int. + # (Ideally we'd fix the schema, but that is non-trivial) + "last_check": str(self.clock.time_msec()), }, ) self.db_pool.simple_update_txn( @@ -2715,15 +2718,23 @@ def mark_event_rejected_txn( self.invalidate_get_event_cache_after_txn(txn, event_id) async def get_events_sent_by_user_in_room( - self, user_id: str, room_id: str, limit: int, filter: list[str] | None = None + self, + user_id: str, + room_id: str, + limit: int, + filter: list[str] | None = None, + before_ts: int | None = None, + after_ts: int | None = None, ) -> list[str] | None: """ - Get a list of event ids of events sent by the user in the specified room + Get a list of event ids of events sent by the user in the specified room in the specified time period Args: user_id: user ID to search against room_id: room ID of the room to search for events in filter: type of events to filter for + before_ts: filter for events that happened before this time (optional) + after_ts: filter for events that happened after this time (optional) limit: maximum number of event ids to return """ @@ -2734,16 +2745,32 @@ def _get_events_by_user_in_room_txn( filter: list[str] | None, batch_size: int, offset: int, + before_ts: int | None = None, + after_ts: int | None = None, ) -> tuple[list[str] | None, int]: + clause = "" if filter: base_clause, args = make_in_list_sql_clause( txn.database_engine, "type", filter ) clause = f"AND {base_clause}" - parameters = (user_id, room_id, *args, batch_size, offset) + parameters = (user_id, room_id, *args) else: - clause = "" - parameters = (user_id, room_id, batch_size, offset) + parameters = (user_id, room_id) + + if before_ts: + if clause: + clause += " AND " + clause += "origin_server_ts <= ?" + parameters += (before_ts,) + + if after_ts: + if clause: + clause += " AND " + clause += "origin_server_ts >= ?" + parameters += (after_ts,) + + parameters += (batch_size, offset) sql = f""" SELECT event_id FROM events @@ -2777,6 +2804,8 @@ def _get_events_by_user_in_room_txn( filter, batch_size, offset, + before_ts, + after_ts, ) if res: selected_ids = selected_ids + res diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index 2019ad9904..c334457af3 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -156,7 +156,7 @@ async def get_user_filter( # filter_id is BIGINT UNSIGNED, so if it isn't a number, fail # with a coherent error message rather than 500 M_UNKNOWN. try: - int(filter_id) + filter_id = int(filter_id) except ValueError: raise SynapseError(400, "Invalid filter ID", Codes.INVALID_PARAM) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 9b787e19a3..68548434a9 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -293,15 +293,17 @@ async def get_profile_fields(self, user_id: UserID) -> dict[str, str]: async def create_profile(self, user_id: UserID) -> None: """ - Create a blank profile for a user. + Create a blank profile for a user, if one does not already exist. Args: user_id: The user to create the profile for. """ user_localpart = user_id.localpart - await self.db_pool.simple_insert( + await self.db_pool.simple_upsert( table="profiles", - values={"user_id": user_localpart, "full_user_id": user_id.to_string()}, + keyvalues={"full_user_id": user_id.to_string()}, + values={}, + insertion_values={"user_id": user_localpart}, desc="create_profile", ) diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index d361166cec..5086f4639b 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import ( TYPE_CHECKING, Any, @@ -31,7 +32,7 @@ from twisted.internet import defer -from synapse.api.errors import StoreError +from synapse.api.errors import Codes, StoreError, SynapseError from synapse.config.homeserver import ExperimentalConfig from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.replication.tcp.streams import PushRulesStream @@ -112,6 +113,22 @@ def _load_rules( return filtered_rules +def _push_rule_size_for_limits(*, conditions_json: str, actions_json: str) -> int: + """ + Returns the size of a push rule, as used for applying the size limit. + + We aren't tied to any particular definition, but currently this is + simply the size in bytes of the conditions and actions JSON added together, + so not rocket science, but this function provides a 'label' for it. + """ + # This is not a very predictable way of calculating the size from the + # point of view of the client, but since it's an out-of-spec limit + # entirely at our discretion, we don't really have to worry about + # the exact calculation. + # FIXME: Spec a predictable push rule size limit + return len(conditions_json.encode("utf-8")) + len(actions_json.encode("utf-8")) + + class PushRulesWorkerStore( ApplicationServiceWorkerStore, PusherWorkerStore, @@ -170,6 +187,8 @@ def __init__( self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") + self._config = hs.config.push_rules + def get_max_push_rules_stream_id(self) -> int: """Get the position of the push rules stream. @@ -409,6 +428,29 @@ async def add_push_rule( conditions_json = json_encoder.encode(conditions) actions_json = json_encoder.encode(actions) + + rule_id_len = len(rule_id.encode("utf-8")) + if rule_id_len > self._config.limits.rule_id_length: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ID length exceeds server limit ({rule_id_len} bytes > {self._config.limits.rule_id_length} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + async with self._push_rules_stream_id_gen.get_next() as stream_id: event_stream_ordering = self._stream_id_gen.get_current_token() @@ -578,13 +620,19 @@ def _upsert_push_rule_txn( actions_json: str, update_stream: bool = True, ) -> None: + """Specialised version of simple_upsert_txn that picks a push_rule_id + using the _push_rule_id_gen if it needs to insert the rule. + + Preconditions: + - this worker is a push writer + - the "push_rules" table is locked + - the push rule has already been validated, + including for rule ID length and rule body size. + """ + if not self._is_push_writer: raise Exception("Not a push writer") - """Specialised version of simple_upsert_txn that picks a push_rule_id - using the _push_rule_id_gen if it needs to insert the rule. It assumes - that the "push_rules" table is locked""" - sql = ( "UPDATE push_rules" " SET priority_class = ?, priority = ?, conditions = ?, actions = ?" @@ -597,6 +645,27 @@ def _upsert_push_rule_txn( ) if txn.rowcount == 0: + # About to add a new rule, so check our limits first. + txn.execute( + """ + SELECT COUNT(*) FROM push_rules + WHERE user_name = ? + """, + (user_id,), + ) + (num_push_rules,) = cast(tuple[int], txn.fetchone()) + if num_push_rules >= self._config.limits.rule_count: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Creating a push rule would exceed the limit on the number of push rules associated with your account ({num_push_rules + 1} rules > {self._config.limits.rule_count} rules)", + # FIXME: Provide a better error code, especially for this case. + # None of the existing options seem entirely correct, though. + # `M_USER_LIMIT_EXCEEDED` comes closest but needs an `info_uri`. + # Should follow up and add a built-in error page template, similarly to + # https://github.com/element-hq/synapse/pull/18876 ? + Codes.UNKNOWN, + ) + # We didn't update a row with the given rule_id so insert one push_rule_id = self._push_rule_id_gen.get_next() @@ -839,6 +908,28 @@ def set_push_rule_actions_txn( ) else: try: + # Before updating the push rule, we need to check that we won't exceed + # the size limit on push rules. + # For that, we need to fetch the `conditions` JSON. + conditions_json = self.db_pool.simple_select_one_onecol_txn( + txn, + "push_rules", + {"user_name": user_id, "rule_id": rule_id}, + "conditions", + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + self.db_pool.simple_update_one_txn( txn, "push_rules", diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 95aa2cb7dc..768bf6e94f 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -245,6 +245,12 @@ async def _flag_existing_quarantined_media( last_remote_media_id = progress.get("last_remote_media_id", "") last_remote_origin = progress.get("last_remote_origin", "") + # Once a table has been fully processed we record it in the progress so that + # we stop re-running its (now empty) query on every subsequent iteration while + # the other table is still being worked through. + local_done = progress.get("local_done", False) + remote_done = progress.get("remote_done", False) + # The `ORDER BY` here would normally miss records if the admin (un)quarantined a # record, but that doesn't affect the background update because we also insert # into the stream table upon quarantine status changing. Worst case is the admin @@ -266,54 +272,75 @@ async def _flag_existing_quarantined_media( # is further reinforced by not all changes being captured by the table anyway. # See https://github.com/element-hq/synapse/issues/19672 for more details. def flag_quarantined(txn: LoggingTransaction) -> int: - # It doesn't matter which order we do these in, as long as we do both of them. - txn.execute( - """ - SELECT NULL AS media_origin, media_id - FROM local_media_repository - WHERE quarantined_by IS NOT NULL - AND media_id > ? - ORDER BY media_id - LIMIT ? - """, - (last_local_media_id, batch_size), - ) - local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) - if len(local_media_result) > 0: - self._insert_quarantine_changes_txn(txn, local_media_result, True) - - # We use a >= ? on the media origin to avoid missing records when media IDs - # collide between origins (the table's unique constraint is on `(media_origin, media_id)`). - # Filtering by `(media_origin, media_id)` also makes sure we're using an index. - txn.execute( - """ - SELECT media_origin, media_id - FROM remote_media_cache - WHERE quarantined_by IS NOT NULL - AND media_origin >= ? AND media_id > ? - ORDER BY media_origin, media_id - LIMIT ? - """, - (last_remote_origin, last_remote_media_id, batch_size), - ) - remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) - if len(remote_media_result) > 0: - self._insert_quarantine_changes_txn(txn, remote_media_result, True) + local_media_result: list[tuple[str | None, str]] = [] + remote_media_result: list[tuple[str | None, str]] = [] + + # It doesn't matter which order we do these in, as long as we do both of + # them. We skip a table once it's been fully processed so we don't keep + # running an empty query for it every iteration until the other finishes. + if not local_done: + txn.execute( + """ + SELECT NULL AS media_origin, media_id + FROM local_media_repository + WHERE quarantined_by IS NOT NULL + AND media_id > ? + ORDER BY media_id + LIMIT ? + """, + (last_local_media_id, batch_size), + ) + local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(local_media_result) > 0: + self._insert_quarantine_changes_txn(txn, local_media_result, True) + + # We page through `remote_media_cache` with a tuple comparison on + # `(media_origin, media_id)`. This matches a unique index, and so + # will a) page through all rows, and b) will be fast. + # + # Comparing the columns independently (e.g. `media_origin >= ? AND + # media_id > ?`) would incorrectly skip rows in a newly-reached + # origin whose media_id is <= the last processed media_id. + if not remote_done: + txn.execute( + """ + SELECT media_origin, media_id + FROM remote_media_cache + WHERE quarantined_by IS NOT NULL + AND (media_origin, media_id) > (?, ?) + ORDER BY media_origin, media_id + LIMIT ? + """, + (last_remote_origin, last_remote_media_id, batch_size), + ) + remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(remote_media_result) > 0: + self._insert_quarantine_changes_txn(txn, remote_media_result, True) + + # Carry the previous progress forward, then for each table advance its + # cursor to the last row we fetched, or mark it done if its query (which + # only runs while it isn't already done) came back empty. + new_progress = { + "last_local_media_id": last_local_media_id, + "last_remote_media_id": last_remote_media_id, + "last_remote_origin": last_remote_origin, + "local_done": local_done, + "remote_done": remote_done, + } + if local_media_result: + new_progress["last_local_media_id"] = local_media_result[-1][1] + else: + new_progress["local_done"] = True + if remote_media_result: + new_progress["last_remote_origin"] = remote_media_result[-1][0] + new_progress["last_remote_media_id"] = remote_media_result[-1][1] + else: + new_progress["remote_done"] = True self.db_pool.updates._background_update_progress_txn( txn, _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, - { - "last_local_media_id": local_media_result[-1][1] - if len(local_media_result) > 0 - else last_local_media_id, - "last_remote_media_id": remote_media_result[-1][1] - if len(remote_media_result) > 0 - else last_remote_media_id, - "last_remote_origin": remote_media_result[-1][0] - if len(remote_media_result) > 0 - else last_remote_origin, - }, + new_progress, ) return len(local_media_result) + len(remote_media_result) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 736f3e4c78..667ad1ace8 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -971,12 +971,22 @@ async def do_users_share_a_room_joined_or_invited( return {u for u, share_room in user_dict.items() if share_room} - async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of users who share a room with `user_id`""" + async def get_users_who_share_room_with_user( + self, user_id: str, excluded_rooms: AbstractSet[str] = frozenset() + ) -> set[str]: + """Returns the set of users who share a room with `user_id`. + + Args: + user_id: The user to find the co-occupants of. + excluded_rooms: Rooms which should not, on their own, count as a + shared room. + """ room_ids = await self.get_rooms_for_user(user_id) user_who_share_room: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue user_ids = await self.get_users_in_room(room_id) user_who_share_room.update(user_ids) diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 9a09c0f9b5..a5d6cd2548 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -96,6 +96,22 @@ def __init__( replaces_index="sliding_sync_membership_snapshots_user_id", ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_connections_last_used_ts_idx", + index_name="sliding_sync_connections_last_used_ts_idx", + table="sliding_sync_connections", + columns=("last_used_ts",), + where_clause="last_used_ts IS NOT NULL", + ) + + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_connection_lazy_members_conn_pos_idx", + index_name="sliding_sync_connection_lazy_members_conn_pos_idx", + table="sliding_sync_connection_lazy_members", + columns=("connection_position",), + where_clause="connection_position IS NOT NULL", + ) + if self.hs.config.worker.run_background_tasks: self.clock.looping_call( self.delete_old_sliding_sync_connections, @@ -186,15 +202,35 @@ def persist_per_connection_state_txn( # First we fetch (or create) the connection key associated with the # previous connection position. if previous_connection_position is not None: + lock_clause = "" + if isinstance(self.database_engine, PostgresEngine): + # Lock the sliding sync connection row for update upfront, + # to prevent deadlocks between concurrent transactions + # (which can retry again and again without making progress). + # + # (We don't need to explicitly lock in the other branch, + # where we re-create the connection, as that implies a lock + # anyway) + # + # Specifically, the statements seen to deadlock against + # each other were + # `INSERT INTO sliding_sync_connection_lazy_members` + # with conflicting tuples on + # "sliding_sync_connection_lazy_members_idx" UNIQUE, btree + # (connection_key, room_id, user_id) + # https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS + lock_clause = "FOR NO KEY UPDATE OF sliding_sync_connections" + # The `previous_connection_position` is a user-supplied value, so we # need to make sure that the one they supplied is actually theirs. - sql = """ + sql = f""" SELECT connection_key FROM sliding_sync_connection_positions INNER JOIN sliding_sync_connections USING (connection_key) WHERE connection_position = ? AND user_id = ? AND effective_device_id = ? AND conn_id = ? + {lock_clause} """ txn.execute( sql, (previous_connection_position, user_id, device_id, conn_id) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 6c5abc71ae..e3f16ef754 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -127,7 +127,7 @@ def _make_staging_area(txn: LoggingTransaction) -> None: sql = f""" CREATE TABLE IF NOT EXISTS {TEMP_TABLE}_position ( - position TEXT NOT NULL + position BIGINT NOT NULL ) """ txn.execute(sql) diff --git a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql index 78ba5129af..4abe0ccaf4 100644 --- a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql +++ b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql @@ -22,6 +22,8 @@ CREATE TABLE delayed_events ( state_key TEXT, origin_server_ts BIGINT, content bytea NOT NULL, + -- is_processed = TRUE means that the work of sending the delayed event has begun. + -- Once the send is complete, the delayed event is removed from this table. is_processed BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY (user_localpart, delay_id) ); diff --git a/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql index 747ba7a144..d8faac314d 100644 --- a/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql +++ b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql @@ -18,10 +18,3 @@ -- may want to either backfill this or delete all rows with a NULL value (and -- then make it NOT NULL). ALTER TABLE sliding_sync_connections ADD COLUMN last_used_ts BIGINT; - --- Note: We don't add an index on this column to allow HOT updates on PostgreSQL --- to reduce the cost of the updates to the column. c.f. --- https://www.postgresql.org/docs/current/storage-hot.html --- --- We do query this column directly to find expired connections, but we expect --- that to be an infrequent operation and a sequential scan should be fine. diff --git a/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql b/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql new file mode 100644 index 0000000000..6881b9419c --- /dev/null +++ b/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql @@ -0,0 +1,25 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- The `flag_existing_quarantined_media` background update (added in 94/03) originally +-- shipped with a broken remote media query that skipped some already-quarantined remote +-- media. Now that it's fixed, re-run the update from scratch so any media missed on the +-- first run gets flagged. +-- +-- We delete any existing row first: on servers where the update already completed the row +-- was removed, and on servers where it's still pending/mid-run this clears the stale +-- progress so the re-insert below starts cleanly (and avoids a primary key collision). +DELETE FROM background_updates WHERE update_name = 'flag_existing_quarantined_media'; + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9405, 'flag_existing_quarantined_media', '{}'); diff --git a/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql b/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql new file mode 100644 index 0000000000..427d255759 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `sliding_sync_connections(last_used_ts)` so that finding and +-- deleting expired connections (in `delete_old_sliding_sync_connections`) does +-- not require a sequential scan of the table. +-- +-- This is a partial index as we only ever query for rows with a non-NULL +-- `last_used_ts`. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9406, 'sliding_sync_connections_last_used_ts_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql b/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql new file mode 100644 index 0000000000..7c55763828 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql @@ -0,0 +1,24 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `sliding_sync_connection_lazy_members(connection_position)` +-- so that deleting from `sliding_sync_connection_positions` is efficient. This +-- is needed because `connection_position` has an `ON DELETE CASCADE` foreign key +-- constraint, and without this index Postgres has to sequentially scan the whole +-- table for each deleted position. +-- +-- This is a partial index as we only ever need to find rows with a non-NULL +-- `connection_position`. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9407, 'sliding_sync_connection_lazy_members_conn_pos_idx', '{}'); diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index a2d464b62b..53bb2868f8 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -13,7 +13,7 @@ from typing import Any, Iterator, Mapping from synapse.synapse_rust.room_versions import RoomVersion -from synapse.types import JsonDict, JsonMapping, StrSequence +from synapse.types import JsonDict, JsonMapping, Requester, StrSequence from synapse.util.duration import Duration class EventInternalMetadata: @@ -309,6 +309,173 @@ class Event: ``SynapseDuration`` representing the sticky duration. Otherwise returns ``None``.""" +class ThreadAggregation: + """The bundled thread summary for an event.""" + + def __init__( + self, + latest_event: Event, + count: int, + current_user_participated: bool, + ) -> None: ... + @property + def latest_event(self) -> Event: + """The latest event in the thread.""" + + @property + def count(self) -> int: + """The total number of events in the thread.""" + + @property + def current_user_participated(self) -> bool: + """Whether the requesting user has sent an event to the thread.""" + +class BundledAggregations: + """The bundled aggregations for an event. + + Some values require additional processing during serialization. + """ + + def __init__( + self, + references: JsonMapping | None = None, + replace: Event | None = None, + thread: ThreadAggregation | None = None, + ) -> None: ... + @property + def references(self) -> JsonMapping | None: ... + @property + def replace(self) -> Event | None: ... + @property + def thread(self) -> ThreadAggregation | None: ... + def __bool__(self) -> bool: ... + +class EventFormat: + """The format used to convert an event to the shape sent to clients.""" + + Raw: EventFormat + ClientV1: EventFormat + ClientV2: EventFormat + ClientV2WithoutRoomId: EventFormat + +class SerializeEventConfig: + """Configuration for serializing an event for clients.""" + + def __init__( + self, + *, + as_client_event: bool, + event_format: EventFormat, + requester: Requester | None, + event_field_allowlist: list[str] | None, + include_stripped_room_state: bool, + include_admin_metadata: bool, + msc4354_enabled: bool, + ) -> None: ... + @property + def as_client_event(self) -> bool: + """Whether to apply the client event format transform (v1/v2/raw). When + ``False``, the federation-format event is returned as-is.""" + + @property + def event_format(self) -> EventFormat: + """Which client event format variant to apply (only used when + ``as_client_event`` is ``True``).""" + + @property + def requester(self) -> Requester | None: + """The entity requesting the event. Used to gate sender-only fields such + as ``transaction_id`` and ``delay_id``.""" + + @property + def event_field_allowlist(self) -> list[str] | None: + """If set, only include these field paths in the output. An empty list + returns an empty event; ``None`` returns all fields. + + The fields can be "dotted" fields, e.g. ``content.body``.""" + + @property + def include_stripped_room_state(self) -> bool: + """Whether to include ``invite_room_state`` / ``knock_room_state`` in + ``unsigned``. These are stripped by default and only included for + specific endpoints (e.g. ``/sync`` invite/knock handling).""" + + @property + def include_admin_metadata(self) -> bool: + """When ``True``, add server-admin-only metadata to ``unsigned`` + (``io.element.synapse.soft_failed``, + ``io.element.synapse.policy_server_spammy``).""" + + @property + def msc4354_enabled(self) -> bool: + """Whether MSC4354 (sticky events) is enabled. When ``True``, the + remaining stickiness TTL is computed and added to ``unsigned``.""" + +def serialize_events( + events: list[tuple[Event, str | None]], + time_now_ms: int, + config: SerializeEventConfig, + *, + bundle_aggregations: Mapping[str, BundledAggregations] | None = None, + redaction_map: Mapping[str, Event] | None = None, + unsigned_additions: Mapping[str, JsonDict] | None = None, +) -> list[JsonDict]: + """Synchronously serialize a batch of events for clients using pre-fetched data. + + All DB/IO must already have been done by the caller; the keyword maps below + are all keyed by event ID and shared across the whole batch. + + Args: + events: The events to serialize, as `(event, membership)` pairs. + `membership` is the requesting user's membership at the time of the + event, injected into `unsigned.membership` (MSC4115). + time_now_ms: The current time in milliseconds. + config: The serialization config. + bundle_aggregations: Map from event_id to the `BundledAggregations` to + bundle into the event's `unsigned.m.relations`. + redaction_map: Map from redaction event_id to the redaction `Event`, + used to populate `unsigned.redacted_because` for redacted events. + unsigned_additions: Map from event_id to extra `unsigned` fields + contributed by module callbacks. + + Returns: + The serialized events, in the same order as `events`. + """ + +# The standalone `format_event_*` transforms below are a backwards +# compatibility hack: they have never been part of the module API and modules +# shouldn't be pulling them in, but some in the wild import them (via +# `synapse.events.utils`) anyway. They may be removed in the future; nothing +# in Synapse itself should use them. + +def format_event_raw(d: JsonDict) -> JsonDict: + """Return the event dict unchanged (federation format). + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v1(d: JsonDict) -> JsonDict: + """Apply the legacy `/events`-style v1 client format to `d` in place. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v2(d: JsonDict) -> JsonDict: + """Apply the `/sync`-style v2 client format to `d` in place. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: + """Apply the v2 client format to `d` in place, additionally stripping `room_id`. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + def redact_event(event: Event) -> Event: """Returns a pruned version of the given event, which removes all keys we don't know about or think could potentially be dodgy. diff --git a/synapse/synapse_rust/handlers.pyi b/synapse/synapse_rust/handlers.pyi new file mode 100644 index 0000000000..4c4f71b7bd --- /dev/null +++ b/synapse/synapse_rust/handlers.pyi @@ -0,0 +1,35 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import TYPE_CHECKING, Optional + +from twisted.internet.defer import Deferred + +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +class VersionsHandler: + def get_versions(self, user_id: Optional[str] = None) -> Deferred[JsonDict]: + """ + Assemble a `/versions` response. + + The returned deferred follows Synapse logcontext rules. + """ + +class RustHandlers: + """The collection of Rust-implemented request handlers.""" + + def __init__(self, homeserver: "HomeServer") -> None: ... + @property + def versions(self) -> VersionsHandler: ... diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index a6fc806701..b42893e39d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -37,6 +37,7 @@ MutableMapping, NoReturn, Optional, + Sequence, TypedDict, TypeVar, Union, @@ -102,6 +103,47 @@ # A JSON-serialisable object. JsonSerializable = object +StrictJsonValue = Union[ + None, + bool, + int, + float, + str, + "StrictJsonList", + "StrictJsonDict", + "StrictJsonSequence", + "StrictJsonMapping", +] +""" +Type that represents any valid JSON value, recursively. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonValue`. + +Can also represent immutable mapping and tuple types. +(Not sure if we would be better splitting them out.) +""" + +StrictJsonList = list["StrictJsonValue"] +""" +Type that represents a list of any valid JSON value. +""" + +StrictJsonDict = dict[str, "StrictJsonValue"] +""" +Type that represents a dict with string keys (as per JSON) and values of any +valid JSON type. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonDict`. +""" + +StrictJsonSequence = Sequence["StrictJsonValue"] +""" +Like `StrictJsonList` but using a `Sequence` as the collection type. +""" + +StrictJsonMapping = Mapping[str, "StrictJsonValue"] +""" +Like `StrictJsonDict` but using `Mapping` as the collection type. +""" + # Collection[str] that does not include str itself; str being a Sequence[str] # is very misleading and results in bugs. # diff --git a/synapse/util/clock.py b/synapse/util/clock.py index 7232a1331c..8c05675732 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -62,6 +62,19 @@ logging.setLoggerClass(original_logger_class) +CLOCK_SCHEDULE_EPSILON = Duration(microseconds=1) +""" +The smallest value we can use that will schedule tasks "as soon as possible", while +still allowing other tasks to run between runs. + +This should be a non-zero value as the Twisted Reactor API does not specify how calls +get scheduled. If we used `0`, a weird reactor implementation could run it immediately +or run it any order with the other calls that are scheduled now. + +We want the semantics of run this in the "next reactor iteration". +""" + + def _try_wakeup_deferred(d: Deferred) -> None: """Try to wake up a deferred, but ignore any exceptions raised by the callback. This is useful when we want to wake up a deferred that may have diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 0f5d1c7338..20d314cb68 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -76,6 +76,13 @@ async def shutdown() -> None: self.get_success(shutdown()) + # XXX: There can be a few already dispatched database queries (from normal + # background tasks in Synapse) and the threadless `ThreadPool` that we use in + # tests uses *untracked* clock calls to pass database results back so `shutdown` + # doesn't cancel those calls. This is a quirk of our test infrastructure + # (threadless `ThreadPool`) so this kind of "hack" is fine. + self.reactor.advance(0) + # Cleanup the internal reference in our test case del self.hs @@ -106,7 +113,7 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. store = weakref.proxy(self.hs.get_datastores().main) - self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) + self.get_success(store.db_pool.updates.do_next_background_update(False)) hs_ref = weakref.ref(self.hs) @@ -127,6 +134,13 @@ async def shutdown() -> None: self.get_success(shutdown()) + # XXX: There can be a few already dispatched database queries (from normal + # background tasks in Synapse) and the threadless `ThreadPool` that we use in + # tests uses *untracked* clock calls to pass database results back so `shutdown` + # doesn't cancel those calls. This is a quirk of our test infrastructure + # (threadless `ThreadPool`) so this kind of "hack" is fine. + self.reactor.advance(0) + # Cleanup the internal reference in our test case del self.hs diff --git a/tests/config/test_experimental.py b/tests/config/test_experimental.py new file mode 100644 index 0000000000..0fede98ee9 --- /dev/null +++ b/tests/config/test_experimental.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +import yaml +from parameterized import parameterized + +from synapse.config._base import RootConfig +from synapse.config.experimental import ExperimentalConfig +from synapse.config.homeserver import HomeServerConfig +from synapse.types import JsonDict + +from tests import unittest + + +class ExperimentalConfigTestCase(unittest.TestCase): + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc3575_enabled": True, + } + }, + ], + [ + "multi", + { + "experimental_features": { + "msc3575_enabled": True, + "msc3030_enabled": True, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + + +def _read_config(config_values: JsonDict) -> None: + ExperimentalConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values, + allow_secrets_in_config=False, + ) diff --git a/tests/config/test_load.py b/tests/config/test_load.py index c1b787346e..8d94390acf 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -149,8 +149,6 @@ def test_depreciated_identity_server_flag_throws_error(self) -> None: "recaptcha_public_key_path: /does/not/exist", "form_secret_path: /does/not/exist", "worker_replication_secret_path: /does/not/exist", - "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", - "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] * (hiredis is not None), ] @@ -192,14 +190,6 @@ def test_secret_files_missing(self, config_str: str) -> None: "worker_replication_secret_path: {}", lambda c: c.worker.worker_replication_secret.encode("utf-8"), ), - ( - "experimental_features:\n msc3861:\n client_secret_path: {}", - lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), - ), - ( - "experimental_features:\n msc3861:\n admin_token_path: {}", - lambda c: c.experimental.msc3861.admin_token().encode("utf-8"), - ), *[ ( "redis:\n enabled: true\n password_path: {}", @@ -232,29 +222,6 @@ def test_secret_files_existing( "recaptcha_public_key: ¬53C237", "form_secret: 53C237", "worker_replication_secret: 53C237", - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " client_secret: 53C237" - ] - * (authlib is not None), - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " client_auth_method: private_key_jwt\n" - ' jwk: {{"mock": "mock"}}' - ] - * (authlib is not None), - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " admin_token: 53C237\n" - " client_secret_path: {secret_file}" - ] - * (authlib is not None), *["redis:\n enabled: true\n password: 53C237"] * (hiredis is not None), ] ) @@ -304,15 +271,6 @@ def test_no_secrets_in_config_but_in_files(self) -> None: f"recaptcha_public_key_path: {secret_file.name}", f"form_secret_path: {secret_file.name}", f"worker_replication_secret_path: {secret_file.name}", - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - f" admin_token_path: {secret_file.name}\n" - f" client_secret_path: {secret_file.name}\n" - # f" jwk_path: {secret_file.name}" - ] - * (authlib is not None), *[f"redis:\n enabled: true\n password_path: {secret_file.name}"] * (hiredis is not None), ] diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index e0698764a1..17fb3a3410 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -54,234 +54,6 @@ def __init__(self, config: None, api: ModuleApi): ) -@skip_unless(HAS_AUTHLIB, "requires authlib") -class MSC3861OAuthDelegation(TestCase): - """Test that the Homeserver fails to initialize if the config is invalid.""" - - def setUp(self) -> None: - self.config_dict: JsonDict = { - **default_config(server_name="test"), - "public_baseurl": BASE_URL, - "enable_registration": False, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - } - }, - } - - def parse_config(self) -> HomeServerConfig: - config = HomeServerConfig() - config.parse_config_dict(self.config_dict, "", "") - return config - - def test_client_secret_post_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_post", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_post_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_post", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_client_secret_basic_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_basic", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_basic_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_basic", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_client_secret_jwt_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_jwt", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_jwt_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_jwt", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_invalid_client_auth_method(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="invalid", - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_private_key_jwt_requires_jwk(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="private_key_jwt", - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_private_key_jwt_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="private_key_jwt", - jwk={ - "p": "-frVdP_tZ-J_nIR6HNMDq1N7aunwm51nAqNnhqIyuA8ikx7LlQED1tt2LD3YEvYyW8nxE2V95HlCRZXQPMiRJBFOsbmYkzl2t-MpavTaObB_fct_JqcRtdXddg4-_ihdjRDwUOreq_dpWh6MIKsC3UyekfkHmeEJg5YpOTL15j8", - "kty": "RSA", - "q": "oFw-Enr_YozQB1ab-kawn4jY3yHi8B1nSmYT0s8oTCflrmps5BFJfCkHL5ij3iY15z0o2m0N-jjB1oSJ98O4RayEEYNQlHnTNTl0kRIWzpoqblHUIxVcahIpP_xTovBJzwi8XXoLGqHOOMA-r40LSyVgP2Ut8D9qBwV6_UfT0LU", - "d": "WFkDPYo4b4LIS64D_QtQfGGuAObPvc3HFfp9VZXyq3SJR58XZRHE0jqtlEMNHhOTgbMYS3w8nxPQ_qVzY-5hs4fIanwvB64mAoOGl0qMHO65DTD_WsGFwzYClJPBVniavkLE2Hmpu8IGe6lGliN8vREC6_4t69liY-XcN_ECboVtC2behKkLOEASOIMuS7YcKAhTJFJwkl1dqDlliEn5A4u4xy7nuWQz3juB1OFdKlwGA5dfhDNglhoLIwNnkLsUPPFO-WB5ZNEW35xxHOToxj4bShvDuanVA6mJPtTKjz0XibjB36bj_nF_j7EtbE2PdGJ2KevAVgElR4lqS4ISgQ", - "e": "AQAB", - "kid": "test", - "qi": "cPfNk8l8W5exVNNea4d7QZZ8Qr8LgHghypYAxz8PQh1fNa8Ya1SNUDVzC2iHHhszxxA0vB9C7jGze8dBrvnzWYF1XvQcqNIVVgHhD57R1Nm3dj2NoHIKe0Cu4bCUtP8xnZQUN4KX7y4IIcgRcBWG1hT6DEYZ4BxqicnBXXNXAUI", - "dp": "dKlMHvslV1sMBQaKWpNb3gPq0B13TZhqr3-E2_8sPlvJ3fD8P4CmwwnOn50JDuhY3h9jY5L06sBwXjspYISVv8hX-ndMLkEeF3lrJeA5S70D8rgakfZcPIkffm3tlf1Ok3v5OzoxSv3-67Df4osMniyYwDUBCB5Oq1tTx77xpU8", - "dq": "S4ooU1xNYYcjl9FcuJEEMqKsRrAXzzSKq6laPTwIp5dDwt2vXeAm1a4eDHXC-6rUSZGt5PbqVqzV4s-cjnJMI8YYkIdjNg4NSE1Ac_YpeDl3M3Colb5CQlU7yUB7xY2bt0NOOFp9UJZYJrOo09mFMGjy5eorsbitoZEbVqS3SuE", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - }, - ) - self.parse_config() - - def test_registration_cannot_be_enabled(self) -> None: - self.config_dict["enable_registration"] = True - with self.assertRaises(ConfigError): - self.parse_config() - - def test_user_consent_cannot_be_enabled(self) -> None: - tmpdir = self.mktemp() - os.mkdir(tmpdir) - self.config_dict["user_consent"] = { - "require_at_registration": True, - "version": "1", - "template_dir": tmpdir, - "server_notice_content": { - "msgtype": "m.text", - "body": "foo", - }, - } - with self.assertRaises(ConfigError): - self.parse_config() - - def test_password_config_cannot_be_enabled(self) -> None: - self.config_dict["password_config"] = {"enabled": True} - with self.assertRaises(ConfigError): - self.parse_config() - - def test_oidc_sso_cannot_be_enabled(self) -> None: - self.config_dict["oidc_providers"] = [ - { - "idp_id": "microsoft", - "idp_name": "Microsoft", - "issuer": "https://login.microsoftonline.com//v2.0", - "client_id": "", - "client_secret": "", - "scopes": ["openid", "profile"], - "authorization_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/authorize", - "token_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/token", - "userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo", - } - ] - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_cas_sso_cannot_be_enabled(self) -> None: - self.config_dict["cas_config"] = { - "enabled": True, - "server_url": "https://cas-server.com", - "displayname_attribute": "name", - "required_attributes": {"userGroup": "staff", "department": "None"}, - } - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_auth_providers_cannot_be_enabled(self) -> None: - self.config_dict["modules"] = [ - { - "module": f"{__name__}.{CustomAuthModule.__qualname__}", - "config": {}, - } - ] - - # This requires actually setting up an HS, as the module will be run on setup, - # which should raise as the module tries to register an auth provider - config = self.parse_config() - reactor, clock = get_clock() - with self.assertRaises(ConfigError): - setup_test_homeserver( - cleanup_func=self.addCleanup, - config=config, - reactor=reactor, - clock=clock, - ) - - def test_jwt_auth_cannot_be_enabled(self) -> None: - self.config_dict["jwt_config"] = { - "enabled": True, - "secret": "my-secret-token", - "algorithm": "HS256", - } - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_login_via_existing_session_cannot_be_enabled(self) -> None: - self.config_dict["login_via_existing_session"] = {"enabled": True} - with self.assertRaises(ConfigError): - self.parse_config() - - def test_captcha_cannot_be_enabled(self) -> None: - self.config_dict.update( - enable_registration_captcha=True, - recaptcha_public_key="test", - recaptcha_private_key="test", - ) - with self.assertRaises(ConfigError): - self.parse_config() - - def test_refreshable_tokens_cannot_be_enabled(self) -> None: - self.config_dict.update( - refresh_token_lifetime="24h", - refreshable_access_token_lifetime="10m", - nonrefreshable_access_token_lifetime="24h", - ) - with self.assertRaises(ConfigError): - self.parse_config() - - def test_session_lifetime_cannot_be_set(self) -> None: - self.config_dict["session_lifetime"] = "24h" - with self.assertRaises(ConfigError): - self.parse_config() - - def test_enable_3pid_changes_cannot_be_enabled(self) -> None: - self.config_dict["enable_3pid_changes"] = True - with self.assertRaises(ConfigError): - self.parse_config() - - class MasAuthDelegation(TestCase): """Test that the Homeserver fails to initialize if the config is invalid.""" diff --git a/tests/config/test_server.py b/tests/config/test_server.py index d3c59ae14c..f718e20a2a 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -18,11 +18,16 @@ # # + +from typing import Any + import yaml +from parameterized import parameterized from synapse.config._base import ConfigError, RootConfig from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved +from synapse.types import JsonDict from tests import unittest @@ -189,6 +194,77 @@ def test_listeners_set_correctly_open_private_ports_true(self) -> None: self.assertEqual(conf["listeners"], expected_listeners) + def test_max_delayed_events_enforces_positive(self) -> None: + """ + Test that the configured maximum allowed delay must be a positive value if set, + as per documentation + """ + + def generate_config(value: int) -> JsonDict: + return {"max_event_delay_duration": value} + + _read_config(generate_config(1)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(0)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(-1)) + + def test_max_delayed_events_per_user_enforces_non_negative_int(self) -> None: + """ + Test that the configured maximum number of delayed events must be a non-negative value if set, + as a negative limit can never be satisfied + """ + + def generate_config(value: Any) -> JsonDict: + return { + "experimental_features": {"msc4140_max_delayed_events_per_user": value} + } + + for allowed_value in (0, 1): + _read_config(generate_config(allowed_value)) + + for disallowed_value in (-1, 0.5): + with self.assertRaises(ConfigError): + _read_config(generate_config(disallowed_value)) + + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc4140_max_delayed_events_per_user": 3, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + + +def _read_config(config_values: JsonDict) -> None: + ServerConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values + ) + class GenerateIpSetTestCase(unittest.TestCase): def test_empty(self) -> None: diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 6bc935f272..01561b0d41 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -499,7 +499,7 @@ async def get_json(destination: str, path: str, **kwargs: Any) -> JsonDict: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) # we expect it to be encoded as canonical json *before* it hits the db @@ -614,7 +614,7 @@ def test_get_keys_from_perspectives(self) -> None: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) self.assertEqual(res.key_json, canonicaljson.encode_canonical_json(response)) @@ -732,7 +732,7 @@ def test_get_perspectives_own_key(self) -> None: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) self.assertEqual(res.key_json, canonicaljson.encode_canonical_json(response)) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 8f78ae4944..8435d6308a 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -22,24 +22,22 @@ import unittest as stdlib_unittest from typing import TYPE_CHECKING, Any, Mapping -from parameterized import parameterized - from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, - SerializeEventConfig, - _split_field, clone_event, copy_and_fixup_power_levels_contents, + format_event_for_client_v1, + format_event_for_client_v2, + format_event_for_client_v2_without_room_id, format_event_raw, - make_config_for_admin, maybe_upsert_event_field, prune_event, ) -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict from synapse.util.frozenutils import freeze from tests.test_utils.event_builders import make_test_event @@ -665,9 +663,11 @@ def serialize( self._event_serializer.serialize_event( FilteredEvent(event=ev, membership=None), 1479807801915, - config=SerializeEventConfig( - only_event_fields=fields, - include_admin_metadata=include_admin_metadata, + config=self.get_success( + self._event_serializer.create_config( + event_field_allowlist=fields, + include_admin_metadata=include_admin_metadata, + ) ), redaction_map=redaction_map, ) @@ -788,13 +788,19 @@ def test_event_fields_all_fields_if_empty(self) -> None: def test_event_fields_fail_if_fields_not_str(self) -> None: with self.assertRaises(TypeError): - SerializeEventConfig( - only_event_fields=["room_id", 4], # type: ignore[list-item] + self.get_success_or_raise( + self._event_serializer.create_config( + event_field_allowlist=["room_id", 4], # type: ignore[list-item] + ) ) def test_default_serialize_config_excludes_admin_metadata(self) -> None: # We just really don't want this to be set to True accidentally - self.assertFalse(SerializeEventConfig().include_admin_metadata) + self.assertFalse( + self.get_success( + self._event_serializer.create_config() + ).include_admin_metadata + ) def test_event_flagged_for_admins(self) -> None: # Default behaviour should be *not* to include it @@ -875,34 +881,10 @@ def test_event_flagged_for_admins(self) -> None: }, ) - def test_make_serialize_config_for_admin_retains_other_fields(self) -> None: - non_default_config = SerializeEventConfig( - include_admin_metadata=False, # should be True in a moment - as_client_event=False, # default True - event_format=format_event_raw, # default format_event_for_client_v1 - requester=create_requester("@example:example.org"), # default None - only_event_fields=["foo"], # default None - include_stripped_room_state=True, # default False - ) - admin_config = make_config_for_admin(non_default_config) - self.assertEqual( - admin_config.as_client_event, non_default_config.as_client_event - ) - self.assertEqual(admin_config.event_format, non_default_config.event_format) - self.assertEqual(admin_config.requester, non_default_config.requester) - self.assertEqual( - admin_config.only_event_fields, non_default_config.only_event_fields - ) - self.assertEqual( - admin_config.include_stripped_room_state, - admin_config.include_stripped_room_state, - ) - self.assertTrue(admin_config.include_admin_metadata) - def test_redacted_because_is_filtered_out(self) -> None: """If an event's unsigned dict has a `redacted_by` field, then the `redacted_because` should be filtered out if not specified in - `only_event_fields`.""" + `event_field_allowlist`.""" redaction_id = "$redaction_event_id" @@ -1021,38 +1003,97 @@ def test_invalid_nesting_raises_type_error(self) -> None: copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] -class SplitFieldTestCase(stdlib_unittest.TestCase): - @parameterized.expand( - [ - # A field with no dots. - ["m", ["m"]], - # Simple dotted fields. - ["m.foo", ["m", "foo"]], - ["m.foo.bar", ["m", "foo", "bar"]], - # Backslash is used as an escape character. - [r"m\.foo", ["m.foo"]], - [r"m\\.foo", ["m\\", "foo"]], - [r"m\\\.foo", [r"m\.foo"]], - [r"m\\\\.foo", ["m\\\\", "foo"]], - [r"m\foo", [r"m\foo"]], - [r"m\\foo", [r"m\foo"]], - [r"m\\\foo", [r"m\\foo"]], - [r"m\\\\foo", [r"m\\foo"]], - # Ensure that escapes at the end don't cause issues. - ["m.foo\\", ["m", "foo\\"]], - ["m.foo\\", ["m", "foo\\"]], - [r"m.foo\.", ["m", "foo."]], - [r"m.foo\\.", ["m", "foo\\", ""]], - [r"m.foo\\\.", ["m", r"foo\."]], - # Empty parts (corresponding to properties which are an empty string) are allowed. - [".m", ["", "m"]], - ["..m", ["", "", "m"]], - ["m.", ["m", ""]], - ["m..", ["m", "", ""]], - ["m..foo", ["m", "", "foo"]], - # Invalid escape sequences. - [r"\m", [r"\m"]], - ] - ) - def test_split_field(self, input: str, expected: str) -> None: - self.assertEqual(_split_field(input), expected) +class FormatEventForClientTestCase(stdlib_unittest.TestCase): + """Tests for the standalone `format_event_*` transforms. + + These are Rust reimplementations kept purely as a backwards compatibility + hack for modules in the wild that import them from `synapse.events.utils` + (they were never part of the module API, and nothing in Synapse itself uses + them); like the original Python implementations they must mutate the dict + in place and return it. + """ + + def make_event(self) -> JsonDict: + return { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "auth_events": [], + "prev_events": [], + "hashes": {}, + "signatures": {}, + "depth": 5, + "origin": "test", + "prev_state": [], + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + } + + def test_raw(self) -> None: + event_dict = self.make_event() + result = format_event_raw(event_dict) + self.assertIs(result, event_dict) + self.assertEqual(result, self.make_event()) + + def test_v2_drops_federation_keys(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v2(event_dict) + self.assertIs(result, event_dict) + self.assertEqual( + result, + { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + }, + ) + + def test_v2_without_room_id(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v2_without_room_id(event_dict) + self.assertIs(result, event_dict) + self.assertNotIn("room_id", result) + + def test_v1_copies_unsigned_keys(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v1(event_dict) + self.assertIs(result, event_dict) + self.assertEqual( + result, + { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "user_id": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "age": 100, + "replaces_state": "$old", + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + }, + ) + + def test_v1_no_sender(self) -> None: + event_dict = self.make_event() + del event_dict["sender"] + result = format_event_for_client_v1(event_dict) + self.assertNotIn("user_id", result) + + def test_non_json_values_pass_through(self) -> None: + # The transforms only move keys around; values that aren't + # JSON-serializable must survive untouched. + marker = object() + event_dict = { + "sender": "@sender:test", + "auth_events": marker, + "content": marker, + "unsigned": {"age": marker}, + } + result = format_event_for_client_v1(event_dict) + self.assertIs(result["content"], marker) + self.assertIs(result["age"], marker) + self.assertNotIn("auth_events", result) diff --git a/tests/federation/_remote_join.py b/tests/federation/_remote_join.py new file mode 100644 index 0000000000..94fa44e200 --- /dev/null +++ b/tests/federation/_remote_join.py @@ -0,0 +1,354 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import urllib.parse +from http import HTTPStatus +from typing import Callable, TypeVar +from unittest.mock import Mock + +import attr + +from synapse.api.constants import EventContentFields, EventTypes, Membership +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase +from synapse.events.utils import strip_event +from synapse.federation.transport.client import SendJoinResponse +from synapse.http.matrixfederationclient import ByteParser +from synapse.http.types import QueryParams +from synapse.types import JsonDict + +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event +from tests.unittest import FederatingHomeserverTestCase + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, auto_attribs=True) +class RemoteStateEvent: + """ + A state event to insert into the remote room `/send_join` response. + """ + + type: str + state_key: str + content: JsonDict + # Defaults to the remote room creator. + sender: str | None = None + + +class RemoteJoinHelper: + """ + Helps a `FederatingHomeserverTestCase` join a remote (non-resident) room + over federation. + + Spiritually inspired by `test_federation_out_of_band_membership.py` + but in a more reusable form. + + Constructor Args: + test_case: the current `FederatingHomeserverTestCase` + federation_http_client: Mocked form of the main test homeserver's federation HTTP client; + should have method slots for `get_json` and `put_json`. + remote_creator_user_id: User ID of the remote user creating the room + room_version: Desired room version + create_content: Desired `content` of the `m.room.create` event + state_events: Extra state events to create in the mock room + (They will be made available in the `/send_join` response) + + Usage: + + helper = RemoteJoinHelper( + self, + create_content={"predecessor": {"room_id": old_room_id, ...}}, + ) + # helper.room_id is available now; you can e.g. point a tombstone at it. + helper.join(local_user_id, local_user_tok) + """ + + room_id: str + """ + The room ID of the mock remote room that will be joined. + """ + + def __init__( + self, + test_case: FederatingHomeserverTestCase, + federation_http_client: Mock, + *, + remote_creator_user_id: str | None = None, + room_version: RoomVersion = RoomVersions.V10, + create_content: JsonDict | None = None, + state_events: list[RemoteStateEvent] | None = None, + ) -> None: + if remote_creator_user_id is None: + remote_creator_user_id = f"@remote-user:{test_case.OTHER_SERVER_NAME}" + + self._test_case = test_case + self._federation_http_client = federation_http_client + self._remote_creator_user_id = remote_creator_user_id + self._room_version = room_version + + # 1. Create the room creation event + create_content_full: JsonDict = { + EventContentFields.ROOM_CREATOR: remote_creator_user_id, + EventContentFields.ROOM_VERSION: room_version.identifier, + } + if create_content is not None: + create_content_full.update(create_content) + + create_event_dict: JsonDict = { + "sender": remote_creator_user_id, + "depth": 1, + "origin_server_ts": 1, + "type": EventTypes.Create, + "state_key": "", + "content": create_content_full, + "auth_events": [], + "prev_events": [], + } + if not room_version.msc4291_room_ids_as_hashes: + # For room versions that _don't_ derive the room ID from the content, + # we need to set our own. + # We could consider exposing a parameter to allow varying the localpart + # (and perturb the create event for hashes-as-room-ID rooms) + create_event_dict["room_id"] = f"!remote-room:{test_case.OTHER_SERVER_NAME}" + + room_create_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + create_event_dict, + ), + room_version=room_version, + ) + + self.room_id = room_create_event.room_id + + # 2. Create the room creator's membership event + creator_membership_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": remote_creator_user_id, + "depth": 2, + "origin_server_ts": 2, + "type": EventTypes.Member, + "state_key": remote_creator_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [room_create_event.event_id], + "prev_events": [room_create_event.event_id], + } + ), + room_version=room_version, + ) + + # 3. Create requested extra state events (in a linear chain from the membership) + extra_state_events: list[EventBase] = [] + prev_event = creator_membership_event + depth = 3 + for spec in state_events or []: + sender = spec.sender or remote_creator_user_id + event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": sender, + "depth": depth, + "origin_server_ts": depth, + "type": spec.type, + "state_key": spec.state_key, + "content": spec.content, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [prev_event.event_id], + } + ), + room_version=room_version, + ) + extra_state_events.append(event) + prev_event = event + depth += 1 + + self._room_create_event = room_create_event + self._creator_membership_event = creator_membership_event + self._extra_state_events = extra_state_events + + def join(self, local_user_id: str, local_user_tok: str) -> None: + """ + Invite `local_user_id` and perform the federation join dance. + """ + remote_room_id = self.room_id + room_version = self._room_version + + room_create_event = self._room_create_event + creator_membership_event = self._creator_membership_event + extra_events = self._extra_state_events + + # 1. Create an invite event and make it appear on the 'real' homeserver + depth = 3 + len(extra_events) + + invite_membership_event = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": self._remote_creator_user_id, + "depth": depth, + "origin_server_ts": depth, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [ + extra_events[-1].event_id + if extra_events + else creator_membership_event.event_id + ], + } + ), + room_version=room_version, + ) + + channel = self._test_case.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{invite_membership_event.event_id}", + content={ + "event": invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event(room_create_event), + ], + "room_version": room_version.identifier, + }, + ) + assert channel.code == HTTPStatus.OK, channel.json_body + + # 2. Mock `/make_join` and `/send_join`. + # Start by creating a join membership event. + join_membership_event_template = make_test_event( + { + "room_id": remote_room_id, + "sender": local_user_id, + "depth": depth + 1, + "origin_server_ts": depth + 1, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + room_create_event.event_id, + invite_membership_event.event_id, + ], + "prev_events": [invite_membership_event.event_id], + }, + room_version=room_version, + ) + + T = TypeVar("T") + + async def _get_json( + destination: str, + path: str, + args: QueryParams | None = None, + retry_on_dns_fail: bool = True, + timeout: int | None = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + ) -> JsonDict | T: + make_join_path = ( + f"/_matrix/federation/v1/make_join/" + f"{urllib.parse.quote_plus(remote_room_id)}/{urllib.parse.quote_plus(local_user_id)}" + ) + if path == make_join_path: + return { + "event": join_membership_event_template.get_pdu_json(), + "room_version": room_version.identifier, + } + raise NotImplementedError( + "We have not mocked a response for `get_json(...)` for the following endpoint yet: " + + f"{destination}{path}" + ) + + self._federation_http_client.get_json.side_effect = _get_json + + send_join_state = [ + room_create_event, + creator_membership_event, + *extra_events, + invite_membership_event, + ] + + async def _put_json( + destination: str, + path: str, + args: QueryParams | None = None, + data: JsonDict | None = None, + json_data_callback: Callable[[], JsonDict] | None = None, + long_retries: bool = False, + timeout: int | None = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + backoff_on_all_error_codes: bool = False, + ) -> JsonDict | T | SendJoinResponse: + if ( + path.startswith( + f"/_matrix/federation/v2/send_join/{urllib.parse.quote_plus(remote_room_id)}/" + ) + and data is not None + and data.get("type") == EventTypes.Member + and data.get("state_key") == local_user_id + and parser is not None + ): + # As the remote server, sign the join event before returning it. + join_membership_event_signed = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server(data), + room_version=room_version, + ) + return SendJoinResponse( + auth_events=[ + room_create_event, + invite_membership_event, + ], + state=send_join_state, + event_dict=join_membership_event_signed.get_pdu_json(), + event=join_membership_event_signed, + members_omitted=False, + servers_in_room=[ + self._test_case.OTHER_SERVER_NAME, + ], + ) + + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + # Just acknowledge everything. + return { + make_test_pdu_event(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self._federation_http_client.put_json.side_effect = _put_json + + # 3. Issue the client-server API request to join the room + self._test_case.helper.join(remote_room_id, local_user_id, tok=local_user_tok) + + # 4. Reset mocks + self._federation_http_client.get_json.side_effect = None + self._federation_http_client.put_json.side_effect = None diff --git a/tests/federation/test_federation_join_upgraded_room.py b/tests/federation/test_federation_join_upgraded_room.py new file mode 100644 index 0000000000..4557cf1b55 --- /dev/null +++ b/tests/federation/test_federation_join_upgraded_room.py @@ -0,0 +1,308 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from unittest.mock import Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, RoomAlias +from synapse.util.clock import Clock + +from tests import unittest +from tests.federation._remote_join import RemoteJoinHelper + +logger = logging.getLogger(__name__) + + +def _predecessor(room_id: str) -> JsonDict: + """`create_content` for a remote room that claims `room_id` as its predecessor.""" + return { + "predecessor": { + "room_id": room_id, + # inert dummy + "event_id": "$some_tombstone_event:test", + } + } + + +class FederationJoinUpgradedRoomTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self._federation_http_client = Mock( + # The problem with using `spec=MatrixFederationHttpClient` here is that it + # requires everything to be mocked which is a lot of work that I don't want + # to do when the code only uses a few methods (`get_json` and `put_json`). + ) + return self.setup_test_homeserver( + federation_http_client=self._federation_http_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + + def _room_is_public(self, room_id: str) -> bool: + """`is_public` flag from the `rooms` table (asserts the row exists).""" + room = self.get_success(self.store.get_room(room_id)) + assert room is not None, f"no rooms row for {room_id}" + is_public, _ = room + return is_public + + def test_alias_transferred_on_federation_join_to_upgraded_room(self) -> None: + """ + Tests that joining an upgraded room over federation, + where the predecessor has a valid corresponding tombstone, + should transfer all room aliases from the old room to the new room. + + The old room is set up with a couple of local aliases and a tombstone + event pointing at the new (remote) room. + After the federation join, all room aliases should have been + transferred from the old room to the new room. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and point 2 room aliases at it + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias1 = RoomAlias.from_string("#old_room:test") + alias2 = RoomAlias.from_string("#old_room_alt:test") + for alias in (alias1, alias2): + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The new room should have acquired the aliases... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, [alias1.to_string(), alias2.to_string()]) + + # ...and the old room therefore must have given them up + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertEqual(old_aliases, []) + + def test_room_directory_visibility_transferred(self) -> None: + """ + On a valid federation join of an upgraded room, the room directory public + flag should move from the old room to the new room. + + A public old room becomes private (so people don't accidentally join it) + and the newly-joined room is marked public. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.get_success(self.store.set_room_is_public(old_room_id, True)) + self.assertTrue(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The room directory publicity should have shifted to the new room + # (So users don't accidentally join the old room from the directory) + self.assertFalse(self._room_is_public(old_room_id)) + self.assertTrue(self._room_is_public(join_helper.room_id)) + + def test_room_directory_visibility_not_transferred_for_private_room(self) -> None: + """ + On a valid federation join of an upgraded room, a private old room should + leave the room directory visibility untouched for both the old and new + rooms (both remain private). + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.assertFalse(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + self.assertFalse(self._room_is_public(old_room_id)) + self.assertFalse(self._room_is_public(join_helper.room_id)) + + def test_no_transfer_when_predecessor_room_has_no_tombstone(self) -> None: + """ + Tests that when joining a remote room over federation, + if the room has an illegitimate predecessor (a predecessor pointing + to a room that does not have a corresponding tombstone to vouch for it + as the successor), room aliases are not transferred. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + + # Notably, we do NOT set up a tombstone in the 'old' room. + + # Do the remote room join dance + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) + + def test_no_transfer_when_tombstone_does_not_match(self) -> None: + """ + A predecessor room whose tombstone points to a different room than the + one being joined must not trigger an alias transfer. + + The tombstone's `replacement_room` must match the joined room for the + upgrade link to be considered valid. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Tombstone points at a _different_, room. + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": "!the_real_replacement_room:example.com"}, + tok=local_user_tok, + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index ced98a8b00..2a9c2f0fc4 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -37,6 +37,7 @@ from synapse.storage.databases.main.events_worker import EventMetadata from synapse.types import JsonDict, ReadReceipt from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.unittest import HomeserverTestCase @@ -517,6 +518,24 @@ async def record_transaction( self.edus.extend(data["edus"]) return {} + def wait_for_device_list_updates_to_be_sent(self) -> None: + """ + Wait for the device list update EDU's to get pushed out over federation + + For example, each login does a fire-and-forget (`LoginRestServlet` -> + `register_device` -> `notify_device_update` -> `handle_new_device_update` -> + `send_device_messages(hosts, immediate=False)`) which adds to the + `_DestinationWakeupQueue` which has a background process that sends depending on + how `federation_rr_transactions_per_room_per_second` is configured. + + The default `federation_rr_transactions_per_room_per_second` is `50` (1s/50 -> + 0.02s) + """ + self.reactor.advance( + 1.0 + / self.hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + def test_send_device_updates(self) -> None: """Basic case: each device update should result in an EDU""" # create a device @@ -527,9 +546,7 @@ def test_send_device_updates(self) -> None: self.assertEqual(len(self.edus), 1) stream_id = self.check_device_update_edu(self.edus.pop(0), u1, "D1", None) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # a second call should produce no new device EDUs self.get_success( @@ -568,7 +585,7 @@ def test_dont_send_device_updates_for_remote_users(self) -> None: ) ) - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # We shouldn't see an EDU for that update self.assertEqual(self.edus, []) @@ -590,6 +607,8 @@ def test_upload_signatures(self) -> None: self.login(u1, "pass", device_id="D1") self.login(u1, "pass", device_id="D2") + self.wait_for_device_list_updates_to_be_sent() + # expect two edus self.assertEqual(len(self.edus), 2) stream_id: int | None = None @@ -600,9 +619,7 @@ def test_upload_signatures(self) -> None: device1_signing_key = self.generate_and_upload_device_signing_key(u1, "D1") device2_signing_key = self.generate_and_upload_device_signing_key(u1, "D2") - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect two more edus self.assertEqual(len(self.edus), 2) @@ -637,9 +654,7 @@ def test_upload_signatures(self) -> None: e2e_handler.upload_signing_keys_for_user(u1, cross_signing_keys) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect signing key update edu self.assertEqual(len(self.edus), 2) @@ -662,9 +677,7 @@ def test_upload_signatures(self) -> None: ) self.assertEqual(ret["failures"], {}) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect two edus, in one or two transactions. We don't know what order the # devices will be updated. @@ -689,9 +702,7 @@ def test_delete_devices(self) -> None: self.login("user", "pass", device_id="D2") self.login("user", "pass", device_id="D3") - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect three edus self.assertEqual(len(self.edus), 3) @@ -702,9 +713,7 @@ def test_delete_devices(self) -> None: # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect three edus, in an unknown order self.assertEqual(len(self.edus), 3) @@ -730,15 +739,18 @@ def test_unreachable_server(self) -> None: # create devices u1 = self.register_user("user", "pass") self.login("user", "pass", device_id="D1") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D2") + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() self.assertGreaterEqual(mock_send_txn.call_count, 4) @@ -748,9 +760,7 @@ def test_unreachable_server(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # for each device, there should be a single update self.assertEqual(len(self.edus), 3) @@ -777,16 +787,20 @@ def test_prune_outbound_device_pokes1(self) -> None: # create devices u1 = self.register_user("user", "pass") self.login("user", "pass", device_id="D1") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D2") + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() + # Ensure that we tried sending the device list update EDU's out self.assertGreaterEqual(mock_send_txn.call_count, 4) # run the prune job @@ -801,9 +815,7 @@ def test_prune_outbound_device_pokes1(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # there should be a single update for this user. self.assertEqual(len(self.edus), 1) @@ -835,15 +847,17 @@ def test_prune_outbound_device_pokes2(self) -> None: mock_send_txn.side_effect = AssertionError("fail") self.login("user", "pass", device_id="D2") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") - - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) + self.wait_for_device_list_updates_to_be_sent() + self.assertGreaterEqual(mock_send_txn.call_count, 3) # run the prune job @@ -858,9 +872,7 @@ def test_prune_outbound_device_pokes2(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # ... and we should get a single update for this user. self.assertEqual(len(self.edus), 1) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 22d5b0c094..417ca49fa0 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -27,7 +27,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import FederationError +from synapse.api.errors import Codes, FederationError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures @@ -38,7 +38,7 @@ from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest @@ -94,6 +94,293 @@ async def failing_handler(_origin: str, _content: JsonDict) -> None: self.assertEqual(500, channel.code, channel.result) +class GetMissingEventsRoomCheckTests(unittest.FederatingHomeserverTestCase): + """ + Regression tests for room confusion in /get_missing_events + https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76 + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Local user + self.local_user_id = self.register_user("alice", "pass") + self.local_user_token = self.login("alice", "pass") + self.local_user = UserID.from_string(self.local_user_id) + + # Create 2 rooms (one with the remote server, one without). + # - The remote server will be in this room + self.room_allowed = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + self.inject_room_member( + self.room_allowed, f"@remote:{self.OTHER_SERVER_NAME}", "join" + ) + # - The remote server will _not_ be in this room + self.room_blocked = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + + # Insert a linear chain of events in both rooms + self.room_allowed_event_ids = self.helper.send_messages( + self.room_allowed, num_events=5, tok=self.local_user_token + ) + self.room_blocked_event_ids = self.helper.send_messages( + self.room_blocked, num_events=5, tok=self.local_user_token + ) + + def _extract_returned_event_ids(self, json_body: JsonDict) -> set[str]: + """ + Given the response body of `/get_missing_events`, return the event IDs + of the events that were returned in the response. + This only includes event IDs from `self.room_allowed_event_ids` and + `self.room_blocked_event_ids`; other events are ignored. + + As the federation PDU format doesn't include event IDs + (at least not for every room version), we match on the + `(room_id, content.body, prev_events)` triple against the events + we sent in the setup. + """ + store = self.hs.get_datastores().main + events = self.get_success( + store.get_events_as_list( + list(self.room_allowed_event_ids) + list(self.room_blocked_event_ids) + ) + ) + # (room_id, content.body, prev_events) -> event ID + event_lookup: dict[tuple[str, str, tuple[str, ...]], str] = {} + for event in events: + key = ( + event.room_id, + event.content["body"], + tuple(event.prev_event_ids()), + ) + event_lookup[key] = event.event_id + + returned_event_ids: set[str] = set() + for pdu in json_body["events"]: + key = ( + pdu.get("room_id"), + pdu.get("content", {}).get("body"), + tuple(pdu.get("prev_events", [])), + ) + event_id = event_lookup.get(key) + if event_id is None: + # Not one of the events we created; ignore it. + continue + returned_event_ids.add(event_id) + return returned_event_ids + + def test_get_missing_events_returns_events_from_correct_room(self) -> None: + """ + Tests the happy path when `latest_events` and `earliest_events` + are both in the correct room. + + returned + | + v + e1 <- e2 <- e3 <- e4 <- e5 + ^ ^ + | | + earliest latest + + Not a regression test; I'm just filling a gap in our (in-repo) testing + as far as I can tell. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[1]], + "latest_events": [self.room_allowed_event_ids[3]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + {self.room_allowed_event_ids[2]}, + ) + + def test_get_missing_events_with_empty_earliest_events(self) -> None: + """ + Tests that `/get_missing_events`, when given no `earliest_events`, + walks back to the start of the room, capped at `limit`. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_unknown_earliest_event(self) -> None: + """ + Tests that `/get_missing_events` ignores unknown event IDs given in + `earliest_events`. + + This makes sense as the `earliest_events` are intuitively + 'events to stop at' when walking backwards. + Since we don't know about those events, we don't use them as stopping conditions. + (In other words, this falls back to the same behaviour as + `test_get_missing_events_with_empty_earliest_events`.) + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someUnknownEventId"], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_no_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + no events in `latest_events`, the response is 200 OK + with an empty `events` list. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": [], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_with_unknown_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + unknown events in `latest_events`, the response is 200 OK + with an empty `events` list. + + I imagine this makes sense as you might request several events + in `latest_events` to start walking back from and we need to be + tolerant of the fact that servers don't always know about every event. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": ["$someUnknownEventId"], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` and `latest_events` from the wrong room + treats them the same as being unknown. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_blocked_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_skips_latest_events_from_other_room(self) -> None: + """ + Tests that providing `latest_events` from the wrong room + treats it as being unknown, even if `earliest_events` are from the correct + room. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_earliest_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` from the wrong room causes those + events to be ignored as stopping conditions, + even though `latest_events` are from the correct room. + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This test was previously fine, but is an obvious extra case. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + # Use [-3] here as we want to see if the walk-back algorithm + # confuses depth (topological ordering) across the two rooms. + "earliest_events": [self.room_blocked_event_ids[-3]], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def _create_acl_event(content: JsonDict) -> EventBase: return make_test_event( { @@ -324,6 +611,95 @@ def test_needs_to_be_in_room(self) -> None: self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") +class TimestampToEventTests(unittest.FederatingHomeserverTestCase): + """Tests for `GET /_matrix/federation/v1/timestamp_to_event/`.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a room and join the remote server so it's allowed to query + user = self.register_user("u1", "pass") + tok = self.login("u1", "pass") + self.room_id = self.helper.create_room_as(user, tok=tok) + # Send one event at time = 1000s + self.reactor.advance(1000) + self.event_at_1000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 4000s + self.reactor.advance(3000) + self.event_at_4000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 8000s + self.reactor.advance(4000) + self.event_at_8000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + # Query backwards from 5000s, should find the event at 4000s + (5000000, "b"), + # Query forwards from 1100s, should find the event at 4000s + (1100000, "f"), + ] + ) + def test_happy_path(self, ts: int, dir: str) -> None: + """ + Tests that a server in the room gets 200 OK + with the closest event IDs as requested for a given timestamp, + in both forward and backward directions. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["event_id"], self.event_at_4000) + + @parameterized.expand( + [ + # Query backwards at 0s, no events to be found. + (0, "b"), + # Query forwards from 8100s, no events to be found. + (8100000, "f"), + ] + ) + def test_no_matching_event(self, ts: int, dir: str) -> None: + """ + Tests that a 404 / M_NOT_FOUND is returned when no event occurs + in the requested direction of a timestamp. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_requires_server_in_room(self) -> None: + """ + Tests that a server not in the room is rejected with 403 / M_FORBIDDEN. + """ + # Notably: _don't_ join the remote server to the room + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts=2000000&dir=b", + ) + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") + + class UnstableGetExtremitiesTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, @@ -473,6 +849,84 @@ def test_get_extremities_unstable_not_enabled(self) -> None: self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED") +class EventAuthFederationTests(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a local user + self.user_id = self.register_user("alice", "password") + self.user_tok = self.login("alice", "password") + + # Set up a room and join the remote server to it + self.room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + self.inject_room_member( + self.room_id, f"@remote:{self.OTHER_SERVER_NAME}", Membership.JOIN + ) + + # Create a known event whose auth chain we can request back. + self.event_id = self.helper.send_messages( + self.room_id, num_events=1, tok=self.user_tok + )[0] + + return super().prepare(reactor, clock, hs) + + def test_event_auth_unknown_event_returns_404(self) -> None: + """ + Tests that requesting the auth chain of an unknown event + returns 404 / M_NOT_FOUND. + """ + + # Request an event that doesn't exist in self.room_id. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/$unknownevent", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + def test_event_auth_wrong_room_returns_404(self) -> None: + """ + Tests that a request whose `room_id` is wrong for the event + acts the same as though it were an unknown event. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r + """ + + # Create a second room with its own event. + other_room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + other_room_event_id = self.helper.send_messages( + other_room_id, num_events=1, tok=self.user_tok + )[0] + + # Request the chain of other_room_id's event, but pretend it's part of the room + # we are in. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/{other_room_event_id}", + ) + + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + class SendJoinFederationTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, diff --git a/tests/handlers/test_deactivate_account.py b/tests/handlers/test_deactivate_account.py index 1b749cee1f..f8b4098c71 100644 --- a/tests/handlers/test_deactivate_account.py +++ b/tests/handlers/test_deactivate_account.py @@ -483,3 +483,70 @@ def test_rooms_forgotten_upon_deactivation(self) -> None: # Validate that the created room is forgotten self.assertTrue(room_id in forgotten_rooms) + + def _get_profile_row(self) -> str | None: + """Return the `user_id` of `self.user`'s `profiles` row, or None if absent.""" + return self.get_success( + self._store.db_pool.simple_select_one_onecol( + table="profiles", + keyvalues={"full_user_id": self.user}, + retcol="user_id", + allow_none=True, + desc="_get_profile_row", + ) + ) + + def test_reactivation_recreates_profile(self) -> None: + """ + Tests that reactivating an erased user recreates their profile row, so + that subsequent profile operations work. + """ + self.assertIsNotNone(self._get_profile_row()) + + # Erasure deletes the whole profiles row. + self._deactivate_my_account() + self.assertIsNone(self._get_profile_row()) + + # Reactivating recreates a blank profile row. + deactivate_handler = self.hs.get_deactivate_account_handler() + self.get_success(deactivate_handler.activate_account(self.user)) + self.assertIsNotNone(self._get_profile_row()) + + # Setting a display name now works again. + user = UserID.from_string(self.user) + self.get_success( + self.hs.get_profile_handler().set_displayname( + user, create_requester(user), "Reactivated", by_admin=True + ) + ) + self.assertEqual( + self.get_success(self._store.get_profile_displayname(user)), + "Reactivated", + ) + + def test_reactivation_without_erasure_keeps_profile(self) -> None: + """ + Reactivating a user whose profile row still exists leaves it untouched. + """ + user = UserID.from_string(self.user) + self.get_success( + self.hs.get_profile_handler().set_displayname( + user, create_requester(user), "Original", by_admin=True + ) + ) + + # Deactivate without erasure, so the profile row is left intact. + deactivate_handler = self.hs.get_deactivate_account_handler() + self.get_success( + deactivate_handler.deactivate_account( + self.user, erase_data=False, requester=create_requester(user) + ) + ) + self.assertIsNotNone(self._get_profile_row()) + + # Reactivating must not raise despite the existing profile row. + self.get_success(deactivate_handler.activate_account(self.user)) + self.assertEqual( + self.get_success(self._store.get_profile_displayname(user)), + "Original", + ) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 736f251c27..cb047d118a 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -566,17 +566,20 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: SynapseError, ) - # Send a message to the dehydrated device - ensureDeferred( - self.message_handler.send_device_message( - requester=requester, - message_type="test.message", - messages={user_id: {stored_dehydrated_device_id: {"body": "foo"}}}, + # Send some messages to the dehydrated device + for i in range(12): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={ + user_id: {stored_dehydrated_device_id: {"body": f"foo_{i}"}} + }, + ) ) - ) self.pump() - # Fetch the message of the dehydrated device + # Fetch the first batch of messages from the dehydrated device res = self.get_success( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -586,12 +589,14 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: ) ) - self.assertTrue(len(res["next_batch"]) > 1) - self.assertEqual(len(res["events"]), 1) - self.assertEqual(res["events"][0]["content"]["body"], "foo") + self.assertTrue(res.limited) + # This batch contains the first 10 events + self.assertEqual(len(res.events), 10) + self.assertEqual(res.events[0]["content"]["body"], "foo_0") + self.assertEqual(res.events[1]["content"]["body"], "foo_1") - # Fetch the message of the dehydrated device again, which should return - # the same message as it has not been deleted + # Fetch the first batch again, which should return the same messages as they + # have not been deleted res = self.get_success( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -600,9 +605,26 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: limit=10, ) ) - self.assertTrue(len(res["next_batch"]) > 1) - self.assertEqual(len(res["events"]), 1) - self.assertEqual(res["events"][0]["content"]["body"], "foo") + self.assertTrue(res.limited) + self.assertEqual(len(res.events), 10) + self.assertEqual(res.events[0]["content"]["body"], "foo_0") + self.assertEqual(res.events[7]["content"]["body"], "foo_7") + + # Fetch the next batch + res = self.get_success( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=stored_dehydrated_device_id, + since_token=res.stream_id, + limit=10, + ) + ) + # This is the last batch + self.assertFalse(res.limited) + # This batch contains the last 2 events + self.assertEqual(len(res.events), 2) + self.assertEqual(res.events[0]["content"]["body"], "foo_10") + self.assertEqual(res.events[1]["content"]["body"], "foo_11") @patch("synapse.crypto.keyring.Keyring.process_request", AsyncMock(return_value=None)) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 794c0a3185..0c7edbaa2d 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -357,7 +357,6 @@ def create_invite() -> EventBase: event.room_version, ), exc=LimitExceededError, - by=0.5, ) def _build_and_send_join_event( diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index c88f2c2d15..995a1134b2 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -21,44 +21,30 @@ import json import threading -import time -from http import HTTPStatus from http.server import BaseHTTPRequestHandler, HTTPServer -from io import BytesIO -from typing import Any, ClassVar, Coroutine, Generator, TypeVar, Union -from unittest.mock import ANY, AsyncMock, Mock +from typing import Any, ClassVar, TypeVar +from unittest.mock import AsyncMock, Mock from urllib.parse import parse_qs from parameterized.parameterized import parameterized_class -from signedjson.key import ( - encode_verify_key_base64, - generate_signing_key, - get_verify_key, -) -from signedjson.sign import sign_json -from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import ( AuthError, Codes, - HttpResponseException, InvalidClientTokenError, SynapseError, ) from synapse.appservice import ApplicationService -from synapse.http.site import SynapseRequest from synapse.rest import admin from synapse.rest.client import account, devices, keys, login, logout, register from synapse.server import HomeServer from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock -from tests.server import FakeChannel -from tests.test_utils import get_awaitable_result -from tests.unittest import HomeserverTestCase, override_config, skip_unless +from tests.unittest import HomeserverTestCase, skip_unless from tests.utils import HAS_AUTHLIB, checked_cast, mock_getRawHeaders # These are a few constants that are used as config parameters in the tests. @@ -107,599 +93,6 @@ async def get_json(url: str) -> JsonDict: @skip_unless(HAS_AUTHLIB, "requires authlib") -@parameterized_class( - ("device_scope_prefix", "api_scope"), - [ - ("urn:matrix:client:device:", "urn:matrix:client:api:*"), - ( - "urn:matrix:org.matrix.msc2967.client:device:", - "urn:matrix:org.matrix.msc2967.client:api:*", - ), - ], -) -class MSC3861OAuthDelegation(HomeserverTestCase): - device_scope_prefix: ClassVar[str] - api_scope: ClassVar[str] - - @property - def device_scope(self) -> str: - return self.device_scope_prefix + DEVICE - - servlets = [ - account.register_servlets, - keys.register_servlets, - ] - - def default_config(self) -> dict[str, Any]: - config = super().default_config() - config["public_baseurl"] = BASE_URL - config["disable_registration"] = True - config["experimental_features"] = { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - "admin_token": "admin_token_value", - } - } - return config - - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - self.http_client = Mock(spec=["get_json"]) - self.http_client.get_json.side_effect = get_json - self.http_client.user_agent = b"Synapse Test" - - hs = self.setup_test_homeserver(proxied_http_client=self.http_client) - - # Import this here so that we've checked that authlib is available. - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - self.auth = checked_cast(MSC3861DelegatedAuth, hs.get_auth()) - - self._rust_client = Mock(spec=["post"]) - self.auth._rust_http_client = self._rust_client - - return hs - - def prepare( - self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer - ) -> None: - # Provision the user and the device we use in the tests. - store = homeserver.get_datastores().main - self.get_success(store.register_user(USER_ID)) - self.get_success( - store.store_device(USER_ID, DEVICE, initial_device_display_name=None) - ) - - def _set_introspection_returnvalue(self, response_value: Any) -> AsyncMock: - self._rust_client.post = mock = AsyncMock( - return_value=json.dumps(response_value).encode("utf-8") - ) - return mock - - def _assertParams(self) -> None: - """Assert that the request parameters are correct.""" - params = parse_qs(self._rust_client.post.call_args[1]["request_body"]) - self.assertEqual(params["token"], ["mockAccessToken"]) - self.assertEqual(params["client_id"], [CLIENT_ID]) - self.assertEqual(params["client_secret"], [CLIENT_SECRET]) - - def test_inactive_token(self) -> None: - """The handler should return a 403 where the token is inactive.""" - - self._set_introspection_returnvalue({"active": False}) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_no_scope(self) -> None: - """The handler should return a 403 where no scope is given.""" - - self._set_introspection_returnvalue({"active": True}) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_user_no_subject(self) -> None: - """The handler should return a 500 when no subject is present.""" - - self._set_introspection_returnvalue( - {"active": True, "scope": " ".join([self.api_scope])}, - ) - - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_no_user_scope(self) -> None: - """The handler should return a 500 when no subject is present.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.device_scope]), - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_admin_not_user(self) -> None: - """The handler should raise when the scope has admin right but not user.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_admin(self) -> None: - """The handler should return a requester with admin rights.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - def test_active_admin_highest_privilege(self) -> None: - """The handler should resolve to the most permissive scope.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - def test_active_user(self) -> None: - """The handler should return a requester with normal user rights.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - - def test_active_user_with_device(self) -> None: - """The handler should return a requester with normal user rights and a device ID.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope, self.device_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - self.assertEqual(requester.device_id, DEVICE) - - def test_active_user_with_device_explicit_device_id(self) -> None: - """The handler should return a requester with normal user rights and a device ID, given explicitly, as supported by MAS 0.15+""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope]), - "device_id": DEVICE, - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - # It should have called with the 'X-MAS-Supports-Device-Id: 1' header - self.assertEqual( - self._rust_client.post.call_args[1]["headers"].get( - "X-MAS-Supports-Device-Id", - ), - "1", - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - self.assertEqual(requester.device_id, DEVICE) - - def test_multiple_devices(self) -> None: - """The handler should raise an error if multiple devices are found in the scope.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - self.api_scope, - f"{self.device_scope_prefix}AABBCC", - f"{self.device_scope_prefix}DDEEFF", - ] - ), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), AuthError) - - def test_unavailable_introspection_endpoint(self) -> None: - """The handler should return an internal server error.""" - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - - # The introspection endpoint is returning an error. - self._rust_client.post = AsyncMock( - side_effect=HttpResponseException( - code=500, msg="Internal Server Error", response=b"{}" - ) - ) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint request fails. - self._rust_client.post = AsyncMock(side_effect=Exception()) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return a JSON object. - self._set_introspection_returnvalue(["this is an array", "not an object"]) - - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return valid JSON. - self._set_introspection_returnvalue("this is not valid JSON") - - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - def test_cached_expired_introspection(self) -> None: - """The handler should raise an error if the introspection response gives - an expiry time, the introspection response is cached and then the entry is - re-requested after it has expired.""" - - introspection_mock = self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - self.api_scope, - f"{self.device_scope_prefix}AABBCC", - ] - ), - "username": USERNAME, - "expires_in": 60, - } - ) - - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - - # The first CS-API request causes a successful introspection - self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(introspection_mock.call_count, 1) - - # Sleep for 60 seconds so the token expires. - self.reactor.advance(60.0) - - # Now the CS-API request fails because the token expired - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - # Ensure another introspection request was not sent - self.assertEqual(introspection_mock.call_count, 1) - - def make_device_keys(self, user_id: str, device_id: str) -> JsonDict: - # We only generate a master key to simplify the test. - master_signing_key = generate_signing_key(device_id) - master_verify_key = encode_verify_key_base64(get_verify_key(master_signing_key)) - - return { - "master_key": sign_json( - { - "user_id": user_id, - "usage": ["master"], - "keys": {"ed25519:" + master_verify_key: master_verify_key}, - }, - user_id, - master_signing_key, - ), - } - - def test_cross_signing(self) -> None: - """Try uploading device keys with OAuth delegation enabled.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope, self.device_scope]), - "username": USERNAME, - } - ) - keys_upload_body = self.make_device_keys(USER_ID, DEVICE) - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - keys_upload_body, - access_token="mockAccessToken", - ) - - self.assertEqual(channel.code, 200, channel.json_body) - - # Try uploading *different* keys; it should cause a 501 error. - keys_upload_body = self.make_device_keys(USER_ID, DEVICE) - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - keys_upload_body, - access_token="mockAccessToken", - ) - - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - - def test_admin_token(self) -> None: - """The handler should return a requester with admin rights when admin_token is used.""" - self._set_introspection_returnvalue({"active": False}) - - request = Mock(args={}) - request.args[b"access_token"] = [b"admin_token_value"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual( - requester.user.to_string(), - OIDC_ADMIN_USERID, - ) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - # There should be no call to the introspection endpoint - self._rust_client.post.assert_not_called() - - @override_config({"mau_stats_only": True}) - def test_request_tracking(self) -> None: - """Using an access token should update the client_ips and MAU tables.""" - # To start, there are no MAU users. - store = self.hs.get_datastores().main - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 0) - - known_token = "token-token-GOOD-:)" - - async def mock_http_client_request( - url: str, request_body: str, **kwargs: Any - ) -> bytes: - """Mocked auth provider response.""" - token = parse_qs(request_body)["token"][0] - if token == known_token: - return json.dumps( - { - "active": True, - "scope": self.api_scope, - "sub": SUBJECT, - "username": USERNAME, - }, - ).encode("utf-8") - - return json.dumps({"active": False}).encode("utf-8") - - self._rust_client.post = mock_http_client_request - - EXAMPLE_IPV4_ADDR = "123.123.123.123" - EXAMPLE_USER_AGENT = "httprettygood" - - # First test a known access token - channel = FakeChannel(self.site, self.reactor) - # type-ignore: FakeChannel is a mock of an HTTPChannel, not a proper HTTPChannel - req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] - req.client.host = EXAMPLE_IPV4_ADDR - req.requestHeaders.addRawHeader("Authorization", f"Bearer {known_token}") - req.requestHeaders.addRawHeader("User-Agent", EXAMPLE_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual(channel.json_body["user_id"], USER_ID, channel.json_body) - - # Expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(USER_ID)) - ) - self.assertEqual(len(conn_infos), 1, conn_infos) - conn_info = conn_infos[0] - self.assertEqual(conn_info["access_token"], known_token) - self.assertEqual(conn_info["ip"], EXAMPLE_IPV4_ADDR) - self.assertEqual(conn_info["user_agent"], EXAMPLE_USER_AGENT) - - # Now test MAS making a request using the special __oidc_admin token - MAS_IPV4_ADDR = "127.0.0.1" - MAS_USER_AGENT = "masmasmas" - - channel = FakeChannel(self.site, self.reactor) - req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] - req.client.host = MAS_IPV4_ADDR - req.requestHeaders.addRawHeader( - "Authorization", f"Bearer {self.auth._admin_token()}" - ) - req.requestHeaders.addRawHeader("User-Agent", MAS_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual( - channel.json_body["user_id"], OIDC_ADMIN_USERID, channel.json_body - ) - - # Still expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(OIDC_ADMIN_USERID)) - ) - self.assertEqual(conn_infos, []) - - class FakeMasHandler(BaseHTTPRequestHandler): server: "FakeMasServer" @@ -809,31 +202,6 @@ class MasAuthDelegation(HomeserverTestCase): def device_scope(self) -> str: return self.device_scope_prefix + DEVICE - def till_deferred_has_result( - self, - awaitable: Union[ - "Coroutine[Deferred[Any], Any, T]", - "Generator[Deferred[Any], Any, T]", - "Deferred[T]", - ], - ) -> "Deferred[T]": - """Wait until a deferred has a result. - - This is useful because the Rust HTTP client will resolve the deferred - using reactor.callFromThread, which are only run when we call - reactor.advance. - """ - deferred = ensureDeferred(awaitable) - tries = 0 - while not deferred.called: - time.sleep(0.1) - self.reactor.advance(0) - tries += 1 - if tries > 100: - raise Exception("Timed out waiting for deferred to resolve") - - return deferred - def default_config(self) -> dict[str, Any]: config = super().default_config() config["public_baseurl"] = BASE_URL @@ -883,11 +251,7 @@ def test_simple_introspection(self) -> None: "expires_in": 60, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertEqual(requester.device_id, DEVICE) @@ -906,11 +270,7 @@ def test_unexpiring_token(self) -> None: "username": USERNAME, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertEqual(requester.device_id, DEVICE) @@ -931,9 +291,7 @@ def test_inexistent_device(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), InvalidClientTokenError, ) self.assertEqual(failure.value.code, 401) @@ -948,9 +306,7 @@ def test_inexistent_user(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), AuthError, ) # This is a 500, it should never happen really @@ -966,9 +322,7 @@ def test_missing_scope(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), InvalidClientTokenError, ) self.assertEqual(failure.value.code, 401) @@ -977,9 +331,7 @@ def test_invalid_response(self) -> None: self.server.introspection_response = {} failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), SynapseError, ) self.assertEqual(failure.value.code, 503) @@ -994,11 +346,7 @@ def test_device_id_in_body(self) -> None: "device_id": DEVICE, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.device_id, DEVICE) @@ -1011,11 +359,7 @@ def test_admin_scope(self) -> None: "expires_in": 60, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertTrue(self.get_success(self._auth.is_server_admin(requester))) @@ -1040,17 +384,15 @@ def test_cached_expired_introspection(self) -> None: request.requestHeaders.getRawHeaders = mock_getRawHeaders() # The first CS-API request causes a successful introspection - self.get_success( - self.till_deferred_has_result(self._auth.get_user_by_req(request)) - ) + self.get_success(self._auth.get_user_by_req(request)) self.assertEqual(self.server.calls, 1) # Sleep for 60 seconds so the token expires. self.reactor.advance(60.0) # Now the CS-API request fails because the token expired - self.assertFailure( - self.till_deferred_has_result(self._auth.get_user_by_req(request)), + self.get_failure( + self._auth.get_user_by_req(request), InvalidClientTokenError, ) # Ensure another introspection request was not sent @@ -1095,25 +437,7 @@ def test_metadata_url_uses_subpath(self) -> None: }, }, ), - ] - # Run the tests with experimental delegation only if authlib is available - + [ - ( - { - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - "admin_token": "admin_token_value", - } - } - }, - ), - ] - * HAS_AUTHLIB, + ], ) class DisabledEndpointsTestCase(HomeserverTestCase): servlets = [ diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 62b84c77a4..b81c2954dc 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -960,7 +960,7 @@ def test_exchange_code_jwt_key(self) -> None: # advance the clock a bit before we start, so we aren't working with zero # timestamps. self.reactor.advance(1000) - start_time = self.reactor.seconds() + start_time_s = int(self.reactor.seconds()) ret = self.get_success(self.provider._exchange_code(code, code_verifier="")) self.assertEqual(ret, token) @@ -981,8 +981,8 @@ def test_exchange_code_jwt_key(self) -> None: self.assertEqual(claims["aud"], ISSUER) self.assertEqual(claims["iss"], "DEFGHI") self.assertEqual(claims["sub"], CLIENT_ID) - self.assertEqual(claims["iat"], start_time) - self.assertGreater(claims["exp"], start_time) + self.assertEqual(claims["iat"], start_time_s) + self.assertGreater(claims["exp"], start_time_s) # check the rest of the POSTed data self.assertEqual(args["grant_type"], ["authorization_code"]) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 44f1e6432d..a562050842 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -19,7 +19,7 @@ # # import itertools -from typing import cast +from typing import Any, cast from unittest.mock import Mock, call from parameterized import parameterized @@ -36,6 +36,11 @@ from synapse.api.room_versions import ( RoomVersion, ) +from synapse.config.server import ( + DEFAULT_IDLE_TIMER, + DEFAULT_LAST_ACTIVE_GRANULARITY, + DEFAULT_SYNC_ONLINE_TIMEOUT, +) from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import EventBase, make_event_from_dict from synapse.federation.sender import FederationSender @@ -44,10 +49,10 @@ EXTERNAL_PROCESS_EXPIRY, FEDERATION_PING_INTERVAL, FEDERATION_TIMEOUT, - IDLE_TIMER, - LAST_ACTIVE_GRANULARITY, - SYNC_ONLINE_TIMEOUT, PresenceHandler, + WorkerPresenceHandler, + get_interested_parties, + get_interested_remotes, handle_timeout, handle_update, ) @@ -94,6 +99,9 @@ def test_offline_to_online(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -105,16 +113,20 @@ def test_offline_to_online(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, + ), + call( + now=now, + obj=user_id, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -142,6 +154,9 @@ def test_online_to_online(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -154,16 +169,20 @@ def test_online_to_online(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, + ), + call( + now=now, + obj=user_id, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -177,7 +196,7 @@ def test_online_to_online_last_active_noop(self) -> None: prev_state = UserPresenceState.default(user_id) prev_state = prev_state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 10, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 10, currently_active=True, ) @@ -193,6 +212,9 @@ def test_online_to_online_last_active_noop(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -205,16 +227,20 @@ def test_online_to_online_last_active_noop(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, + ), + call( + now=now, + obj=user_id, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -228,7 +254,7 @@ def test_online_to_online_last_active(self) -> None: prev_state = UserPresenceState.default(user_id) prev_state = prev_state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 1, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 1, currently_active=True, ) @@ -242,6 +268,9 @@ def test_online_to_online_last_active(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -253,11 +282,15 @@ def test_online_to_online_last_active(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 2) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, + ), + call( + now=now, + obj=user_id, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ), ], any_order=True, @@ -283,6 +316,9 @@ def test_remote_ping_timer(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -323,6 +359,9 @@ def test_online_to_offline(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -351,6 +390,9 @@ def test_online_to_idle(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -365,7 +407,7 @@ def test_online_to_idle(self) -> None: call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ) ], any_order=True, @@ -442,6 +484,9 @@ def test_override(self, initial_state: str, final_state: str) -> None: wheel_timer=wheel_timer, now=now, persist=True, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) wheel_timer.insert.assert_not_called() @@ -506,6 +551,9 @@ def _test_ratelimit_offline_to_online_to_unavailable( wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) # Check that the user is offline. @@ -556,7 +604,7 @@ def test_idle_timer(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - IDLE_TIMER - 1, + last_active_ts=now - DEFAULT_IDLE_TIMER - 1, last_user_sync_ts=now, status_msg=status_msg, ) @@ -574,6 +622,9 @@ def test_idle_timer(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -594,7 +645,7 @@ def test_busy_no_idle(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.BUSY, - last_active_ts=now - IDLE_TIMER - 1, + last_active_ts=now - DEFAULT_IDLE_TIMER - 1, last_user_sync_ts=now, status_msg=status_msg, ) @@ -612,6 +663,9 @@ def test_busy_no_idle(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -629,7 +683,7 @@ def test_sync_timeout(self) -> None: state = state.copy_and_replace( state=PresenceState.ONLINE, last_active_ts=0, - last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1, + last_user_sync_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, status_msg=status_msg, ) device_state = UserDevicePresenceState( @@ -646,6 +700,9 @@ def test_sync_timeout(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -662,8 +719,8 @@ def test_sync_online(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - SYNC_ONLINE_TIMEOUT - 1, - last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1, + last_active_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, + last_user_sync_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, status_msg=status_msg, ) device_state = UserDevicePresenceState( @@ -680,6 +737,9 @@ def test_sync_online(self) -> None: syncing_device_ids={(user_id, device_id)}, user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -715,6 +775,9 @@ def test_federation_ping(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -746,6 +809,9 @@ def test_no_timeout(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNone(new_state) @@ -766,7 +832,14 @@ def test_federation_timeout(self) -> None: # Note that this is a remote user so we do not have their device information. new_state = handle_timeout( - state, is_mine=False, syncing_device_ids=set(), user_devices={}, now=now + state, + is_mine=False, + syncing_device_ids=set(), + user_devices={}, + now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -783,7 +856,7 @@ def test_last_active(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 1, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 1, last_user_sync_ts=now, last_federation_update_ts=now, status_msg=status_msg, @@ -802,6 +875,9 @@ def test_last_active(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -860,7 +936,7 @@ def test_restored_presence_idles(self) -> None: self.assertEqual(state.state, PresenceState.ONLINE) # Advance such that the user should timeout. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) self.reactor.pump([5]) # Check that the user is now offline. @@ -900,7 +976,7 @@ def test_restored_presence_online_after_sync( self.assertEqual(state.state, PresenceState.ONLINE) # Advance slightly and sync. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000 / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000 / 2) self.get_success( presence_handler.user_syncing( self.user_id, @@ -917,7 +993,7 @@ def test_restored_presence_online_after_sync( self.assertEqual(state.state, expected_state) # Advance such that the user's preloaded data times out, but not the new sync. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000 / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000 / 2) self.reactor.pump([5]) # Check that the user is in the sync state (as the client is currently syncing still). @@ -926,6 +1002,237 @@ def test_restored_presence_online_after_sync( ) self.assertEqual(state.state, sync_state) + @unittest.override_config({"presence": {"enabled": False}}) + def test_restored_presence_flushed_offline_when_presence_disabled(self) -> None: + """If presence is disabled, any non-offline presence states left in the + database from when presence was enabled should be marked as offline at + startup, and the updates streamed out to clients. + """ + main_store = self.hs.get_datastores().main + before_token = main_store.get_current_presence_token() + + # Get the handler, which schedules the startup flush. + presence_handler = self.hs.get_presence_handler() + + # Fire pending `call_when_running` hooks and let the flush complete. + self.reactor.run() + self.reactor.advance(0) + + # The user should now be offline, both in memory and in the database. + state = self.get_success( + presence_handler.get_state(UserID.from_string(self.user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + db_state = self.get_success(main_store.get_presence_for_users([self.user_id]))[ + self.user_id + ] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # The flush must advance the presence stream so that syncing clients + # are sent the offline updates. + self.assertGreater(main_store.get_current_presence_token(), before_token) + + +class PresenceDisabledSyncTestCase(unittest.HomeserverTestCase): + """Tests that stale presence states left over from when presence was + enabled reach clients over /sync, and that the startup flush marks them as + offline and sends the offline updates down /sync too. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + @unittest.override_config({"presence": {"enabled": False}}) + def test_stale_presence_flushed_offline_and_sent_on_sync(self) -> None: + user1 = self.register_user("alice", "pass") + user1_tok = self.login(user1, "pass") + user2 = self.register_user("bob", "pass") + user2_tok = self.login(user2, "pass") + + room_id = self.helper.create_room_as(user1, tok=user1_tok) + self.helper.join(room_id, user2, tok=user2_tok) + + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] + + # Seed a stale online presence state for user1, left over from when + # presence was enabled: in the database, and in the presence handler's + # in-memory state (which at startup is preloaded from the database). + now = self.clock.time_msec() + stale_state = UserPresenceState( + user_id=user1, + state=PresenceState.ONLINE, + last_active_ts=now, + last_federation_update_ts=now, + last_user_sync_ts=now, + status_msg=None, + currently_active=True, + ) + main_store = self.hs.get_datastores().main + self.get_success(main_store.update_presence([stale_state])) + + presence_handler = self.hs.get_presence_handler() + assert isinstance(presence_handler, PresenceHandler) + presence_handler.user_to_current_state[user1] = stale_state + + # The stale state comes down user2's incremental sync, even though + # presence is disabled. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.ONLINE)], + ) + next_batch = channel.json_body["next_batch"] + + # Run the startup flush, as scheduled when the presence writer starts + # up with presence disabled. + self.get_success(presence_handler._mark_stale_presence_as_offline()) + + # The stale state should have been marked offline in the database... + db_state = self.get_success(main_store.get_presence_for_users([user1]))[user1] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # ... and the offline update also comes down user2's sync. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.OFFLINE)], + ) + + # Once caught up, further syncs include no presence. + next_batch = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body.get("presence", {}).get("events", []), []) + + +# Timer values used by `PresenceConfigurableTimersTestCase`, all larger than +# the corresponding defaults. +_CUSTOM_TIMERS_CONFIG = { + "presence": { + "last_active_granularity": "2m", + "sync_online_timeout": "3m", + "idle_timeout": "20m", + } +} + + +class PresenceConfigurableTimersTestCase(unittest.HomeserverTestCase): + """Tests that the presence state machine timers can be changed via the + `presence` config section. + + Each test checks that nothing happens where the default timer would have + fired, and that the transition then occurs once the configured timer + elapses. + """ + + device_id = "dev-1" + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.user_id = f"@test:{hs.config.server.server_name}" + self.user_id_obj = UserID.from_string(self.user_id) + + def _get_state(self) -> UserPresenceState: + return self.get_success(self.presence_handler.get_state(self.user_id_obj)) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_config_parsing(self) -> None: + config = self.hs.config.server + self.assertEqual(config.presence_last_active_granularity, 2 * 60 * 1000) + self.assertEqual(config.presence_sync_online_timeout, 3 * 60 * 1000) + self.assertEqual(config.presence_idle_timeout, 20 * 60 * 1000) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_sync_online_timeout(self) -> None: + """A user only goes offline once the configured sync timeout passes.""" + with self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ): + pass + + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Well past the DEFAULT_SYNC_ONLINE_TIMEOUT, but short of the + # configured 3m: still online. + self.reactor.advance(2 * DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Past the configured timeout: offline. + self.reactor.advance(3 * 60) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.OFFLINE) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_idle_timeout(self) -> None: + """A continuously syncing but inactive user only goes idle once the + configured idle timeout passes.""" + # Leave the sync open so the device never times out. + self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ) + + # Well past the DEFAULT_IDLE_TIMER, but short of the configured 20m: + # still online. + self.reactor.advance(2 * DEFAULT_IDLE_TIMER / 1000) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Past the configured timeout: idle. + self.reactor.advance(15 * 60) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.UNAVAILABLE) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_last_active_granularity(self) -> None: + """A user remains "currently active" for the configured duration + after their last activity.""" + with self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ): + pass + + self.assertTrue(self._get_state().currently_active) + + # Past the DEFAULT_LAST_ACTIVE_GRANULARITY, but short of the + # configured 2m: still currently active. + self.reactor.advance(90) + self.reactor.pump([5]) + state = self._get_state() + self.assertEqual(state.state, PresenceState.ONLINE) + self.assertTrue(state.currently_active) + + # Past the configured granularity (but short of the 3m sync timeout): + # no longer currently active, but still online. + self.reactor.advance(60) + self.reactor.pump([5]) + state = self._get_state() + self.assertEqual(state.state, PresenceState.ONLINE) + self.assertFalse(state.currently_active) + class PresenceHandlerTestCase(BaseMultiWorkerStreamTestCase): user_id = "@test:server" @@ -951,8 +1258,7 @@ def test_external_process_timeout(self) -> None: self.get_success( worker_presence_handler.user_syncing( self.user_id, self.device_id, True, PresenceState.ONLINE - ), - by=0.1, + ) ) # Check that if we wait a while without telling the handler the user has @@ -980,14 +1286,14 @@ def test_user_goes_offline_by_timeout_status_msg_remain(self) -> None: # Check that if we wait a while without telling the handler the user has # stopped syncing that their presence state doesn't get timed out. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 2) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) self.assertEqual(state.state, PresenceState.ONLINE) self.assertEqual(state.status_msg, status_msg) # Check that if the timeout fires, then the syncing user gets timed out - self.reactor.advance(SYNC_ONLINE_TIMEOUT) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) # status_msg should remain even after going offline @@ -1270,12 +1576,11 @@ def test_set_presence_from_syncing_multi_device( "dev-1", affect_presence=dev_1_state != PresenceState.OFFLINE, presence_state=dev_1_state, - ), - by=0.01, + ) ) # 2. Wait half the idle timer. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.1]) # 3. Sync with the second device. @@ -1285,8 +1590,7 @@ def test_set_presence_from_syncing_multi_device( "dev-2", affect_presence=dev_2_state != PresenceState.OFFLINE, presence_state=dev_2_state, - ), - by=0.01, + ) ) # 4. Assert the expected presence state. @@ -1303,7 +1607,7 @@ def test_set_presence_from_syncing_multi_device( # When testing with workers, make another random sync (with any *different* # user) to keep the process information from expiring. # - # This is due to EXTERNAL_PROCESS_EXPIRY being equivalent to IDLE_TIMER. + # This is due to EXTERNAL_PROCESS_EXPIRY being equivalent to DEFAULT_IDLE_TIMER. if test_with_workers: with self.get_success( worker_presence_handler.user_syncing( @@ -1311,14 +1615,13 @@ def test_set_presence_from_syncing_multi_device( "dev-3", affect_presence=True, presence_state=PresenceState.ONLINE, - ), - by=0.01, + ) ): pass # 5. Advance such that the first device should be discarded (the idle timer), # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.01]) # 6. Assert the expected presence state. @@ -1334,7 +1637,7 @@ def test_set_presence_from_syncing_multi_device( # 7. Advance such that the second device should be discarded (half the idle timer), # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.1]) # 8. The devices are still "syncing" (the sync context managers were never @@ -1507,8 +1810,7 @@ def test_set_presence_from_non_syncing_multi_device( "dev-1", affect_presence=dev_1_state != PresenceState.OFFLINE, presence_state=dev_1_state, - ), - by=0.1, + ) ) # 2. Sync with the second device. @@ -1518,8 +1820,7 @@ def test_set_presence_from_non_syncing_multi_device( "dev-2", affect_presence=dev_2_state != PresenceState.OFFLINE, presence_state=dev_2_state, - ), - by=0.1, + ) ) # 3. Assert the expected presence state. @@ -1539,7 +1840,7 @@ def test_set_presence_from_non_syncing_multi_device( # 5. Advance such that the first device should be discarded (the sync timeout), # then pump so _handle_timeouts function to called. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) self.reactor.pump([5]) # 6. Assert the expected presence state. @@ -1562,7 +1863,7 @@ def test_set_presence_from_non_syncing_multi_device( if dev_1_state == PresenceState.BUSY or dev_2_state == PresenceState.BUSY: timeout = BUSY_ONLINE_TIMEOUT else: - timeout = SYNC_ONLINE_TIMEOUT + timeout = DEFAULT_SYNC_ONLINE_TIMEOUT self.reactor.advance(timeout / 1000) self.reactor.pump([5]) @@ -1625,8 +1926,7 @@ def test_set_presence_from_syncing_keeps_busy( self.get_success( worker_to_sync_against.get_presence_handler().user_syncing( self.user_id, self.device_id, True, PresenceState.ONLINE - ), - by=0.1, + ) ) # Check against the main process that the user's presence did not change. @@ -1636,7 +1936,7 @@ def test_set_presence_from_syncing_keeps_busy( # Advance such that the device would be discarded if it was not busy, # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000) self.reactor.pump([5]) # The account should still be busy. @@ -1689,7 +1989,7 @@ def test_untracked_does_not_idle(self) -> None: self.assertEqual(state.state, PresenceState.ONLINE) # The timeout should not fire and the state should be the same. - self.reactor.advance(SYNC_ONLINE_TIMEOUT) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) self.assertEqual(state.state, PresenceState.ONLINE) @@ -2142,3 +2442,391 @@ def create_fake_event_from_remote_server( ) return event + + +class PresenceExcludeRoomsTestCase(unittest.HomeserverTestCase): + """Tests that `exclude_rooms_from_presence` stops presence being routed + between users solely because they share an excluded room.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.store = hs.get_datastores().main + self.presence_router = hs.get_presence_router() + self.presence_handler = hs.get_presence_handler() + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + + def test_excluded_rooms_not_routed(self) -> None: + # Two rooms that user1 is joined to. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + + state = UserPresenceState.default(self.user1) + + # Without any exclusions both rooms are interested in user1's presence. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties(self.store, self.presence_router, [state]) + ) + self.assertIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # Excluding one room drops it as an interested party, but the other + # (non-excluded) room still routes presence... + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # ...and the user always receives their own presence, even when all of + # their rooms are excluded. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room, shared_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertNotIn(shared_room, room_ids_to_states) + self.assertIn(self.user1, users_to_states) + + @override_config({"exclude_rooms_from_presence": ["!excluded:test"]}) + def test_config_populates_handler(self) -> None: + """The config option should be plumbed through to the presence handler + and the presence event source as a frozenset.""" + self.assertEqual( + self.presence_handler._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + event_source = self.hs.get_event_sources().sources.presence + self.assertEqual( + event_source._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + def test_is_visible_respects_excluded_rooms(self) -> None: + """`is_visible` (which drives the read side of /sync) should not + consider two users to share presence solely via an excluded room.""" + user1 = UserID.from_string(self.user1) + user2 = UserID.from_string(self.user2) + + # A single shared room: the two users can see each other's presence. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # Excluding the only shared room hides presence between them. + self.presence_handler._rooms_to_exclude_from_presence = frozenset( + {excluded_room} + ) + self.assertFalse( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # But a second, non-excluded shared room restores visibility. + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + def test_get_interested_remotes_respects_excluded_rooms(self) -> None: + """The federation fan-out side (`get_interested_remotes`) must not route + presence to servers reached solely via an excluded room.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + state = UserPresenceState.default(self.user1) + + def hosts_for(excluded: frozenset) -> set: + result = self.get_success( + get_interested_remotes( + self.store, self.presence_router, [state], excluded + ) + ) + hosts: set[str] = set() + for room_hosts, _ in result: + hosts.update(room_hosts) + return hosts + + # The local server is a host in the room (all members are local here), + # so presence would be routed there... + self.assertIn("test", hosts_for(frozenset())) + # ...but excluding the only room removes it as a source of destinations. + self.assertNotIn("test", hosts_for(frozenset({excluded_room}))) + + +class PresenceGetNewEventsStreamTestCase(unittest.HomeserverTestCase): + """Tests the incremental (`from_key`) branch of + `PresenceEventSource.get_new_events`, which decides which updated users are + interesting to the syncing user by intersecting their cached room sets. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.event_source = hs.get_event_sources().sources.presence + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + self.user3 = self.register_user("user3", "pass") + self.token3 = self.login("user3", "pass") + + def _set_presence(self, user_id: str, state: str = "online") -> None: + self.get_success( + self.presence_handler.set_state( + UserID.from_string(user_id), "dev", {"presence": state} + ) + ) + + def _updated_users_seen_by(self, user_id: str, from_key: int) -> set[str]: + states, _ = self.get_success( + self.event_source.get_new_events( + user=UserID.from_string(user_id), from_key=from_key + ) + ) + return {state.user_id for state in states} + + def test_incremental_interest(self) -> None: + """A syncing user sees updates from users they share a room with (and + themselves), but not from strangers.""" + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + # user3 is in an unrelated room. + self.helper.create_room_as(self.user3, tok=self.token3) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1) + self._set_presence(self.user2) + self._set_presence(self.user3) + + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + self.assertIn(self.user2, seen) # always sees own updates + self.assertNotIn(self.user3, seen) + + def test_incremental_interest_excluded_room(self) -> None: + """Sharing only an excluded room does not make an updated user + interesting; sharing an additional normal room does.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.event_source._rooms_to_exclude_from_presence = frozenset({excluded_room}) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "online") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertNotIn(self.user1, seen) + + # A second, non-excluded shared room restores interest. (Use a + # different presence state, as repeating the same one would not + # generate a new update.) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "unavailable") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + + +class WorkerPresenceThrottleTestCase(BaseMultiWorkerStreamTestCase): + """Tests that sync workers suppress the per-sync-request presence updates + that the presence writer would discard anyway, while relaying genuine + state changes immediately.""" + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = f"@throttled:{hs.config.server.server_name}" + self.user_id_obj = UserID.from_string(self.user_id) + self.device_id = "dev-1" + # In this test setup the main process is the presence writer. + self.writer_handler = hs.get_presence_handler() + + def _make_sync_worker(self) -> tuple[Any, list, list]: + """Create a sync worker whose proxied presence calls are recorded.""" + worker = self.make_worker_hs( + "synapse.app.generic_worker", {"worker_name": "synchrotron"} + ) + presence = worker.get_presence_handler() + assert isinstance(presence, WorkerPresenceHandler) + + set_state_calls: list = [] + bump_calls: list = [] + real_set_state = presence._set_state_client + real_bump = presence._bump_active_client + + async def recording_set_state(**kwargs: Any) -> Any: + set_state_calls.append(kwargs) + return await real_set_state(**kwargs) + + async def recording_bump(**kwargs: Any) -> Any: + bump_calls.append(kwargs) + return await real_bump(**kwargs) + + presence._set_state_client = recording_set_state + presence._bump_active_client = recording_bump + return presence, set_state_calls, bump_calls + + def _sync(self, presence: Any, state: str = PresenceState.ONLINE) -> Any: + # Note: `get_success` only advances the fake clock by tiny epsilon steps + # while pumping the replication traffic; the throttle window is + # time-sensitive and these tests advance time explicitly. + return self.get_success( + presence.user_syncing(self.user_id, self.device_id, True, state), + ) + + def test_repeated_syncs_are_throttled(self) -> None: + presence, set_state_calls, _ = self._make_sync_worker() + + # Several syncs in quick succession only relay one set_state. + for _ in range(3): + self._sync(presence) + self.assertEqual(len(set_state_calls), 1) + + # The user did come online on the writer. + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.ONLINE) + + # Once the relay window has passed, the next sync relays again. + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) + + def test_state_changes_are_relayed_immediately(self) -> None: + presence, set_state_calls, _ = self._make_sync_worker() + + self._sync(presence, PresenceState.ONLINE) + self._sync(presence, PresenceState.UNAVAILABLE) + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + # A repeat of the current state within the window is suppressed. + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + def test_resends_after_device_stops_syncing(self) -> None: + """After a USER_SYNC stop is sent the writer may time the user out, so + a device that reconnects within the window must be relayed afresh.""" + presence, set_state_calls, _ = self._make_sync_worker() + + with self._sync(presence): + pass + self.assertEqual(len(set_state_calls), 1) + + # Wait for the going-offline grace period to elapse: USER_SYNC stop is + # sent and the throttle entry evicted. The writer then times the user + # out to offline. Advance in steps (rather than one jump) so the + # replicated stop command is delivered before the writer's timeout + # loop fires (which only starts 30s after startup). + for _ in range(4): + self.reactor.advance(12) + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.OFFLINE) + + # Reconnecting relays the state immediately, even though the presence + # value is unchanged from the last relayed one. + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.ONLINE) + + def test_bumps_are_throttled(self) -> None: + presence, set_state_calls, bump_calls = self._make_sync_worker() + + # While we recently relayed an online state, bumps are suppressed. + self._sync(presence, PresenceState.ONLINE) + for _ in range(3): + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 0) + + # After the window passes, a bump goes through (and then suppresses + # further bumps). + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + for _ in range(2): + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 1) + + def test_explicit_set_state_always_relayed_and_resets(self) -> None: + """An explicit (non-sync) set_state is always relayed, and resets the + throttle so the next sync-driven update is relayed afresh.""" + presence, set_state_calls, _ = self._make_sync_worker() + + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 1) + + # An explicit update of the same state within the window still goes + # through (it isn't sync-driven)... + self.get_success( + presence.set_state( + self.user_id_obj, + self.device_id, + {"presence": PresenceState.ONLINE}, + ), + ) + self.assertEqual(len(set_state_calls), 2) + + # ...and the following sync-driven update is relayed rather than + # suppressed, re-establishing the writer's sync timestamps. + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + def test_bump_after_non_online_state_goes_through(self) -> None: + presence, set_state_calls, bump_calls = self._make_sync_worker() + + # The user is unavailable; a bump may un-idle them so it must not be + # suppressed. + self._sync(presence, PresenceState.UNAVAILABLE) + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 1) + + @override_config({"presence": {"sync_online_timeout": "12s"}}) + def test_relay_interval_scales_with_config(self) -> None: + """The throttle window is derived from the configurable presence timers, + so it stays comfortably below a lowered sync online timeout rather than + being a hardcoded 25s (which would make users flap).""" + presence, set_state_calls, _ = self._make_sync_worker() + + # 5/6 of min(12s sync online timeout, 60s default last-active + # granularity). + self.assertEqual(presence._sync_presence_relay_interval, 10 * 1000) + + # A repeat within the (now shorter) window is still suppressed... + self._sync(presence) + self._sync(presence) + self.assertEqual(len(set_state_calls), 1) + + # ...and once it passes, the next sync relays again. + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5152e8fc53..561b45827f 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -200,7 +200,7 @@ async def slow_update_membership(*args: Any, **kwargs: Any) -> tuple[str, int]: self.assertEqual(membership[state_tuple].content["displayname"], "Frank") # Let's be sure we are over the delay introduced by slow_update_membership - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) membership = self.get_success( self.storage_controllers.state.get_current_state( @@ -278,7 +278,7 @@ async def potentially_slow_update_membership( # Let's be sure we are over the delay introduced by slow_update_membership # and that the task was not executed as expected - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) membership = self.get_success( self.storage_controllers.state.get_current_state( @@ -299,8 +299,10 @@ async def potentially_slow_update_membership( ) ) + # Wait for the `TaskScheduler.SCHEDULE_INTERVAL` + self.reactor.advance(Duration(minutes=1).as_secs()) # Let's be sure we are over the delay introduced by slow_update_membership - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) # Updates should have been resumed from room 2 after the restart # so room 1 should not have been updated this time diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index d5b95e4ef6..0a7475856a 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -71,7 +71,6 @@ def test_local_user_local_joins_contribute_to_limit_and_are_limited(self) -> Non action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) @override_config({"rc_joins_per_room": {"per_second": 0.1, "burst_count": 2}}) @@ -213,7 +212,6 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: remote_room_hosts=[self.OTHER_SERVER_NAME], ), LimitExceededError, - by=0.5, ) # TODO: test that remote joins to a room are rate limited. @@ -281,7 +279,6 @@ def test_local_users_joining_on_another_worker_contribute_to_rate_limit( action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) # Try to join as Chris on the original worker. Should get denied because Alice @@ -294,7 +291,6 @@ def test_local_users_joining_on_another_worker_contribute_to_rate_limit( action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index c67ea9b0e0..6d912fd79d 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -12,17 +12,22 @@ # . # # +from http import HTTPStatus from unittest import mock import signedjson +from parameterized import parameterized from signedjson.key import encode_verify_key_base64, get_verify_key +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase +from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync @@ -31,7 +36,7 @@ from synapse.util.clock import Clock from tests import unittest -from tests.test_utils import event_injection +from tests.test_utils import FakeResponse, event_injection from tests.test_utils.event_builders import make_test_event @@ -546,3 +551,92 @@ def _fetch_federation_event(self, event_id: str) -> JsonDict | None: if ev["event_id"] == event_id: return ev return None + + def _mock_policy_server_response_with_http_error( + self, + status: HTTPStatus, + error_body: JsonDict, + ) -> None: + """ + Make the policy server reply to its `/sign` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/_matrix/policy/v1/sign" in uri, ( + f"unexpected outbound request to {uri!r}" + ) + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent = mock.create_autospec(Agent, spec_set=True) + fake_agent.request.side_effect = request + self.handler._federation_client.transport_layer = TransportLayerClient(self.hs) + self.hs.get_federation_http_client().agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_policy_server_error_bubbling_to_client( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Tests how errors from the policy server are forwarded back to clients. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + + # Mock the policy server (at the HTTP level) to return + # the configured error + self._mock_policy_server_response_with_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + response_body = self.helper.send_event( + self.room_id, + "m.room.message", + {"body": "honk", "msgtype": "m.text"}, + tok=self.creator_token, + expect_code=expected_client_facing_error_status, + ) + + self.assertEqual(response_body, expected_client_facing_error_body) diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py index eea88cd136..acb88343f2 100644 --- a/tests/handlers/test_send_email.py +++ b/tests/handlers/test_send_email.py @@ -146,7 +146,7 @@ def test_send_email(self) -> None: ) # the message should now get delivered - self.get_success(d, by=0.1) + self.get_success(d) # check it arrived self.assertEqual(len(message_delivery.messages), 1) @@ -213,7 +213,7 @@ def test_send_email_force_tls(self) -> None: ) # the message should now get delivered - self.get_success(d, by=0.1) + self.get_success(d) # check it arrived self.assertEqual(len(message_delivery.messages), 1) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 623eef0ecb..0bbe084547 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -248,6 +248,14 @@ def test_started_typing_remote_send(self) -> None: ) ) + # Wait for the EDU to get pushed out over federation + # + # `started_typing` is fire-and-forget and handles the remote federation part as + # part of a background process which isn't waited on. + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + self.mock_federation_client.put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", @@ -367,6 +375,14 @@ def test_stopped_typing(self) -> None: [call(StreamKeyType.TYPING, 1, rooms=[ROOM_ID])] ) + # Wait for the EDU to get pushed out over federation + # + # `stopped_typing` is fire-and-forget and handles the remote federation part as + # part of a background process which isn't waited on. + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + self.mock_federation_client.put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index f50fa1f4a0..dc6738ca28 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -555,7 +555,15 @@ def test_process_join_after_server_leaves_room(self) -> None: # Process the leave and join in one go. dir_handler.update_user_directory = True dir_handler.notify_new_event() - self.wait_for_background_updates() + + # Wait for the user directory to update + # + # `notify_new_event` is fire-and-forget and the actual changes happen as part of + # a background process loop which isn't waited on. + # + # We're specifically waiting for the database queries in the `notify_new_event` + # background process. + self.reactor.advance(0) # The user sharing tables should have been updated. public3 = self.get_success(self.user_dir_helper.get_users_in_public_rooms()) @@ -1124,7 +1132,6 @@ def test_local_user_leaving_room_remains_in_user_directory(self) -> None: # Alice leaves the other. She should still be in the directory. self.helper.leave(room2, alice, tok=alice_token) - self.wait_for_background_updates() users, in_public, in_private = self.get_success( self.user_dir_helper.get_tables() ) diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index a28d8e34d3..5563ec4683 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -19,9 +19,6 @@ # # -import logging -import platform - from twisted.internet import defer from twisted.internet.testing import MemoryReactor @@ -37,11 +34,8 @@ from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase -from tests.unittest import COVERAGE_RUNNING from tests.utils import test_timeout -logger = logging.getLogger(__name__) - class WorkerLockTestCase(unittest.HomeserverTestCase): def prepare( @@ -150,41 +144,19 @@ def _pump_by( while self.reactor.seconds() < end_time_s: self.reactor.advance(by.as_secs()) - # We do not want to run this test while collecting coverage. Some environments are - # resource constrained and this test will flake/fail as it is a stress test and not - # a compliance test. The coverage included from this test is found in other tests. - @unittest.skip_unless( - not COVERAGE_RUNNING, - "Test requested to not be run when collecting coverage data.", - ) def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 - current_machine = platform.machine().lower() - if current_machine.startswith("riscv"): - # RISC-V specific settings - timeout_seconds = 15 # Increased timeout for RISC-V - # add a print or log statement here for visibility in CI logs - logger.info( # use logger.info - "Detected RISC-V architecture (%s). " - "Adjusting test_lock_contention: timeout=%ss", - current_machine, - timeout_seconds, - ) - else: - # Settings for other architectures. The 5 seconds originally given to this - # test was insufficient when under load. Sometimes it would fail on CI. For - # lack of a better/reliable Twisted mechanism option, this uses signal which - # watches wallclock time instead of Twisted's simulated time. - timeout_seconds = 15 - # It takes around 0.5s on a 5+ years old laptop - with test_timeout(timeout_seconds): # Use the dynamically set timeout - d = self._take_locks( - nb_locks_to_test - ) # Use the (potentially adjusted) number of locks - self.assertEqual( - self.get_success(d), nb_locks_to_test - ) # Assert against the used number of locks + + # This test is a performance-regression canary: before #16840 taking the + # locks below spent ~30s spinning the CPU, afterwards ~0.5s. We budget + # CPU time rather than wall-clock time so that time spent waiting on + # database round-trips (significant on PostgreSQL) or lost to a loaded + # CI machine doesn't make the test flaky: a healthy run costs well + # under 1s of CPU on either database engine. + with test_timeout(5, cpu_time=True): + d = self._take_locks(nb_locks_to_test) + self.assertEqual(self.get_success(d), nb_locks_to_test) async def _take_locks(self, nb_locks: int) -> int: locks = [ diff --git a/tests/http/test_site.py b/tests/http/test_site.py index 654ec3190b..c93aadfee1 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -19,6 +19,8 @@ # # +from parameterized import parameterized + from twisted.internet.address import IPv6Address from twisted.internet.testing import MemoryReactor, StringTransport @@ -92,7 +94,16 @@ def test_large_request(self) -> None: # that. self.assertEqual(sent, 50 * 1024 * 1024 + 1024) - def test_content_type_multipart(self) -> None: + @parameterized.expand( + [ + (b"multipart/form-data",), + # Also check with a boundary + (b"multipart/form-data; boundary=abc123",), + # Headers are case-insensitive, so test that too. + (b"Multipart/Form-Data",), + ] + ) + def test_content_type_multipart(self, content_type: bytes) -> None: """HTTP POST requests with `content-type: multipart/form-data` should be rejected""" self.hs.start_listening() @@ -133,7 +144,7 @@ def test_content_type_multipart(self) -> None: b"POST / HTTP/1.1\r\n" b"Connection: close\r\n" b"Transfer-Encoding: chunked\r\n" - b"Content-Type: multipart/form-data\r\n" + b"Content-Type: " + content_type + b"\r\n" b"\r\n" b"0\r\n" b"\r\n" diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f25b507aac..855a623ec0 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -132,12 +132,7 @@ async def test_ensure_media() -> None: # This uses a real blocking threadpool so we have to wait for it to be # actually done :/ - x = defer.ensureDeferred(test_ensure_media()) - - # Hotloop until the threadpool does its job... - self.wait_on_thread(x) - - self.get_success(x) + self.get_success(test_ensure_media()) @attr.s(auto_attribs=True, slots=True, frozen=True) diff --git a/tests/push/test_http.py b/tests/push/test_http.py index ca2ced01ed..47521a773f 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -25,12 +25,13 @@ from twisted.internet.defer import Deferred from twisted.internet.testing import MemoryReactor -import synapse.rest.admin from synapse.logging.context import make_deferred_yieldable from synapse.push import PusherConfig, PusherConfigException +from synapse.rest import admin from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import login, push_rule, pusher, receipts, room, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -39,7 +40,7 @@ class HTTPPusherTests(HomeserverTestCase): servlets = [ - synapse.rest.admin.register_servlets_for_client_rest_resource, + admin.register_servlets_for_client_rest_resource, room.register_servlets, login.register_servlets, receipts.register_servlets, @@ -1024,33 +1025,6 @@ def test_device_id_feature_flag(self) -> None: lookup_result.device_id, ) - def test_msc3881_client_versions_flag(self) -> None: - """Tests that MSC3881 only appears in /versions if user has it enabled.""" - - user_id = self.register_user("user", "pass") - access_token = self.login("user", "pass") - - # Check feature is disabled in /versions - channel = self.make_request( - "GET", "/_matrix/client/versions", access_token=access_token - ) - self.assertEqual(channel.code, 200) - self.assertFalse(channel.json_body["unstable_features"]["org.matrix.msc3881"]) - - # Enable feature for user - self.get_success( - self.hs.get_datastores().main.set_features_for_user( - user_id, {ExperimentalFeature.MSC3881: True} - ) - ) - - # Check feature is now enabled in /versions for user - channel = self.make_request( - "GET", "/_matrix/client/versions", access_token=access_token - ) - self.assertEqual(channel.code, 200) - self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc3881"]) - @override_config({"push": {"jitter_delay": "10s"}}) def test_jitter(self) -> None: """Tests that enabling jitter actually delays sending push.""" @@ -1245,3 +1219,71 @@ def test_push_backoff(self) -> None: self.push_attempts[3][2]["notification"]["content"]["body"], "Message 3" ) self.push_attempts[3][0].callback({}) + + +class MSC3881VersionsTestCase(HomeserverTestCase): + """ + Tests that MSC3881 (Remotely toggle push notifications for another client) support + is correctly advertised in /versions. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + versions.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def test_msc3881_client_versions_flag(self) -> None: + """Tests that MSC3881 only appears in /versions if user has it enabled.""" + + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Check feature is disabled in /versions + channel = self.make_request( + "GET", "/_matrix/client/versions", access_token=access_token + ) + self.assertEqual(channel.code, 200) + self.assertFalse(channel.json_body["unstable_features"]["org.matrix.msc3881"]) + + # Enable feature for user + self.get_success( + self.hs.get_datastores().main.set_features_for_user( + user_id, {ExperimentalFeature.MSC3881: True} + ) + ) + + # Check feature is now enabled in /versions for user + channel = self.make_request( + "GET", "/_matrix/client/versions", access_token=access_token + ) + self.assertEqual(channel.code, 200) + self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc3881"]) diff --git a/tests/replication/tcp/test_handler.py b/tests/replication/tcp/test_handler.py index a8eb7fc523..d35191e654 100644 --- a/tests/replication/tcp/test_handler.py +++ b/tests/replication/tcp/test_handler.py @@ -147,6 +147,12 @@ def test_wait_for_stream_position(self) -> None: # ... but worker1 finishing (and so sending an update) should. self.get_success(ctx_worker1.__aexit__(None, None, None)) + # Wait for the stream position to be replicated to the master process + # + # Replication travels over `FakeTransport` and we're specifically flushing the + # write + self.reactor.advance(0) + self.assertTrue(d.called) def test_wait_for_stream_position_rdata(self) -> None: @@ -206,6 +212,12 @@ def test_wait_for_stream_position_rdata(self) -> None: # Finish the context manager, triggering the data to be sent to master. self.get_success(ctx_worker1.__aexit__(None, None, None)) + # Wait for the stream position to be replicated to the master process + # + # Replication travels over `FakeTransport` and we're specifically flushing the + # write + self.reactor.advance(0) + # Master should get told about `next_token2`, so the deferred should # resolve. self.assertTrue(d.called) diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py index e6b9ea5383..c8de7b1fad 100644 --- a/tests/replication/test_federation_ack.py +++ b/tests/replication/test_federation_ack.py @@ -81,6 +81,14 @@ def test_federation_ack_sent(self) -> None: ) ) + # Wait for the FEDERATION_ACK to be sent + # + # `on_rdata` handles this as part of a fire-and-forget background process (see + # `FederationSenderHandler.update_token`) + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + # now check that the FEDERATION_ACK was sent mock_connection.send_command.assert_called_once() cmd = mock_connection.send_command.call_args[0][0] diff --git a/tests/rest/admin/test_jwks.py b/tests/rest/admin/test_jwks.py deleted file mode 100644 index ee5588951b..0000000000 --- a/tests/rest/admin/test_jwks.py +++ /dev/null @@ -1,106 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2023 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - - -from twisted.web.resource import Resource - -from synapse.rest.synapse.client import build_synapse_client_resource_tree - -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - - -@skip_unless(HAS_AUTHLIB, "requires authlib") -class JWKSTestCase(HomeserverTestCase): - """Test /_synapse/jwks JWKS data.""" - - def create_resource_dict(self) -> dict[str, Resource]: - d = super().create_resource_dict() - d.update(build_synapse_client_resource_tree(self.hs)) - return d - - def test_empty_jwks(self) -> None: - """Test that the JWKS endpoint is not present by default.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(404, channel.code, channel.result) - - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer/", - "client_id": "test-client-id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - }, - }, - } - ) - def test_empty_jwks_for_msc3861_client_secret_post(self) -> None: - """Test that the JWKS endpoint is empty when plain auth is used.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(200, channel.code, channel.result) - self.assertEqual({"keys": []}, channel.json_body) - - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer/", - "client_id": "test-client-id", - "client_auth_method": "private_key_jwt", - "jwk": { - "p": "-frVdP_tZ-J_nIR6HNMDq1N7aunwm51nAqNnhqIyuA8ikx7LlQED1tt2LD3YEvYyW8nxE2V95HlCRZXQPMiRJBFOsbmYkzl2t-MpavTaObB_fct_JqcRtdXddg4-_ihdjRDwUOreq_dpWh6MIKsC3UyekfkHmeEJg5YpOTL15j8", - "kty": "RSA", - "q": "oFw-Enr_YozQB1ab-kawn4jY3yHi8B1nSmYT0s8oTCflrmps5BFJfCkHL5ij3iY15z0o2m0N-jjB1oSJ98O4RayEEYNQlHnTNTl0kRIWzpoqblHUIxVcahIpP_xTovBJzwi8XXoLGqHOOMA-r40LSyVgP2Ut8D9qBwV6_UfT0LU", - "d": "WFkDPYo4b4LIS64D_QtQfGGuAObPvc3HFfp9VZXyq3SJR58XZRHE0jqtlEMNHhOTgbMYS3w8nxPQ_qVzY-5hs4fIanwvB64mAoOGl0qMHO65DTD_WsGFwzYClJPBVniavkLE2Hmpu8IGe6lGliN8vREC6_4t69liY-XcN_ECboVtC2behKkLOEASOIMuS7YcKAhTJFJwkl1dqDlliEn5A4u4xy7nuWQz3juB1OFdKlwGA5dfhDNglhoLIwNnkLsUPPFO-WB5ZNEW35xxHOToxj4bShvDuanVA6mJPtTKjz0XibjB36bj_nF_j7EtbE2PdGJ2KevAVgElR4lqS4ISgQ", - "e": "AQAB", - "kid": "test", - "qi": "cPfNk8l8W5exVNNea4d7QZZ8Qr8LgHghypYAxz8PQh1fNa8Ya1SNUDVzC2iHHhszxxA0vB9C7jGze8dBrvnzWYF1XvQcqNIVVgHhD57R1Nm3dj2NoHIKe0Cu4bCUtP8xnZQUN4KX7y4IIcgRcBWG1hT6DEYZ4BxqicnBXXNXAUI", - "dp": "dKlMHvslV1sMBQaKWpNb3gPq0B13TZhqr3-E2_8sPlvJ3fD8P4CmwwnOn50JDuhY3h9jY5L06sBwXjspYISVv8hX-ndMLkEeF3lrJeA5S70D8rgakfZcPIkffm3tlf1Ok3v5OzoxSv3-67Df4osMniyYwDUBCB5Oq1tTx77xpU8", - "dq": "S4ooU1xNYYcjl9FcuJEEMqKsRrAXzzSKq6laPTwIp5dDwt2vXeAm1a4eDHXC-6rUSZGt5PbqVqzV4s-cjnJMI8YYkIdjNg4NSE1Ac_YpeDl3M3Colb5CQlU7yUB7xY2bt0NOOFp9UJZYJrOo09mFMGjy5eorsbitoZEbVqS3SuE", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - }, - }, - }, - } - ) - def test_key_returned_for_msc3861_client_secret_post(self) -> None: - """Test that the JWKS includes public part of JWK for private_key_jwt auth is used.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(200, channel.code, channel.result) - self.assertEqual( - { - "keys": [ - { - "kty": "RSA", - "e": "AQAB", - "kid": "test", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - } - ] - }, - channel.json_body, - ) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 4ee9fba158..bce199c564 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -59,7 +59,7 @@ from synapse.server import HomeServer from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY from synapse.types import JsonDict, UserID, create_requester -from synapse.util.clock import Clock +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -5334,6 +5334,69 @@ def test_redact_messages_all_rooms(self) -> None: matched.append(event_id) self.assertEqual(len(matched), len(originals)) + def test_redact_messages_all_rooms_within_timeframe(self) -> None: + """ + Test that request to redact user's events in all rooms within a specific timeframe is successful + """ + # join rooms, send some messages + + # (event_id, timestamp) pairs + all_message_ids: list[tuple[str, int]] = [] + for rm in [self.rm1, self.rm2, self.rm3]: + self.helper.join(rm, self.bad_user, tok=self.bad_user_tok) + + for i in range(4): + for rm in [self.rm1, self.rm2, self.rm3]: + event = {"body": f"hello{i}", "msgtype": "m.text"} + res = self.helper.send_event( + rm, "m.room.message", event, tok=self.bad_user_tok, expect_code=200 + ) + event_id = res["event_id"] + event_ts = self.get_success( + self.store.get_event(event_id) + ).origin_server_ts + all_message_ids.append((event_id, event_ts)) + + expected_saved_message_ids = { + event_id for event_id, _ in all_message_ids[:5] + all_message_ids[10:] + } + expected_redacted_message_ids = { + event_id for event_id, _ in all_message_ids[5:10] + } + + # Redact events 5 up to and including 9 + _after_event_id, after_ts = all_message_ids[5] + _before_event_id, before_ts = all_message_ids[9] + + # redact events in all rooms within specific timeframe + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": [], "after_ts": after_ts, "before_ts": before_ts}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + # Get the set of all redacted event IDs + all_redacted_event_ids: set[str] = set() + for rm in [self.rm1, self.rm2, self.rm3]: + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{rm}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + # Get the IDs of all redacted events + for event in channel.json_body["chunk"]: + assert event["type"] == EventTypes.Redaction + all_redacted_event_ids.add(event["redacts"]) + + # check that only expected messages were redacted + self.assertSetEqual(expected_redacted_message_ids, all_redacted_event_ids) + self.assertSetEqual(expected_saved_message_ids & all_redacted_event_ids, set()) + def test_redact_messages_specific_rooms(self) -> None: """ Test that request to redact events in specified rooms user is member of is successful @@ -5787,23 +5850,25 @@ def test_redact_messages_all_rooms(self) -> None: self.assertEqual(channel.code, 200) id = channel.json_body.get("redact_id") - timeout_s = 10 - start_time = self.clock.time() - redact_result = "" - while redact_result != "complete": - if start_time + timeout_s < self.clock.time(): - self.fail("Timed out waiting for redactions.") - - # It's a background task, give it a shot at actually finishing - self.pump() - channel2 = self.make_request( - "GET", - f"/_synapse/admin/v1/user/redact_status/{id}", - access_token=self.admin_tok, - ) - redact_result = channel2.json_body["status"] - if redact_result == "failed": - self.fail("Redaction task failed.") + # `/redact` just schedules a background task that runs in the background + # (fire-and-forget) so we need to do the waiting here. + # + # Need 1 tick as we send 1 replication request for the redaction of each + # original event. The replication request body is streamed by a `Cooperator` + # that uses the clock to schedule each chunk at a tiny *non-zero* delay + # (`CLOCK_SCHEDULE_EPSILON`), so we need to actually advance the clock for it to + # fire. + for _ in range(len(original_event_ids)): + self.reactor.advance(CLOCK_SCHEDULE_EPSILON.as_secs()) + + # Verify the HTTP `redact_status` endpoint reports completion. + channel2 = self.make_request( + "GET", + f"/_synapse/admin/v1/user/redact_status/{id}", + access_token=self.admin_tok, + ) + self.assertEqual(channel2.code, 200) + self.assertEqual(channel2.json_body["status"], "complete") redaction_ids = set() for rm in [self.rm1, self.rm2, self.rm3]: diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index b1b771ef84..93fdfa59f9 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -13,6 +13,7 @@ # import logging from typing import Any +from unittest.mock import patch from parameterized import parameterized, parameterized_class @@ -26,7 +27,11 @@ from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase -from tests.test_utils.event_injection import create_event +from tests.test_utils.event_injection import ( + create_event, + inject_event, + inject_member_event, +) logger = logging.getLogger(__name__) @@ -1407,3 +1412,239 @@ def test_old_room_with_unknown_room_version(self) -> None: } } response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_name(self, non_string_name: object) -> None: + """ + Test that when the room name is not a string, it gets + treated the same as if there is no room name set; + the `name` field is omitted and `heroes` are populated instead. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # For heroes to be emitted, we need a second user + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + self.helper.join(room_id, user2_id, tok=user2_tok) + + # Set the room name to a non-string + # Need to patch out our client-sent event checks to do this. + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.Name, + state_key="", + content={"name": non_string_name}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The name should be omitted (non-string value treated as unset). + self.assertNotIn( + "name", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + # Since there is no name, heroes should be populated. + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "user2", "user_id": "@user2:test"}], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_avatar(self, non_string_avatar: str) -> None: + """ + Test that when the room avatar is not a string, it gets + treated the same as if there is no room avatar set; + the `avatar` field is omitted. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Set the room avatar to a dict (non-string) instead of a URL string. + # Need to patch out our client-sent event checks to do this + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.RoomAvatar, + state_key="", + content={"url": non_string_avatar}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The avatar should be omitted (non-string value treated as unset). + self.assertNotIn( + "avatar", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_displayname( + self, non_string_name: str + ) -> None: + """ + Test that when a hero's displayname is not a string, it gets + treated the same as if there is no displayname set: + the `displayname` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Inject a membership event for user2 with a non-string displayname. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": non_string_name, + "avatar_url": "mxc://example.org/a-real-mxc", + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn( + "name", response_body["rooms"][room_id], response_body["rooms"][room_id] + ) + + # user2 should be in the heroes list, but without a displayname + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [ + { + "avatar_url": "mxc://example.org/a-real-mxc", + "user_id": "@user2:test", + } + ], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_avatar_url( + self, non_string_avatar: str + ) -> None: + """ + Test that when a hero's avatar URL is not a string, it gets + treated the same as if there is no avatar URL set: + the `avatar_url` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Inject a membership event for user2 with a non-string avatar_url. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": "second user", + "avatar_url": non_string_avatar, + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn("name", response_body["rooms"][room_id]) + + # user2 should be in the heroes list, but without an avatar + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "second user", "user_id": "@user2:test"}], + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index 2fd18f0e54..db1d4ac9c9 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -564,9 +564,16 @@ def test_wait_for_sync_token(self) -> None: ) # Block for 10 seconds to make `notifier.wait_for_stream_token(from_token)` # timeout + # + # First, block for *almost* 10 seconds to make sure we are + # `notifier.wait_for_stream_token(from_token)` with self.assertRaises(TimedOutException): channel.await_result(timeout_ms=9900) - channel.await_result(timeout_ms=200) + # Then wait for the rest of the 10 second timeout, 9900 + 500 > 10000 + # + # `notifier.wait_for_stream_token(from_token)` only checks every 500ms so we + # need to match that in order to make sure we hit the wake-up for sure. + channel.await_result(timeout_ms=500) self.assertEqual(channel.code, 200, channel.json_body) # We expect the next `pos` in the result to be the same as what we requested diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 42102230f0..c0494c606f 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -325,6 +325,7 @@ def test_password_reset_bad_email_inhibit_error(self) -> None: email = "test@example.com" client_secret = "foobar" + session_id = self._request_token(email, client_secret) self.assertIsNotNone(session_id) @@ -368,12 +369,17 @@ def _request_token( body = {"client_secret": client_secret, "email": email, "send_attempt": 1} if next_link is not None: body["next_link"] = next_link + channel = self.make_request( "POST", b"account/password/email/requestToken", body, client_ip=ip, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) if channel.code != 200: raise HttpResponseException( diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py index c13d410636..e9f8597c5a 100644 --- a/tests/rest/client/test_auth_metadata.py +++ b/tests/rest/client/test_auth_metadata.py @@ -19,16 +19,12 @@ # from http import HTTPStatus from typing import ClassVar -from unittest.mock import AsyncMock from parameterized import parameterized_class from synapse.rest.client import auth_metadata -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - -ISSUER = "https://account.example.com/" +from tests.unittest import HomeserverTestCase class AuthIssuerTestCase(HomeserverTestCase): @@ -36,7 +32,7 @@ class AuthIssuerTestCase(HomeserverTestCase): auth_metadata.register_servlets, ] - def test_returns_404_when_msc3861_disabled(self) -> None: + def test_returns_404_when_mas_disabled(self) -> None: # Make an unauthenticated request for the discovery info. channel = self.make_request( "GET", @@ -44,49 +40,6 @@ def test_returns_404_when_msc3861_disabled(self) -> None: ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock(return_value={"issuer": ISSUER}) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - - req_mock.assert_called_with( - "https://account.example.com/.well-known/openid-configuration" - ) - req_mock.reset_mock() - - # Second call it should use the cached value - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - req_mock.assert_not_called() - @parameterized_class( ("endpoint",), @@ -101,45 +54,7 @@ class AuthMetadataTestCase(HomeserverTestCase): auth_metadata.register_servlets, ] - def test_returns_404_when_msc3861_disabled(self) -> None: + def test_returns_404_when_mas_disabled(self) -> None: # Make an unauthenticated request for the discovery info. channel = self.make_request("GET", self.endpoint) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock( - return_value={ - "issuer": ISSUER, - "authorization_endpoint": "https://example.com/auth", - "token_endpoint": "https://example.com/token", - } - ) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request("GET", self.endpoint) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual( - channel.json_body, - { - "issuer": ISSUER, - "authorization_endpoint": "https://example.com/auth", - "token_endpoint": "https://example.com/token", - }, - ) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index c28e0605b5..42926c4359 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -26,6 +26,7 @@ from synapse.rest.client import capabilities, login from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config, skip_unless @@ -203,6 +204,43 @@ def test_get_set_avatar_url_capabilities_avatar_url_disabled_msc4133(self) -> No ["avatar_url"], ) + def test_get_delayed_events_capabilities_default_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], 0 + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 100 + ) + + @override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 50, + }, + } + ) + def test_get_delayed_events_capabilities_custom_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], + Duration(days=1).as_millis(), + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 50 + ) + @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: """Test if change 3pid is disabled that the server responds it.""" diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index c3bfdf7c8d..75d716244a 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -24,8 +24,10 @@ from synapse.rest import admin from synapse.rest.client import delayed_events, login, room, sync, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.server import FakeChannel @@ -39,6 +41,34 @@ class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): servlets = [versions.register_servlets] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def test_false_by_default(self) -> None: channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) @@ -90,6 +120,10 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: room=self.room_id, user=self.user2_user_id, tok=self.user2_access_token ) + # Advance enough time where any requests we made during `prepare(...)` doesn't + # affect the rate-limits in the test itself + self.reactor.advance(Duration(days=1).as_secs()) + def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index 2cf293a962..47026cc0e8 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -212,52 +212,57 @@ def test_dehydrate_msc3814(self) -> None: ) requester = create_requester(user, device_id=new_device_id) - # Send a message to the dehydrated device - ensureDeferred( - self.message_handler.send_device_message( - requester=requester, - message_type="test.message", - messages={user: {device_id: {"body": "test_message"}}}, + # Send enough messages to the dehydrated device that we need 2 batches + for _ in range(110): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user: {device_id: {"body": "test_message"}}}, + ) ) - ) self.pump() - # make sure we can fetch the message with our dehydrated device id + # make sure we can fetch the first batch with our dehydrated device id channel = self.make_request( - "POST", + "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content={}, access_token=token, shorthand=False, ) self.assertEqual(channel.code, 200) expected_content = {"body": "test_message"} self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) - # fetch messages again and make sure that the message was not deleted + # fetch the same messages again to prove that the messages were not deleted channel = self.make_request( - "POST", + "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content={}, access_token=token, shorthand=False, ) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) next_batch_token = channel.json_body.get("next_batch") - # make sure fetching messages with next batch token works - there are no unfetched - # messages so we should receive an empty array - content = {"next_batch": next_batch_token} + # There are more messages to come + self.assertNotEqual(next_batch_token, None) + + # make sure fetching messages with next batch token works channel = self.make_request( - "POST", - f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content=content, + "GET", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events?from={next_batch_token}", access_token=token, shorthand=False, ) self.assertEqual(channel.code, 200) - self.assertEqual(channel.json_body["events"], []) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 10) + + # Now, there are no more messages + self.assertNotIn("next_batch", channel.json_body) # make sure we can delete the dehydrated device channel = self.make_request( @@ -277,6 +282,207 @@ def test_dehydrate_msc3814(self) -> None: ) self.assertEqual(channel.code, 401) + @unittest.override_config({"experimental_features": {"msc3814_enabled": True}}) + def test_dehydrate_msc3814_legacy_post(self) -> None: + """ + This tests the legacy POST API that was originally used in MSC3814. + """ + + user = self.register_user("mikey", "pass") + token = self.login(user, "pass", device_id="device1") + content: JsonDict = { + "device_data": { + "algorithm": "m.dehydration.v1.olm", + }, + "device_id": "device1", + "initial_device_display_name": "foo bar", + "device_keys": { + "user_id": "@mikey:test", + "device_id": "device1", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], + "keys": { + ":": "", + }, + "signatures": { + "": {":": ""} + }, + }, + "fallback_keys": { + "alg1:device1": "f4llb4ckk3y", + "signed_:": { + "fallback": "true", + "key": "f4llb4ckk3y", + "signatures": { + "": {":": ""} + }, + }, + }, + "one_time_keys": {"alg1:k1": "0net1m3k3y"}, + } + channel = self.make_request( + "PUT", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + content=content, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + device_id = channel.json_body.get("device_id") + assert device_id is not None + self.assertIsInstance(device_id, str) + self.assertEqual("device1", device_id) + + # test that we can now GET the dehydrated device info + channel = self.make_request( + "GET", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + returned_device_id = channel.json_body.get("device_id") + self.assertEqual(returned_device_id, device_id) + device_data = channel.json_body.get("device_data") + expected_device_data = { + "algorithm": "m.dehydration.v1.olm", + } + self.assertEqual(device_data, expected_device_data) + + # test that the keys are correctly uploaded + channel = self.make_request( + "POST", + "/_matrix/client/r0/keys/query", + { + "device_keys": { + user: ["device1"], + }, + }, + token, + ) + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body["device_keys"][user][device_id]["keys"], + content["device_keys"]["keys"], + ) + # first claim should return the onetime key we uploaded + res = self.get_success( + self.hs.get_e2e_keys_handler().claim_one_time_keys( + {user: {device_id: {"alg1": 1}}}, + UserID.from_string(user), + timeout=None, + always_include_fallback_keys=False, + ) + ) + self.assertEqual( + res, + { + "failures": {}, + "one_time_keys": {user: {device_id: {"alg1:k1": "0net1m3k3y"}}}, + }, + ) + # second claim should return fallback key + res2 = self.get_success( + self.hs.get_e2e_keys_handler().claim_one_time_keys( + {user: {device_id: {"alg1": 1}}}, + UserID.from_string(user), + timeout=None, + always_include_fallback_keys=False, + ) + ) + self.assertEqual( + res2, + { + "failures": {}, + "one_time_keys": {user: {device_id: {"alg1:device1": "f4llb4ckk3y"}}}, + }, + ) + + # create another device for the user + ( + new_device_id, + _, + _, + _, + ) = self.get_success( + self.registration.register_device( + user_id=user, + device_id=None, + initial_display_name="new device", + ) + ) + requester = create_requester(user, device_id=new_device_id) + + # Send enough messages to the dehydrated device that we need 2 batches + for _ in range(110): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user: {device_id: {"body": "test_message"}}}, + ) + ) + self.pump() + + # make sure we can fetch the first batch with our dehydrated device id + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + expected_content = {"body": "test_message"} + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) + + # fetch the same messages again to prove that the messages were not deleted + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) + next_batch_token = channel.json_body.get("next_batch") + + # There is a next_batch token + self.assertNotEqual(next_batch_token, None) + + # make sure fetching messages with next batch token works + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={"next_batch": next_batch_token}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 10) + next_batch_token = channel.json_body.get("next_batch") + + # There is a next_batch token (even though there are no more messages - this is + # the way the legacy API works) + self.assertNotEqual(next_batch_token, None) + + # Fetch the next batch - it should be empty, indicating we should stop polling + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={"next_batch": next_batch_token}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(len(channel.json_body["events"]), 0) + @unittest.override_config({"experimental_features": {"msc3814_enabled": True}}) def test_msc3814_dehydrated_device_delete_works(self) -> None: user = self.register_user("mikey", "pass") diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index fc21d83ef2..4f3d6a365f 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -18,9 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -import urllib.parse from http import HTTPStatus -from unittest.mock import patch from signedjson.key import ( encode_verify_key_base64, @@ -32,12 +30,10 @@ from synapse.api.errors import Codes from synapse.rest import admin from synapse.rest.client import keys, login -from synapse.types import JsonDict, Requester, create_requester +from synapse.types import JsonDict from tests import unittest from tests.http.server._base import make_request_with_cancellation_test -from tests.unittest import override_config -from tests.utils import HAS_AUTHLIB class KeyUploadTestCase(unittest.HomeserverTestCase): @@ -355,191 +351,3 @@ class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase): OIDC_ADMIN_TOKEN = "_oidc_admin_token" ACCOUNT_MANAGEMENT_URL = "https://my-account.issuer" - - @unittest.skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "enable_registration": False, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "account_management_url": ACCOUNT_MANAGEMENT_URL, - "client_id": "id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - "admin_token": OIDC_ADMIN_TOKEN, - }, - }, - } - ) - def test_master_cross_signing_key_replacement_msc3861(self) -> None: - # Provision a user like MAS would, cribbing from - # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L224-L229 - alice = "@alice:test" - channel = self.make_request( - "PUT", - f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}", - access_token=self.OIDC_ADMIN_TOKEN, - content={}, - ) - self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body) - - # Provision a device like MAS would, cribbing from - # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L260-L262 - alice_device = "alice_device" - channel = self.make_request( - "POST", - f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}/devices", - access_token=self.OIDC_ADMIN_TOKEN, - content={"device_id": alice_device}, - ) - self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body) - - # Prepare a mock MAS access token. - alice_token = "alice_token_1234_oidcwhatyoudidthere" - - async def mocked_get_user_by_access_token( - token: str, allow_expired: bool = False - ) -> Requester: - self.assertEqual(token, alice_token) - return create_requester( - user_id=alice, - device_id=alice_device, - scope=[], - is_guest=False, - ) - - patch_get_user_by_access_token = patch.object( - self.hs.get_auth(), - "get_user_by_access_token", - wraps=mocked_get_user_by_access_token, - ) - - # Copied from E2eKeysHandlerTestCase - master_pubkey = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk" - master_pubkey2 = "fHZ3NPiKxoLQm5OoZbKa99SYxprOjNs4TwJUKP+twCM" - master_pubkey3 = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" - - master_key: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey: master_pubkey}, - } - master_key2: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey2: master_pubkey2}, - } - master_key3: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey3: master_pubkey3}, - } - - with patch_get_user_by_access_token: - # Upload an initial cross-signing key. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key, - }, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Should not be able to upload another master key. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key2, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - # Ensure that the response contains the expected UIA flows from https://spec.matrix.org/v1.17/client-server-api/#oauth-authentication - self.assertIn( - {"stages": ["m.oauth"]}, - channel.json_body["flows"], - "m.oauth flow not found", - ) - self.assertSubstring( - self.ACCOUNT_MANAGEMENT_URL, - channel.json_body["params"]["m.oauth"]["url"], - "m.oauth url does not match account management URL", - ) - self.assertSubstring( - "action=org.matrix.cross_signing_reset", - channel.json_body["params"]["m.oauth"]["url"], - "m.oauth url does not include expected action", - ) - # Unstable version of the flow - self.assertIn( - {"stages": ["org.matrix.cross_signing_reset"]}, - channel.json_body["flows"], - "unstable org.matrix.cross_signing_reset flow not found", - ) - self.assertEqual( - channel.json_body["params"]["org.matrix.cross_signing_reset"]["url"], - channel.json_body["params"]["m.oauth"]["url"], - "unstable org.matrix.cross_signing_reset url does not match m.oauth url", - ) - - # Pretend that MAS did UIA and allowed us to replace the master key. - channel = self.make_request( - "POST", - f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia", - access_token=self.OIDC_ADMIN_TOKEN, - ) - self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) - - with patch_get_user_by_access_token: - # Should now be able to upload master key2. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key2, - }, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Even though we're still in the grace period, we shouldn't be able to - # upload master key 3 immediately after uploading key 2. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key3, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - - # Pretend that MAS did UIA and allowed us to replace the master key. - channel = self.make_request( - "POST", - f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia", - access_token=self.OIDC_ADMIN_TOKEN, - ) - self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) - timestamp_ms = channel.json_body["updatable_without_uia_before_ms"] - - # Advance to 1 second after the replacement period ends. - self.reactor.advance(timestamp_ms - self.clock.time_msec() + 1000) - - with patch_get_user_by_access_token: - # We should not be able to upload master key3 because the replacement has - # expired. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key3, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py index 835336f3d9..d6b2cf054e 100644 --- a/tests/rest/client/test_login_token_request.py +++ b/tests/rest/client/test_login_token_request.py @@ -24,6 +24,7 @@ from synapse.rest import admin from synapse.rest.client import login, login_token_request, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -47,8 +48,31 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: self.hs.config.registration.auto_join_rooms = [] self.hs.config.captcha.enable_registration_captcha = False + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = self.hs.get_proxied_http_client() + _ = HttpClient( + reactor=self.hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + return self.hs + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.user = "user123" self.password = "password" diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index 87969467fb..f2bf1596be 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -22,6 +22,7 @@ from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -111,6 +112,34 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): servlets = [versions.register_servlets] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def test_msc4143_false_by_default(self) -> None: channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/test_msc4388_rendezvous.py b/tests/rest/client/test_msc4388_rendezvous.py index 913a41dedb..f9b7f578f6 100644 --- a/tests/rest/client/test_msc4388_rendezvous.py +++ b/tests/rest/client/test_msc4388_rendezvous.py @@ -175,13 +175,11 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) rendezvous_id = channel.json_body["id"] sequence_token = channel.json_body["sequence_token"] - expires_in_ms = channel.json_body["expires_in_ms"] - self.assertGreater(expires_in_ms, 0) + self.assertGreater(channel.json_body["expires_in_ms"], 0) session_endpoint = rz_endpoint + f"/{rendezvous_id}" # We can get the data back - # Advances clock by 100ms channel = self.make_request( "GET", session_endpoint, @@ -191,10 +189,9 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=bar") self.assertEqual(channel.json_body["sequence_token"], sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 100) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can update the data - # Advances clock by 100ms channel = self.make_request( "PUT", session_endpoint, @@ -207,7 +204,6 @@ def test_rendezvous_public(self) -> None: new_sequence_token = channel.json_body["sequence_token"] # If we try to update it again with the old etag, it should fail - # Advances clock by 100ms channel = self.make_request( "PUT", session_endpoint, @@ -221,7 +217,6 @@ def test_rendezvous_public(self) -> None: ) # We should get the updated data - # Advances clock by 100ms channel = self.make_request( "GET", session_endpoint, @@ -231,7 +226,7 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=baz") self.assertEqual(channel.json_body["sequence_token"], new_sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 400) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can delete the data channel = self.make_request( @@ -252,6 +247,77 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + @override_config( + { + "disable_registration": True, + "matrix_authentication_service": { + "enabled": True, + "secret": "secret_value", + "endpoint": "https://issuer", + }, + "experimental_features": { + "msc4388_mode": "open", + }, + } + ) + def test_rendezvous_put_is_idempotent(self) -> None: + """ + A PUT using the previous sequence_token but with data that already + matches what is currently stored should be treated as an idempotent + retry and succeed (rather than returning 409). This lets clients + safely retry a PUT after a network error without losing the session. + """ + channel = self.make_request( + "POST", + rz_endpoint, + {"data": "foo=bar"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + rendezvous_id = channel.json_body["id"] + initial_sequence_token = channel.json_body["sequence_token"] + session_endpoint = rz_endpoint + f"/{rendezvous_id}" + + # Perform an update. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "foo=baz"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + updated_sequence_token = channel.json_body["sequence_token"] + + # Replaying the same PUT with the previous (now-stale) sequence_token + # and matching data should succeed and return the current token. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "foo=baz"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["sequence_token"], updated_sequence_token) + + # But replaying with the previous token and *different* data must + # still be rejected as a concurrent write. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "something=else"}, + access_token=None, + ) + self.assertEqual(channel.code, 409) + self.assertEqual( + channel.json_body["errcode"], "IO_ELEMENT_MSC4388_CONCURRENT_WRITE" + ) + + # The stored data should be unchanged. + channel = self.make_request("GET", session_endpoint, access_token=None) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["data"], "foo=baz") + self.assertEqual(channel.json_body["sequence_token"], updated_sequence_token) + @override_config( { "disable_registration": True, @@ -305,8 +371,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) rendezvous_id = channel.json_body["id"] sequence_token = channel.json_body["sequence_token"] - expires_in_ms = channel.json_body["expires_in_ms"] - self.assertEqual(expires_in_ms, 120000) + self.assertGreater(channel.json_body["expires_in_ms"], 0) session_endpoint = rz_endpoint + f"/{rendezvous_id}" @@ -320,7 +385,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=bar") self.assertEqual(channel.json_body["sequence_token"], sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 100) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can update the data without authentication channel = self.make_request( @@ -343,7 +408,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=baz") self.assertEqual(channel.json_body["sequence_token"], new_sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 300) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can delete the data without authentication channel = self.make_request( diff --git a/tests/rest/client/test_push_rule_attrs.py b/tests/rest/client/test_push_rule_attrs.py index 53c36b7a9c..f6bbc7496e 100644 --- a/tests/rest/client/test_push_rule_attrs.py +++ b/tests/rest/client/test_push_rule_attrs.py @@ -3,6 +3,7 @@ # # Copyright 2020 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -20,9 +21,13 @@ # from http import HTTPStatus +import canonicaljson +from parameterized.parameterized import parameterized + import synapse from synapse.api.errors import Codes from synapse.rest.client import login, push_rule, room +from synapse.types import JsonDict from tests.unittest import HomeserverTestCase @@ -508,3 +513,191 @@ def test_no_user_defined_postcontent_rules(self) -> None: Codes.INVALID_PARAM, channel.json_body["errcode"], ) + + +class PushRuleLimitTestCase(HomeserverTestCase): + """ + Tests for server-configured limits on push rule size. + + See: https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9 + """ + + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + push_rule.register_servlets, + ] + hijack_auth = False + + def default_config(self) -> JsonDict: + config = super().default_config() + # Set some small limits for push rule sizes so that + # we can easily test them. + config["push_rules"] = { + "limits": { + # Size limit of each rule in bytes (canonical JSON) + "rule_size": 64, + # Size limit of the rule ID (in bytes) + "rule_id_length": 24, + # Limit on how many rules you can have + "rule_count": 2, + } + } + return config + + @parameterized.expand( + ( + ( + { + "actions": ["notify"], + "conditions": [{"kind": "event_match", "key": "a", "pattern": "a"}], + }, + 58, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefg"} + ], + }, + 64, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefgH"} + ], + }, + 65, + False, + ), + ) + ) + def test_limit_on_push_rule_size( + self, body: JsonDict, expected_body_num_bytes: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_size` applies to the size of the push rule. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + # Sanity check the test data: the canonical JSON size of the push rule body + # should be exactly as we expect, otherwise our test is void. + body_bytes = canonicaljson.encode_canonical_json(body) + # We exclude the size of the wrapper, as our implementation currently only + # counts the size of the actions and conditions fragments themselves. + self.assertEqual( + len(body_bytes) - len('{"actions":,"conditions":}'.encode("utf-8")), + expected_body_num_bytes, + ) + + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule1", + body, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ({body_bytes!r}) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ({body_bytes!r}) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + @parameterized.expand( + ( + ( + 18, + True, + ), + ( + 24, + True, + ), + ( + 25, + False, + ), + ) + ) + def test_limit_on_rule_id_length( + self, rule_id_length: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_id_length` applies to the byte length + of the push rule ID. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + PREFIX = "global/underride/" + rule_suffix_length = rule_id_length - len(PREFIX) + assert rule_suffix_length >= 1, "can't construct a rule ID that short" + channel = self.make_request( + "PUT", + f"/pushrules/{PREFIX}{rule_suffix_length * 'a'}", + {"conditions": [], "actions": ["notify"]}, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ID ({rule_id_length} B) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + "Push rule ID ({rule_id_length} B) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + def test_limit_on_push_rule_count(self) -> None: + """ + Tests that we are allowed to create exactly the number of push rules + specified by `push_rules.limits.rule_count`, but not a single one more. + """ + self.register_user("bob", "pass") + token = self.login("bob", "pass") + + # First 2 rules are allowable + for i in range(2): + channel = self.make_request( + "PUT", + f"/pushrules/global/underride/rule{i}", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule within count limit should be allowed: {channel.json_body}", + ) + + # 3rd rule gets denied as it goes over the limit + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule3", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.BAD_REQUEST, + "Push rule exceeding count limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) diff --git a/tests/rest/client/test_read_marker.py b/tests/rest/client/test_read_marker.py index c8bb0da5e6..ad13d3607e 100644 --- a/tests/rest/client/test_read_marker.py +++ b/tests/rest/client/test_read_marker.py @@ -66,6 +66,19 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() + def _get_fully_read_marker(self, room_id: str) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.owner, + room_id, + "m.fully_read", + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_read_marker(self) -> None: room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) @@ -98,6 +111,123 @@ def send_message() -> str: ) self.assertEqual(channel.code, 200, channel.result) + def test_send_read_marker_does_not_move_backwards_by_default(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_can_move_backwards_with_opt_in(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={ + "m.fully_read": older_event_id, + "com.beeper.allow_backward": False, + }, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + def test_send_read_marker_ignores_opt_in_when_feature_disabled(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + def test_send_read_marker_missing_previous_event(self) -> None: """ Test moving a read marker from an event that previously existed but was diff --git a/tests/rest/client/test_receipts.py b/tests/rest/client/test_receipts.py index 3a6a869c54..0835eec6de 100644 --- a/tests/rest/client/test_receipts.py +++ b/tests/rest/client/test_receipts.py @@ -24,6 +24,7 @@ import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, ReceiptTypes +from synapse.api.errors import Codes from synapse.rest.client import login, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -44,6 +45,7 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.url = "/sync?since=%s" self.next_batch = "s0" + self.store = hs.get_datastores().main # Register the first user self.user_id = self.register_user("kermit", "monkey") @@ -59,6 +61,19 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Join the second user self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + def _get_fully_read_marker(self) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.user2, + self.room_id, + ReceiptTypes.FULLY_READ, + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_receipt(self) -> None: # Send a message. res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -258,6 +273,126 @@ def test_read_receipt_with_empty_body_is_rejected(self) -> None: self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON", channel.json_body) + def test_fully_read_receipt_does_not_move_backwards_by_default(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_can_move_backwards_with_opt_in(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": False}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_allow_backward_is_rejected_for_read_receipts(self) -> None: + event_id = self.helper.send(self.room_id, body="1", tok=self.tok)["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_allow_backward_is_ignored_when_feature_disabled(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + def _get_read_receipt(self) -> JsonDict | None: """Syncs and returns the read receipt.""" diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index f66c56a6b1..1b04ea9d5c 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -753,7 +753,11 @@ def test_request_token_existing_email_inhibit_error(self) -> None: "POST", b"register/email/requestToken", {"client_secret": "foobar", "email": email, "send_attempt": 1}, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) self.assertEqual(200, channel.code, channel.result) self.assertIsNotNone(channel.json_body.get("sid")) diff --git a/tests/rest/client/test_rendezvous.py b/tests/rest/client/test_rendezvous.py index dc4f833fa2..c2bd7d1573 100644 --- a/tests/rest/client/test_rendezvous.py +++ b/tests/rest/client/test_rendezvous.py @@ -61,14 +61,11 @@ def test_disabled(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_delegation_endpoint": "https://asd", - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -83,14 +80,11 @@ def test_msc4108_delegation(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -227,14 +221,11 @@ def test_msc4108(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -279,14 +270,11 @@ def test_msc4108_expiration(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -351,14 +339,11 @@ def test_msc4108_capacity(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -415,14 +400,11 @@ def test_msc4108_hard_capacity(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 8d4892ae91..ca95a3a099 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -26,12 +26,14 @@ import json from http import HTTPStatus from typing import Any, Iterable, Literal -from unittest.mock import AsyncMock, Mock, call, patch +from unittest.mock import AsyncMock, Mock, call, create_autospec, patch from urllib import parse as urlparse from parameterized import param, parameterized +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent import synapse.rest.admin from synapse.api.constants import ( @@ -61,11 +63,13 @@ from synapse.server import HomeServer from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase +from tests.test_utils import FakeResponse from tests.test_utils.event_injection import ( create_event, inject_event, @@ -2503,7 +2507,12 @@ def test_send_delayed_invalid_event(self) -> None: {}, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) - self.assertNotIn("org.matrix.msc4140.errcode", channel.json_body) + # Assert that the standard error response uses a valid errcode. + # The specific errcode is irrelevant for the purpose of this test. + self.assertIsInstance( + channel.json_body.get("errcode"), + str, + ) def test_delayed_event_unsupported_by_default(self) -> None: """Test that sending a delayed event is unsupported with the default config.""" @@ -2515,10 +2524,35 @@ def test_delayed_event_unsupported_by_default(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_UNSUPPORTED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 0, + }, + } + ) + def test_delayed_event_disabled_by_limit(self) -> None: + """Test that delayed events are disabled by configuring the per-user limit to 0.""" + channel = self.make_request( + "PUT", + ( + "rooms/%s/send/m.room.message/mid1?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), channel.json_body, ) @@ -2533,12 +2567,177 @@ def test_delayed_event_exceeds_max_delay(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_user_limit_reached(self) -> None: + """Test that users cannot have more delayed events scheduled at once than allowed.""" + # Disable rate-limits for this user. We want to specifically test the storage-based limit, not the request limits + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=15000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + # Send a delayed event to eat up the limit + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Try to send another delayed event (we expect to hit the limit on the max number of delayed events that can be scheduled at once) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_EXCEEDED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], channel.json_body, ) + # Confirm that the response includes the time remaining until the next of the user's + # delayed events to be sent, at which point another delayed event may be scheduled + # without exceeding the limit + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert retry_after_headers + retry_after_sec = int(retry_after_headers[0]) + self.assertGreater(retry_after_sec, 0) + # Confirm that there is only a single value to the Retry-After header, as per RFC9110 + self.assertEqual(1, len(retry_after_headers)) + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_processed_user_limit_reached(self) -> None: + """ + Test that delayed events in the midst of being sent still count towards the limit of + how many delayed events a user may have scheduled at once. + """ + send_after = Duration(seconds=1) + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate the server taking a long time to persist delayed events + simulated_send_lag = Duration(seconds=5) + event_creation_handler = self.hs.get_event_creation_handler() + orig_send_fn = event_creation_handler.create_and_send_nonmember_event + + async def slow_send_fn(*args: Any, **kwargs: Any) -> Any: + await self.clock.sleep(simulated_send_lag) + return await orig_send_fn(*args, **kwargs) + + with patch.object(event_creation_handler, orig_send_fn.__name__, slow_send_fn): + self.reactor.advance(send_after.as_secs()) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response lacks a Retry-After header, because the reason for this limit + # is the server taking an indeterminitely long time to process a delayed event, and the + # server doesn't know how much longer the client should wait before sending more requests + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert not retry_after_headers + + # Wait until the delayed event gets persisted + self.reactor.advance(simulated_send_lag.as_secs()) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 5, + }, + } + ) + def test_delayed_event_user_limit_exceeded(self) -> None: + """ + Test that delayed event limits work properly when + the number of already scheduled events exceeds the configured limit. + + This can be invoked by the server admin lowering the configured limit & restarting the server + while a user has fewer scheduled delayed events than the old limit, but more than the new limit. + """ + send_after: Duration + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": f"test (send after {send_after.as_secs()}s)", "msgtype": "m.text"}, + ) + + for i in range(4): + send_after = Duration(seconds=i) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate restarting the server after having reconfigured the limit + # to be lower than the number of delayed events we just scheduled. + # + # Set the limit > 1 to test not having to wait for _all_ delayed events + # to be sent before being able to schedule a new one. + self.hs.config.server.max_delayed_events_per_user = 2 + + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + retry_after_header = channel.headers.getRawHeaders("Retry-After") + assert retry_after_header + retry_after_sec = int(retry_after_header[0]) + assert retry_after_sec > 0 + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @unittest.override_config({"max_event_delay_duration": "24h"}) def test_delayed_event_with_negative_delay(self) -> None: @@ -2595,7 +2794,7 @@ def test_add_delayed_event_ratelimit(self) -> None: """ # Test that new delayed events are correctly ratelimited. - args = ( + make_delayed_event_request = lambda: self.make_request( "POST", ( "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" @@ -2603,9 +2802,9 @@ def test_add_delayed_event_ratelimit(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. @@ -2614,7 +2813,7 @@ def test_add_delayed_event_ratelimit(self) -> None: ) # Test that the new delayed events aren't ratelimited anymore. - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -5568,3 +5767,117 @@ def test_MSC4293_redaction_applied_via_ban_api(self) -> None: expect_redaction=True, reason="being disruptive", ) + + +class CreateRoomRemoteInviteTestCase(unittest.FederatingHomeserverTestCase): + """ + Tests error propagation from remote invites during /createRoom. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq. + """ + + servlets = [ + room.register_servlets, + login.register_servlets, + register.register_servlets, + admin.register_servlets, + ] + + hijack_auth = False + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("creator", "test") + self.token = self.login("creator", "test") + + def _mock_remote_invite_http_error( + self, + status: int, + error_body: JsonDict, + ) -> None: + """ + Make the remote homeserver reply to its `/invite` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + federation_http_client = self.hs.get_federation_http_client() + + fake_agent = create_autospec(Agent, spec_set=True) + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/invite/" in uri, f"unexpected outbound request to {uri!r}" + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent.request.side_effect = request + federation_http_client.agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_remote_invite_bubbles_errors( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Test that, when creating a room involving a remote invite, + when the remote homeserver returns an error, we bubble it + to the client carefully. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + # Mock the remote homeserver (at the HTTP level) to return the configured error + self._mock_remote_invite_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + channel = self.make_request( + "POST", + "/createRoom", + {"invite": ["@alice:" + self.OTHER_SERVER_NAME]}, + access_token=self.token, + ) + + self.assertEqual( + channel.code, expected_client_facing_error_status, channel.result + ) + self.assertEqual(channel.json_body, expected_client_facing_error_body) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index c06c7312f2..e9b607cf23 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -489,6 +489,60 @@ def test_remote_room_key_request(self) -> None: }, ) + def test_remote_spoofed_sender(self) -> None: + """ + Tests that a to-device message whose sender domain does not match the origin + server is dropped. + """ + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # Send a spoofed to-device message EDU + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:not.the.same.example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"foo": "bar"}}}, + "message_id": "1", + }, + ) + ) + # Also send a valid one as a sentinel, to make sure our test setup + # is working as we'd expect + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"hiss": "meow"}}}, + "message_id": "1", + }, + ) + ) + + # Then do a /sync and be sure it didn't come down + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual( + messages, + [ + # Only the sentinel (valid) to-device message came down, not the spoofed one + { + "content": {"hiss": "meow"}, + "sender": "@user:example.org", + "type": "org.example.test", + } + ], + ) + def test_limited_sync(self) -> None: """If a limited sync for to-devices happens the next /sync should respond immediately.""" diff --git a/tests/rest/client/test_sticky_events.py b/tests/rest/client/test_sticky_events.py index a6e704fe8c..015315a307 100644 --- a/tests/rest/client/test_sticky_events.py +++ b/tests/rest/client/test_sticky_events.py @@ -58,7 +58,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Create a room self.room_id = self.helper.create_room_as(self.user_id, tok=self.token) - def _assert_event_sticky_for(self, event_id: str, sticky_ttl: int) -> None: + def _assert_event_sticky_for(self, event_id: str) -> None: channel = self.make_request( "GET", f"/rooms/{self.room_id}/event/{event_id}", @@ -75,10 +75,10 @@ def _assert_event_sticky_for(self, event_id: str, sticky_ttl: int) -> None: event["unsigned"], f"No {EventUnsignedContentFields.STICKY_TTL} field in {event_id}; event not sticky: {event}", ) - self.assertEqual( + self.assertGreater( event["unsigned"][EventUnsignedContentFields.STICKY_TTL], - sticky_ttl, - f"{event_id} had an unexpected sticky TTL: {event}", + 0, + f"{event_id} had an unexpected sticky TTL (expected some value greater than 0): {event}", ) def _assert_event_not_sticky(self, event_id: str) -> None: @@ -112,17 +112,15 @@ def test_sticky_event_via_event_endpoint(self) -> None: # If we request the event immediately, it will still have # 1 minute of stickiness - # The other 100 ms is advanced in FakeChannel.await_result. - self._assert_event_sticky_for(event_id, 59_900) + self._assert_event_sticky_for(event_id) - # But if we advance time by 59.799 seconds... - # we will get the event on its last millisecond of stickiness - # The other 100 ms is advanced in FakeChannel.await_result. - self.reactor.advance(59.799) - self._assert_event_sticky_for(event_id, 1) + # But if we advance time by 59 seconds... + # we will get the event on its last second of stickiness + self.reactor.advance(Duration(seconds=59).as_secs()) + self._assert_event_sticky_for(event_id) # Advancing time any more, the event is no longer sticky - self.reactor.advance(0.001) + self.reactor.advance(Duration(seconds=1).as_secs()) self._assert_event_not_sticky(event_id) diff --git a/tests/rest/client/test_sync_sticky_events.py b/tests/rest/client/test_sync_sticky_events.py index 7a38debdb9..2e52a0ad16 100644 --- a/tests/rest/client/test_sync_sticky_events.py +++ b/tests/rest/client/test_sync_sticky_events.py @@ -96,10 +96,9 @@ def test_single_sticky_event_appears_in_initial_sync(self) -> None: sticky_event_id, f"Sticky event {sticky_event_id} not found in sync timeline", ) - self.assertEqual( + self.assertGreater( timeline_events[-1]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - # The other 100 ms is advanced in FakeChannel.await_result. - 59_900, + 0, ) self.assertNotIn( @@ -129,7 +128,6 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: # that the /sync will get. regular_event_ids = [] for i in range(10): - # (Note: each one advances time by 100ms) response = self.helper.send( room_id=self.room_id, body=f"regular message {i}", @@ -138,7 +136,6 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: regular_event_ids.append(response["event_id"]) # Send another sticky event - # (Note: this advances time by 100ms) second_sticky_response = self.helper.send_sticky_event( self.room_id, EventTypes.Message, @@ -185,10 +182,9 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: f"Expected exactly 1 item in sticky events section, got {sticky_events}", ) self.assertEqual(sticky_events[0]["event_id"], first_sticky_event_id) - self.assertEqual( - # The 'missing' 1100 ms were elapsed when sending events + self.assertGreater( sticky_events[0]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - 58_800, + 0, ) # Assertions for the second sticky event: should be only in timeline section @@ -197,10 +193,9 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: second_sticky_event_id, f"Second sticky event {second_sticky_event_id} not found in sync timeline", ) - self.assertEqual( + self.assertGreater( timeline_events[-1]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - # The other 100 ms is advanced in FakeChannel.await_result. - 59_900, + 0, ) # (sticky section: we already checked it only has 1 item and # that item was the first above) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py new file mode 100644 index 0000000000..bbdbe38e07 --- /dev/null +++ b/tests/rest/client/test_versions.py @@ -0,0 +1,185 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet.testing import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, versions +from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + +logger = logging.getLogger(__name__) + + +class VersionsTestCase(unittest.HomeserverTestCase): + """ + Test `VersionsRestServlet` + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + versions.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def test_unauthenticated(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/versions", + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated_with_per_user_feature(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Sanity check that the experimental feature should not be enabled yet + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + # Enable the feature for this specific user + self._enable_experimental_feature_for_user( + target_user_id=user1_id, features={"msc3881": True} + ) + + # The experimental feature should be enabled for this user + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + True, + channel.json_body, + ) + + # But not for other users + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + def test_msc4446_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_msc4446_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: + """ + Make sure this looks like a `/_matrix/client/versions` response + """ + self.assertIsInstance( + versions_response["versions"], + list, + f"Expected `versions` to be a list of strings but saw {versions_response}", + ) + self.assertIsInstance( + versions_response["unstable_features"], + dict, + f"Expected `unstable_features` to be a dict mapping feature name to a bool but saw {versions_response}", + ) + + def _enable_experimental_feature_for_user( + self, *, target_user_id: str, features: dict[str, bool] + ) -> None: + """ + Use the admin API to enable an experimental feature for a specific user + """ + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/experimental_features/{target_user_id}", + content={ + "features": features, + }, + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200) diff --git a/tests/rest/synapse/mas/_base.py b/tests/rest/synapse/mas/_base.py index 19d33807a6..fb2419162e 100644 --- a/tests/rest/synapse/mas/_base.py +++ b/tests/rest/synapse/mas/_base.py @@ -25,15 +25,10 @@ class BaseTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() config["enable_registration"] = False - config["experimental_features"] = { - "msc3861": { - "enabled": True, - "issuer": "https://example.com", - "client_id": "dummy", - "client_auth_method": "client_secret_basic", - "client_secret": "dummy", - "admin_token": self.SHARED_SECRET, - } + config["matrix_authentication_service"] = { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": self.SHARED_SECRET, } return config diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index c73717f014..3accad842c 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -17,6 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # + from unittest.mock import AsyncMock from twisted.web.resource import Resource @@ -24,7 +25,6 @@ from synapse.rest.well_known import well_known_resource from tests import unittest -from tests.utils import HAS_AUTHLIB class WellKnownTests(unittest.HomeserverTestCase): @@ -106,23 +106,18 @@ def test_server_well_known_disabled(self) -> None: ) self.assertEqual(channel.code, 404) - @unittest.skip_unless(HAS_AUTHLIB, "requires authlib") @unittest.override_config( { "public_baseurl": "https://homeserver", # this is only required so that client well known is served - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "https://issuer", + "secret": "secret", }, "disable_registration": True, } ) - def test_client_well_known_msc3861_oauth_delegation(self) -> None: + def test_client_well_known_oauth_delegation(self) -> None: # Patch the HTTP client to return the issuer metadata req_mock = AsyncMock( return_value={ diff --git a/tests/server.py b/tests/server.py index 4dfdfc4042..17834ac0c7 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_encoder from tests.utils import ( @@ -255,7 +256,7 @@ def registerProducer(self, producer: IProducer, streaming: bool) -> None: def _produce() -> None: if self._producer: self._producer.resumeProducing() - self._reactor.callLater(0.1, _produce) + self._reactor.callLater(0.0, _produce) if not streaming: self._reactor.callLater(0.0, _produce) @@ -301,15 +302,102 @@ def transport(self) -> "FakeChannel": def await_result(self, timeout_ms: int = 1000) -> None: """ Wait until the request is finished. + + Advances the Twisted reactor clock by 0.1s and suspending execution of the + Python thread (to allow other threads to do work) in a loop until we see a + result. We timeout when both the Twisted reactor clock has been advanced enough + AND we've done at-least 100 iterations (round-trips for other threads to get + work done). + + The loop 1) allows `clock.call_later` scheduled callbacks to run if they are + scheduled to run now and 2) will also allow other threads to make progress. This + could be things spawned on the Twisted reactor threadpool or Tokio runtime + (async Rust code). + + Args: + timeout_ms: The Twisted reactor time we wait until we raise a `TimedOutException` """ - end_time = self._reactor.seconds() + timeout_ms / 1000.0 + timeout = Duration(milliseconds=timeout_ms) + + # TODO: Why? self._reactor.run() + # First, run anything that's scheduled now before we start looping and advancing + # non-zero time increments. + # + # Without this, if some request handler had some database queries followed by + # `self.hs.get_clock().sleep(Duration(seconds=1))`, and called + # `channel.await_result(timeout_ms=1000)`, it wouldn't be called because the + # first `self._reactor.advance(0.1)` would be first spent driving the database + # queries, and only leaving 0.9s remaining (0.1s shy of the sleep finishing) so + # the request would timeout. + # + # The goal is to remove the foot-guns and having to think about this for the + # standard cases. + # + # FIXME: Ideally, we'd advance by `0` but there is a handful of tests that + # assume that time advances in between requests and many requests complete from + # a single advance. Second best, we'd just advance by minuscule amount of time + # (`CLOCK_SCHEDULE_EPSILON`) but some tests assume at-least a millisecond in + # between as our timestamps are often recorded at the millisecond granularity + # (`origin_server_ts`, etc). It's a balance between test convenience of this + # helper and materializing test expectations so we may never fix this. + self._reactor.advance(Duration(milliseconds=1).as_secs()) + + # We only count the looping time (record the start after we advance once above) + start_time_seconds = self._reactor.seconds() + loop_count = 0 while not self.is_finished(): - if self._reactor.seconds() > end_time: + if ( + # Exceeded the Twisted reactor time timeout + # + # We use `>=` for the reactor time condition as it's possible we advance + # exactly the `timeout` amount and we don't want to get stuck in an + # infinite loop + self._reactor.seconds() >= start_time_seconds + timeout.as_secs() + # 100 loops is arbitrary. This also makes the assumption that any work + # on other threads will finish before we give up after sleeping ~0.1s of + # real-time (100 * 0.001). + and loop_count > 100 + ): raise TimedOutException("Timed out waiting for request to finish.") - self._reactor.advance(0.1) + # Suspend execution of this thread to allow other threads to do work. This + # could be things spawned on the Twisted reactor threadpool or Tokio thread + # pool (async Rust code). + # + # Note: Python has a default thread switch interval (5ms for cpython) (see + # `sys.setswitchinterval(interval)`) but we still want this here as we're + # able to preempt and cause the thread context switch to happen faster. + # Also, without any real-time sleeping, this function would complete before + # the 5ms switch ever happened. + # + # After a few cycles, we use `time.sleep(0.001)` instead of `time.sleep(0)` + # to avoid tightlooping on the main thread (CPU 100%) because it's wasteful + # and may starve out other threads. 10 is arbitrary but many cases will have + # none or only a few round-trips so we can just try to go as fast as + # possible. + if loop_count < 10: + time.sleep(0) + else: + time.sleep(0.001) + + # Advance the Twisted reactor and run any scheduled callbacks + # + # Don't advance the Twisted reactor clock further than the timeout duration + # as someone should increase the timeout if they expect things to take + # longer. + if self._reactor.seconds() < start_time_seconds + timeout.as_secs(): + self._reactor.advance(0.1) + else: + # But we want to still keep running whatever might be getting scheduled + # to run now. + # + # For example from other threads, they may have scheduled something on + # the reactor to run (like `reactor.callFromThread(...)`) + self._reactor.advance(0) + + loop_count += 1 def extract_cookies(self, cookies: MutableMapping[str, str]) -> None: """Process the contents of any Set-Cookie headers in the response @@ -940,7 +1028,7 @@ def _produce() -> None: # mypy ignored here because: # - this is part of the test infrastructure (outside of Synapse) so tracking # these calls for for homeserver shutdown doesn't make sense. - d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] + d.addCallback(lambda x: self._reactor.callLater(0.0, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: # mypy ignored here because: diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index e3f79d7670..139906e97c 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -59,8 +59,8 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main async def update(self, progress: JsonDict, count: int) -> int: - duration_ms = 10 - await self.clock.sleep(Duration(milliseconds=count * duration_ms)) + fake_work_duration = Duration(seconds=1) + await self.clock.sleep(fake_work_duration) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", @@ -86,10 +86,15 @@ def test_do_background_update(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.02, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # on the first call, we should get run with the default background update size @@ -143,10 +148,15 @@ def test_background_update_default_batch_set_by_config(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.01, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # on the first call, we should get run with the default background update size specified in the config @@ -265,10 +275,15 @@ def test_background_update_duration_set_in_config(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.02, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # the first update was run with the default batch size, this should be run with 500ms as the @@ -298,9 +313,6 @@ def test_background_update_min_batch_set_in_config(self) -> None: """ Test that the minimum batch size set in the config is used """ - # a very long-running individual update - duration_ms = 50 - self.get_success( self.store.db_pool.simple_insert( "background_updates", @@ -310,7 +322,8 @@ def test_background_update_min_batch_set_in_config(self) -> None: # Run the update with the long-running update item async def update_long(progress: JsonDict, count: int) -> int: - await self.clock.sleep(Duration(milliseconds=count * duration_ms)) + very_long_fake_work_duration = Duration(seconds=5) + await self.clock.sleep(very_long_fake_work_duration) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", @@ -322,10 +335,15 @@ async def update_long(progress: JsonDict, count: int) -> int: self.update_handler.side_effect = update_long self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=1, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `very_long_fake_work_duration` + self.reactor.advance(Duration(seconds=5).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # the first update was run with the default batch size, this should be run with minimum batch size diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index bd68f2aaa1..fd138335f0 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -782,7 +782,11 @@ def _runtest( device_id=device_id, ip=expected_ip, user_agent="Mozzila pizza", - last_seen=123456100, + # Note: The extra 1ms is from `make_request(...)` -> `await_result(...)` + # + # FIXME: This test shouldn't care about this internal detail (don't + # assert exact timing) + last_seen=123456001, ), r, ) diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py index 175a5ffc78..d09437c080 100644 --- a/tests/storage/test_event_chain.py +++ b/tests/storage/test_event_chain.py @@ -755,7 +755,7 @@ def test_background_update_single_large_room(self) -> None: ): iterations += 1 self.get_success( - self.store.db_pool.updates.do_next_background_update(False), by=0.1 + self.store.db_pool.updates.do_next_background_update(False) ) # Ensure that we did actually take multiple iterations to process the @@ -814,7 +814,7 @@ def test_background_update_multiple_large_room(self) -> None: ): iterations += 1 self.get_success( - self.store.db_pool.updates.do_next_background_update(False), by=0.1 + self.store.db_pool.updates.do_next_background_update(False) ) # Ensure that we did actually take multiple iterations to process the diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index cae00c3ba7..80b7705555 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -79,7 +79,7 @@ def _create_id_generator( writers: list[str] | None = None, ) -> MultiWriterIdGenerator: def _create(conn: LoggingDatabaseConnection) -> MultiWriterIdGenerator: - return MultiWriterIdGenerator( + id_gen = MultiWriterIdGenerator( db_conn=conn, db=self.db_pool, notifier=self.hs.get_replication_notifier(), @@ -91,6 +91,15 @@ def _create(conn: LoggingDatabaseConnection) -> MultiWriterIdGenerator: writers=writers or ["master"], positive=self.positive, ) + # Constructing the generator prunes stale `stream_positions` rows + # (writers no longer in the config); commit so that persists for the + # next generator we create. + # + # Note we need to commit manually here as the generator is created + # in a `runWithConnection` call, which doesn't automatically + # commit/rollback. + conn.commit() + return id_gen self.instances[instance_name] = self.get_success_or_raise( self.db_pool.runWithConnection(_create) diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index dbbede812d..6c3506f348 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -637,9 +637,7 @@ def test_batched_state_group_storing(self) -> None: self.get_success( self.store.db_pool.simple_select_list( table="state_group_edges", - keyvalues={ - "state_group": str(context.state_group_after_event) - }, + keyvalues={"state_group": context.state_group_after_event}, retcols=("prev_state_group",), ) ), diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py index 56fab3a0e1..845fe2b503 100644 --- a/tests/synapse_rust/test_http_client.py +++ b/tests/synapse_rust/test_http_client.py @@ -15,9 +15,8 @@ import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any, Coroutine, Generator, TypeVar, Union +from typing import Any, TypeVar -from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor from synapse.logging.context import ( @@ -118,31 +117,6 @@ def tearDown(self) -> None: for callbable, args, kwargs in triggers: callbable(*args, **kwargs) - def till_deferred_has_result( - self, - awaitable: Union[ - "Coroutine[Deferred[Any], Any, T]", - "Generator[Deferred[Any], Any, T]", - "Deferred[T]", - ], - ) -> "Deferred[T]": - """Wait until a deferred has a result. - - This is useful because the Rust HTTP client will resolve the deferred - using reactor.callFromThread, which are only run when we call - reactor.advance. - """ - deferred = ensureDeferred(awaitable) - tries = 0 - while not deferred.called: - time.sleep(0.1) - self.reactor.advance(0) - tries += 1 - if tries > 100: - raise Exception("Timed out waiting for deferred to resolve") - - return deferred - def _check_current_logcontext(self, expected_logcontext_string: str) -> None: context = current_context() assert isinstance(context, LoggingContext) or isinstance(context, _Sentinel), ( @@ -168,7 +142,7 @@ async def do_request() -> None: raw_response = json_decoder.decode(resp_body.decode("utf-8")) self.assertEqual(raw_response, {"ok": True}) - self.get_success(self.till_deferred_has_result(do_request())) + self.get_success(do_request()) self.assertEqual(self.server.calls, 1) def test_request_response_limit_exceeded(self) -> None: @@ -183,8 +157,8 @@ async def do_request() -> None: response_limit=1, ) - self.assertFailure( - self.till_deferred_has_result(do_request()), + self.get_failure( + do_request(), RuntimeError, ) self.assertEqual(self.server.calls, 1) @@ -227,8 +201,15 @@ async def do_request() -> None: # Now wait for the function under test to have run with PreserveLoggingContext(): while not callback_finished: - # await self.hs.get_clock().sleep(0) - time.sleep(0.1) + # Allow the async Rust to run + # + # Suspend execution of this thread to allow other the Tokio thread + # pool to do work. + time.sleep(0) + # Advance the Twisted reactor and run any scheduled callbacks + # + # In terms of other threads, they may have scheduled something on the + # reactor to run (like `reactor.callFromThread(...)`) self.reactor.advance(0) # check that the logcontext is left in a sane state. diff --git a/tests/unittest.py b/tests/unittest.py index 31ea97727d..3a4265919b 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -49,6 +49,7 @@ import unpaddedbase64 from typing_extensions import Concatenate, ParamSpec +from twisted.internet import defer from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor, MemoryReactorClock from twisted.python.failure import Failure @@ -77,7 +78,7 @@ from synapse.server import HomeServer from synapse.storage.keys import FetchKeyResult from synapse.types import ISynapseReactor, JsonDict, Requester, UserID, create_requester -from synapse.util.clock import Clock +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from synapse.util.httpresourcetree import create_resource_tree from tests.server import ( @@ -481,27 +482,13 @@ def tearDown(self) -> None: # Reset to not use frozen dicts. events.USE_FROZEN_DICTS = False - def wait_on_thread(self, deferred: Deferred, timeout: int = 10) -> None: - """ - Wait until a Deferred is done, where it's waiting on a real thread. - """ - start_time = time.time() - - while not deferred.called: - if start_time + timeout < time.time(): - raise ValueError("Timed out waiting for threadpool") - self.reactor.advance(0.01) - time.sleep(0.01) - def wait_for_background_updates(self) -> None: """Block until all background database updates have completed.""" store = self.hs.get_datastores().main while not self.get_success( store.db_pool.updates.has_completed_background_updates() ): - self.get_success( - store.db_pool.updates.do_next_background_update(False), by=0.1 - ) + self.get_success(store.db_pool.updates.do_next_background_update(False)) def make_homeserver( self, reactor: ThreadedMemoryReactorClock, clock: Clock @@ -743,21 +730,165 @@ def pump(self, by: float = 0.0) -> None: # whole chain to completion. self.reactor.pump([by] * 100) - def get_success(self, d: Awaitable[TV], by: float = 0.0) -> TV: + def _wait_for_deferred( + self, + d: "Deferred[Any]", + ) -> None: + """ + Wait for the deferred to finish or raise. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + Args: + d: Twisted Deferred + + Raises: + defer.TimeoutError: If the timeout expires before the deferred completes. + """ + # Wait until the deferred has a result + # + # Checking `d.called` by itself is not sufficient by itself as this is possible: + # + # If you have a first `Deferred` `D1`, you can add a callback which returns + # another `Deferred` `D2`, and `D2` must then complete before any further + # callbacks on `D1` will execute (and later callbacks on `D1` get the *result* + # of `D2` rather than `D2` itself). + # + # So, `D1` might have `called=True` (as in, it has started running its + # callbacks), but any new callbacks added to `D1` won't get run until `D2` + # completes. Fortunately, we can detect this by checking `d.paused`. + loop_count = 0 + while not d.called or d.paused: + # 100 loops is arbitrary but based on previous code which used to "pump" and + # advance the reactor 100 times. This also makes the assumption that any + # work on other threads will finish before we give up after sleeping ~0.1s + # of real-time (100 * 0.001). + if loop_count > 100: + raise defer.TimeoutError("Timed out waiting for deferred to finish") + + # Suspend execution of this thread to allow other threads to do work. This + # could be things spawned on the Twisted reactor threadpool or Tokio thread + # pool (async Rust code). + # + # Note: Python has a default thread switch interval (5ms for cpython) (see + # `sys.setswitchinterval(interval)`) but we still want this here as we're + # able to preempt and cause the thread context switch to happen faster. + # Also, without any real-time sleeping, this function would complete before + # the 5ms switch ever happened. + # + # After a few cycles, we use `time.sleep(0.001)` instead of `time.sleep(0)` + # to avoid tightlooping on the main thread (CPU 100%) because it's wasteful + # and may starve out other threads. 10 is arbitrary but many cases will have + # none or only a few round-trips so we can just try to go as fast as + # possible. + if loop_count < 10: + time.sleep(0) + else: + time.sleep(0.001) + + # Advance the Twisted reactor and run any scheduled callbacks + # + # In terms of other threads, they may have scheduled something on the + # reactor to run (like `reactor.callFromThread(...)`) + # + # Ideally, we'd advance by `0` but the `Cooperator` used in our HTTP clients + # use `CLOCK_SCHEDULE_EPSILON` and we want to make usage in downstream tests + # as simple as possible. A common use case this helps with is anything that + # needs to make a HTTP request (like a replication requests) + self.reactor.advance(CLOCK_SCHEDULE_EPSILON.as_secs()) + + loop_count += 1 + + def get_success( + self, + d: Awaitable[TV], + ) -> TV: + """ + Get the success result of an awaitable. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + If you need to advance the Twisted reactor by an actual time increment, you can + use the following pattern: + ```python + # We use `ensureDeferred(...)` as a `Deferred` can run in the background on its own (unlike a Python coroutine) + task_d = ensureDeferred(my_async_task()) + # Please explain why/what scheduled call you're trying to trigger + self.reactor.advance(Duration(seconds=1).as_secs()) + result = self.get_success(sync_d) + ``` + + Args: + d: awaitable + + Raises: + defer.TimeoutError: If the timeout expires before the awaitable completes. + SynchronousTestCase.failureException: If the awaitable has a failure result or has no result + (although you would probably run into `defer.TimeoutError` in that case). + """ deferred: Deferred[TV] = ensureDeferred(d) # type: ignore[arg-type] - self.pump(by=by) + self._wait_for_deferred(deferred) + return self.successResultOf(deferred) def get_failure( - self, d: Awaitable[Any], exc: type[_ExcType], by: float = 0.0 + self, + d: Awaitable[Any], + exc: type[_ExcType], ) -> _TypedFailure[_ExcType]: """ - Run a Deferred and get a Failure from it. The failure must be of the type `exc`. + Get the failure result of an awaitable. The failure must be of the type `exc`. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + If you need to advance the Twisted reactor by an actual time increment, you can + use the following pattern: + ```python + # We use `ensureDeferred(...)` as a `Deferred` can run in the background on its own (unlike a Python coroutine) + task_d = ensureDeferred(my_async_task()) + # Please explain why/what scheduled call you're trying to trigger + self.reactor.advance(Duration(seconds=1).as_secs()) + result = self.get_success(sync_d) + ``` + + Args: + d: awaitable + exc: Exception type to expect + + Raises: + defer.TimeoutError: If the timeout expires before the awaitable completes. + SynchronousTestCase.failureException: If the awaitable has a success result, + or has an unexpected failure result, or has no result (although you would + probably run into `defer.TimeoutError` in that case). """ deferred: Deferred[Any] = ensureDeferred(d) # type: ignore[arg-type] - self.pump(by) + self._wait_for_deferred(deferred) + return self.failureResultOf(deferred, exc) + # FIXME: Remove as this has the exact same semantics as `get_success()`. In + # https://github.com/matrix-org/synapse/pull/8402#discussion_r495992506 where it was + # introduced, it was claimed that "get_success fails the test if the deferred fails + # rather than raising, which I find a bit unintuitive." but `get_success()` actually + # does raise "@raise SynchronousTestCase.failureException : If the + # L{Deferred} has no result or has a failure + # result." at-least in today's world. + # + # As another alternative, we could also just update `get_success(...)` to have this + # behavior as the default, see + # https://github.com/element-hq/synapse/pull/19871#discussion_r3483616710 def get_success_or_raise(self, d: Awaitable[TV], by: float = 0.0) -> TV: """Drive deferred to completion and return result or raise exception on failure. diff --git a/tests/util/test_httpresourcetree.py b/tests/util/test_httpresourcetree.py new file mode 100644 index 0000000000..b5dc2e0657 --- /dev/null +++ b/tests/util/test_httpresourcetree.py @@ -0,0 +1,66 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import os +from http import HTTPStatus + +from twisted.web.resource import Resource + +import synapse +from synapse.api.errors import Codes +from synapse.api.urls import STATIC_PREFIX +from synapse.http.server import StaticResource + +from tests import unittest + + +class ResourceTreeTestCase(unittest.HomeserverTestCase): + servlets = [] + + def create_resource_dict(self) -> dict[str, Resource]: + """ + Register /_matrix/static for the test. + """ + resources = super().create_resource_dict() + resources[STATIC_PREFIX] = StaticResource( + # as in `synapse/app/homeserver.py` `_configure_named_resource` + os.path.join(os.path.dirname(synapse.__file__), "static") + ) + return resources + + def test_inserted_segment_is_silently_swallowed(self) -> None: + """ + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq + + The path `/_matrix/INSERTED/static/client/login/style.css` used to resolve to the same + as `/_matrix/static/client/login/style.css`. + """ + PATH_SUFFIX = "/static/client/login/style.css" + correct_channel = self.make_request( + "GET", + f"/_matrix{PATH_SUFFIX}", + shorthand=False, + ) + # The correct path should give a 200 OK static resource + self.assertEqual(correct_channel.code, HTTPStatus.OK, correct_channel.result) + + wrong_channel = self.make_request( + "GET", + f"/_matrix/INSERTED{PATH_SUFFIX}", + shorthand=False, + ) + # This prefixed version of the same path should give a 404 + self.assertEqual(wrong_channel.code, HTTPStatus.NOT_FOUND, wrong_channel.result) + self.assertEqual( + wrong_channel.json_body["errcode"], Codes.UNRECOGNIZED, wrong_channel.result + ) diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index 94c1d778e6..cab9695d33 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -260,7 +260,7 @@ async def _incrementing_running_task( await self.task_scheduler.update_task( task.id, result={"counter": current_counter} ) - await self.hs.get_clock().sleep(Duration(microseconds=1)) + await self.hs.get_clock().sleep(Duration(seconds=1)) return TaskStatus.COMPLETE, None, None # type: ignore[unreachable] diff --git a/tests/utils.py b/tests/utils.py index f3d5129097..7ea3a2aded 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -318,20 +318,44 @@ class test_timeout: my_checking_func() time.sleep(0.1) ``` + + Args: + seconds: How long to allow the block to run for before raising + `TestTimeout`. + error_message: Extra text to append to the `TestTimeout` message. + cpu_time: If `True`, `seconds` is a budget of CPU time (user + system, + across all threads) consumed by the process rather than wall-clock + time. Useful for performance-regression tests, as time spent + blocked on I/O (e.g. waiting on the database) or lost to a loaded + CI machine doesn't count against the budget. Note that a block + which hangs while consuming *no* CPU will never trip this variant. """ - def __init__(self, seconds: int, error_message: str | None = None) -> None: - self.error_message = f"Test timed out after {seconds}s" + def __init__( + self, + seconds: float, + error_message: str | None = None, + *, + cpu_time: bool = False, + ) -> None: + self.error_message = f"Test timed out after {seconds}s of {'CPU' if cpu_time else 'wall-clock'} time" if error_message is not None: self.error_message += f": {error_message}" self.seconds = seconds + self.cpu_time = cpu_time def handle_timeout(self, signum: int, frame: FrameType | None) -> None: raise TestTimeout(self.error_message) def __enter__(self) -> None: - signal.signal(signal.SIGALRM, self.handle_timeout) - signal.alarm(self.seconds) + if self.cpu_time: + # `ITIMER_PROF` counts down against process CPU time (user + + # system) and delivers `SIGPROF` when it expires. + signal.signal(signal.SIGPROF, self.handle_timeout) + signal.setitimer(signal.ITIMER_PROF, self.seconds) + else: + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.setitimer(signal.ITIMER_REAL, self.seconds) def __exit__( self, @@ -339,4 +363,4 @@ def __exit__( exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> None: - signal.alarm(0) + signal.setitimer(signal.ITIMER_PROF if self.cpu_time else signal.ITIMER_REAL, 0)