diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2ee494c5..04ecf418 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -47,16 +47,17 @@ jobs: CACHIER_TEST_VS_DOCKERIZED_REDIS: "true" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install package & dependencies - run: python -m pip install -e . -r tests/requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . -r tests/requirements.txt - name: Lower inotify instance limit (Linux only, for Issue #24 test) if: runner.os == 'Linux' && matrix.backend == 'local' @@ -75,10 +76,6 @@ jobs: # For testcontainers to find the Colima socket sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - - name: Install other test dependencies - if: matrix.backend != 'local' - run: python -m pip install -e . -r tests/requirements_${{ matrix.backend }}.txt - # ToDo: find a way to cache docker images #- name: Cache Container Images # if: matrix.backend == 'mongodb' @@ -86,6 +83,7 @@ jobs: # with: # prefix-key: "mongo-db" # images: mongo:latest + - name: Start MongoDB in docker if: matrix.backend == 'mongodb' run: | @@ -96,6 +94,11 @@ jobs: # show running containers docker ps -a + - name: Install MongoDB core test dependencies + if: matrix.backend == 'mongodb' + run: | + python -m pip install -e . -r tests/mongodb_requirements.txt + - name: Unit tests (DB) if: matrix.backend == 'mongodb' run: pytest -m "mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml @@ -115,10 +118,15 @@ jobs: sleep 10 docker ps -a + - name: Install SQL core test dependencies (SQL/Postgres) + if: matrix.backend == 'postgres' + run: | + python -m pip install -e . -r tests/sql_requirements.txt + - name: Unit tests (SQL/Postgres) if: matrix.backend == 'postgres' env: - SQLALCHEMY_DATABASE_URL: postgresql+psycopg://testuser:testpass@localhost:5432/testdb + SQLALCHEMY_DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb run: pytest -m sql --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Start Redis in docker @@ -131,6 +139,11 @@ jobs: sleep 5 docker ps -a + - name: Install Redis core test dependencies + if: matrix.backend == 'redis' + run: | + python -m pip install -e . -r tests/redis_requirements.txt + - name: Unit tests (Redis) if: matrix.backend == 'redis' run: pytest -m redis --cov=cachier --cov-report=term --cov-report=xml:cov.xml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f21b3d6..41b9353f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,8 +22,8 @@ jobs: env: RELEASING_PROCESS: "1" steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -43,7 +43,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Upload dist artifact for the publish job - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: cachier_dist path: dist @@ -56,11 +56,11 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] steps: - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v5 with: python-version: "3.x" - name: Download dist artifact - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: cachier_dist path: dist @@ -77,12 +77,12 @@ jobs: environment: pypi_publish steps: - name: Download dist artifact - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: cachier_dist path: dist - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@v1.13.0 + uses: pypa/gh-action-pypi-publish@v1.12.4 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/Makefile b/Makefile index 9369a313..22918c71 100644 --- a/Makefile +++ b/Makefile @@ -52,9 +52,9 @@ install-dev: install-all: pip install -e .[all] pip install -r tests/requirements.txt - pip install -r tests/requirements_mongodb.txt - pip install -r tests/requirements_redis.txt - pip install -r tests/requirements_postgres.txt + pip install -r tests/mongodb_requirements.txt + pip install -r tests/redis_requirements.txt + pip install -r tests/sql_requirements.txt # Testing targets test: diff --git a/pyproject.toml b/pyproject.toml index b3565e8e..077f6391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,6 +179,7 @@ markers = [ "sql: test the SQL core", "maxage: test the max_age functionality", "asyncio: marks tests as async", + "seriallocal: tests that must run serially for local backends", ] [tool.coverage.report] diff --git a/scripts/README-local-testing.md b/scripts/README-local-testing.md index 5a00563e..2315fbb3 100644 --- a/scripts/README-local-testing.md +++ b/scripts/README-local-testing.md @@ -41,6 +41,8 @@ This guide explains how to run cachier tests locally with Docker containers for - `-k, --keep-running` - Keep Docker containers running after tests - `-h, --html-coverage` - Generate HTML coverage report - `-f, --files` - Specify test files to run (can be used multiple times) +- `-p, --parallel` - Run tests in parallel using pytest-xdist +- `-w, --workers` - Number of parallel workers (default: auto) - `--help` - Show help message ## Examples @@ -96,6 +98,18 @@ CACHIER_TEST_CORES="mongo redis" ./scripts/test-local.sh # Combine file selection with other options ./scripts/test-local.sh redis sql -f tests/test_sql_core.py -v -k + +# Run tests in parallel with automatic worker detection +./scripts/test-local.sh all -p + +# Run tests in parallel with 4 workers +./scripts/test-local.sh external -p -w 4 + +# Run local tests in parallel (memory and pickle) +./scripts/test-local.sh memory pickle -p + +# Combine parallel testing with other options +./scripts/test-local.sh mongo redis -p -v -k ``` ### Docker Compose @@ -107,7 +121,7 @@ make services-start # Run tests manually CACHIER_TEST_HOST=localhost CACHIER_TEST_PORT=27017 CACHIER_TEST_VS_DOCKERIZED_MONGO=true \ CACHIER_TEST_REDIS_HOST=localhost CACHIER_TEST_REDIS_PORT=6379 CACHIER_TEST_VS_DOCKERIZED_REDIS=true \ -SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" \ +SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" \ pytest -m "mongo or redis or sql" # Stop all services @@ -146,7 +160,7 @@ The script automatically sets the required environment variables: ### SQL/PostgreSQL -- `SQLALCHEMY_DATABASE_URL=postgresql+psycopg://testuser:testpass@localhost:5432/testdb` +- `SQLALCHEMY_DATABASE_URL=postgresql://testuser:testpass@localhost:5432/testdb` ## Prerequisites @@ -154,9 +168,9 @@ The script automatically sets the required environment variables: 2. **Python dependencies** - Install test requirements: ```bash pip install -r tests/requirements.txt - pip install -r tests/requirements_mongodb.txt # For MongoDB tests - pip install -r tests/requirements_redis.txt # For Redis tests - pip install -r tests/requirements_postgres.txt # For SQL tests + pip install -r tests/mongodb_requirements.txt # For MongoDB tests + pip install -r tests/redis_requirements.txt # For Redis tests + pip install -r tests/sql_requirements.txt # For SQL tests ``` ## Troubleshooting @@ -193,10 +207,12 @@ The script automatically sets the required environment variables: 2. **For quick iteration**: Use memory and pickle tests (no Docker required) 3. **For debugging**: Use `-k` to keep containers running and inspect them 4. **For CI parity**: Test with the same backends that CI uses +5. **For faster test runs**: Use `-p` to run tests in parallel, especially when testing multiple backends +6. **For parallel testing**: The script automatically installs pytest-xdist when needed +7. **Worker count**: Use `-w auto` (default) to let pytest-xdist determine optimal workers, or specify a number based on your CPU cores ## Future Enhancements - Add MySQL/MariaDB support - Add Elasticsearch support - Add performance benchmarking mode -- Add parallel test execution for multiple backends diff --git a/scripts/docker-compose.all-cores.yml b/scripts/docker-compose.all-cores.yml index 3aeb4ee4..90de27e5 100644 --- a/scripts/docker-compose.all-cores.yml +++ b/scripts/docker-compose.all-cores.yml @@ -76,6 +76,6 @@ networks: # docker-compose -f scripts/docker-compose.all-cores.yml up -d # CACHIER_TEST_HOST=localhost CACHIER_TEST_PORT=27017 CACHIER_TEST_VS_DOCKERIZED_MONGO=true \ # CACHIER_TEST_REDIS_HOST=localhost CACHIER_TEST_REDIS_PORT=6379 CACHIER_TEST_VS_DOCKERIZED_REDIS=true \ -# SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" \ +# SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" \ # pytest -m "mongo or redis or sql" # docker-compose -f scripts/docker-compose.all-cores.yml down diff --git a/scripts/test-local.sh b/scripts/test-local.sh index 0d07661b..c5b3b40c 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -26,6 +26,8 @@ KEEP_RUNNING=false SELECTED_CORES="" INCLUDE_LOCAL_CORES=false TEST_FILES="" +PARALLEL=false +PARALLEL_WORKERS="auto" # Function to print colored messages print_message() { @@ -56,6 +58,8 @@ OPTIONS: -k, --keep-running Keep containers running after tests -h, --html-coverage Generate HTML coverage report -f, --files Specify test files to run (can be used multiple times) + -p, --parallel Run tests in parallel using pytest-xdist + -w, --workers Number of parallel workers (default: auto) --help Show this help message EXAMPLES: @@ -65,6 +69,8 @@ EXAMPLES: $0 external -k # Run external backends, keep containers $0 mongo memory -v # Run MongoDB and memory tests verbosely $0 all -f tests/test_main.py -f tests/test_redis_core_coverage.py # Run specific test files + $0 memory pickle -p # Run local tests in parallel + $0 all -p -w 4 # Run all tests with 4 parallel workers ENVIRONMENT: You can also set cores via CACHIER_TEST_CORES environment variable: @@ -102,6 +108,20 @@ while [[ $# -gt 0 ]]; do usage exit 0 ;; + -p|--parallel) + PARALLEL=true + shift + ;; + -w|--workers) + shift + if [[ $# -eq 0 ]] || [[ "$1" == -* ]]; then + print_message $RED "Error: -w/--workers requires a number argument" + usage + exit 1 + fi + PARALLEL_WORKERS="$1" + shift + ;; -*) print_message $RED "Unknown option: $1" usage @@ -232,11 +252,22 @@ check_dependencies() { } fi + # Check for pytest-xdist if parallel testing is requested + if [ "$PARALLEL" = true ]; then + if ! python -c "import xdist" 2>/dev/null; then + print_message $YELLOW "Installing pytest-xdist for parallel testing..." + pip install pytest-xdist || { + print_message $RED "Failed to install pytest-xdist" + exit 1 + } + fi + fi + # Check MongoDB dependencies if testing MongoDB if echo "$SELECTED_CORES" | grep -qw "mongo"; then if ! python -c "import pymongo" 2>/dev/null; then print_message $YELLOW "Installing MongoDB test requirements..." - pip install -r tests/requirements_mongodb.txt || { + pip install -r tests/mongodb_requirements.txt || { print_message $RED "Failed to install MongoDB requirements" exit 1 } @@ -247,7 +278,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "redis"; then if ! python -c "import redis" 2>/dev/null; then print_message $YELLOW "Installing Redis test requirements..." - pip install -r tests/requirements_redis.txt || { + pip install -r tests/redis_requirements.txt || { print_message $RED "Failed to install Redis requirements" exit 1 } @@ -258,7 +289,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "sql"; then if ! python -c "import sqlalchemy" 2>/dev/null; then print_message $YELLOW "Installing SQL test requirements..." - pip install -r tests/requirements_postgres.txt || { + pip install -r tests/sql_requirements.txt || { print_message $RED "Failed to install SQL requirements" exit 1 } @@ -399,7 +430,7 @@ stop_postgres() { } test_sql() { - export SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" + export SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" } # Main execution @@ -410,14 +441,20 @@ main() { # Check and install dependencies check_dependencies - # Check if we need Docker + # Check if we need Docker, and if we should run serial pickle tests needs_docker=false + run_serial_local_tests=false for core in $SELECTED_CORES; do case $core in mongo|redis|sql) needs_docker=true ;; esac + case $core in + pickle|all) + run_serial_local_tests=true + ;; + esac done if [ "$needs_docker" = true ]; then @@ -484,15 +521,20 @@ main() { sql) test_sql ;; esac done + pytest_markers="$pytest_markers and not seriallocal" # Run pytest # Build pytest command PYTEST_CMD="pytest" + # and the specific pytest command for running serial pickle tests + SERIAL_PYTEST_CMD="pytest -m seriallocal -n0" # Add test files if specified if [ -n "$TEST_FILES" ]; then PYTEST_CMD="$PYTEST_CMD $TEST_FILES" print_message $BLUE "Test files specified: $TEST_FILES" + # and turn off serial local tests, so we run only selected files + run_serial_local_tests=false fi # Add markers if needed (only if no specific test files were given) @@ -504,6 +546,10 @@ main() { if [ "$selected_sorted" != "$all_sorted" ]; then PYTEST_CMD="$PYTEST_CMD -m \"$pytest_markers\"" + else + print_message $BLUE "Running all tests without markers since all cores are selected" + PYTEST_CMD="$PYTEST_CMD -m \"not seriallocal\"" + run_serial_local_tests=true fi else # When test files are specified, still apply markers if not running all cores @@ -519,15 +565,41 @@ main() { # Add verbose flag if needed if [ "$VERBOSE" = true ]; then PYTEST_CMD="$PYTEST_CMD -v" + SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD -v" + fi + + # Add parallel testing options if requested + if [ "$PARALLEL" = true ]; then + PYTEST_CMD="$PYTEST_CMD -n $PARALLEL_WORKERS" + + # Show parallel testing info + if [ "$PARALLEL_WORKERS" = "auto" ]; then + print_message $BLUE "Running tests in parallel with automatic worker detection" + else + print_message $BLUE "Running tests in parallel with $PARALLEL_WORKERS workers" + fi + + # Special note for pickle tests + if echo "$SELECTED_CORES" | grep -qw "pickle"; then + print_message $YELLOW "Note: Pickle tests will use isolated cache directories for parallel safety" + fi fi # Add coverage options PYTEST_CMD="$PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT" + SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT --cov-append" # Print and run the command print_message $BLUE "Running: $PYTEST_CMD" eval $PYTEST_CMD + if [ "$run_serial_local_tests" = true ]; then + print_message $BLUE "Running serial local tests (pickle, memory) with: $SERIAL_PYTEST_CMD" + eval $SERIAL_PYTEST_CMD + else + print_message $BLUE "Skipping serial local tests (pickle, memory) since not requested" + fi + TEST_EXIT_CODE=$? if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..07b54a88 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,472 @@ +# Cachier Test Suite Documentation + +This document provides comprehensive guidelines for writing and running tests for the Cachier package. + +## Table of Contents + +1. [Test Suite Overview](#test-suite-overview) +2. [Test Structure](#test-structure) +3. [Running Tests](#running-tests) +4. [Writing Tests](#writing-tests) +5. [Test Isolation](#test-isolation) +6. [Backend-Specific Testing](#backend-specific-testing) +7. [Parallel Testing](#parallel-testing) +8. [CI/CD Integration](#cicd-integration) +9. [Troubleshooting](#troubleshooting) + +## Test Suite Overview + +The Cachier test suite is designed to comprehensively test all caching backends while maintaining proper isolation between tests. The suite uses pytest with custom markers for backend-specific tests. + +### Supported Backends + +- **Memory**: In-memory caching (no external dependencies) +- **Pickle**: File-based caching using pickle (default backend) +- **MongoDB**: Database caching using MongoDB +- **Redis**: In-memory data store caching +- **SQL**: SQL database caching via SQLAlchemy (PostgreSQL, SQLite, MySQL) + +### Test Categories + +1. **Core Functionality**: Basic caching operations (get, set, clear) +2. **Stale Handling**: Testing `stale_after` parameter +3. **Concurrency**: Thread-safety and multi-process tests +4. **Error Handling**: Exception scenarios and recovery +5. **Performance**: Speed and efficiency tests +6. **Integration**: Cross-backend compatibility + +## Test Structure + +``` +tests/ +├── conftest.py # Shared fixtures and configuration +├── requirements.txt # Base test dependencies (includes pytest-rerunfailures) +├── mongodb_requirements.txt # MongoDB-specific dependencies +├── redis_requirements.txt # Redis-specific dependencies +├── sql_requirements.txt # SQL-specific dependencies +│ +├── test_*.py # Test modules +├── test_mongo_core.py # MongoDB-specific tests +├── test_redis_core.py # Redis-specific tests +├── test_sql_core.py # SQL-specific tests +├── test_memory_core.py # Memory backend tests +├── test_pickle_core.py # Pickle backend tests +├── test_general.py # Cross-backend tests +└── ... +``` + +### Test Markers + +Tests are marked with backend-specific markers: + +```python +@pytest.mark.mongo # MongoDB tests +@pytest.mark.redis # Redis tests +@pytest.mark.sql # SQL tests +@pytest.mark.memory # Memory backend tests +@pytest.mark.pickle # Pickle backend tests +@pytest.mark.maxage # Tests involving stale_after functionality +@pytest.mark.flaky # Flaky tests that should be retried (see Flaky Tests section) +``` + +## Running Tests + +### Quick Start + +```bash +# Run all tests +pytest + +# Run tests for specific backend +pytest -m mongo +pytest -m redis +pytest -m sql + +# Run tests for multiple backends +pytest -m "mongo or redis" + +# Exclude specific backends +pytest -m "not mongo" + +# Run with verbose output +pytest -v +``` + +### Using the Test Script + +The recommended way to run tests with proper backend setup: + +```bash +# Test single backend +./scripts/test-local.sh mongo + +# Test multiple backends +./scripts/test-local.sh mongo redis sql + +# Test all backends +./scripts/test-local.sh all + +# Run tests in parallel +./scripts/test-local.sh all -p + +# Keep containers running for debugging +./scripts/test-local.sh mongo redis -k +``` + +### Parallel Testing + +Tests can be run in parallel using pytest-xdist: + +```bash +# Run with automatic worker detection +./scripts/test-local.sh all -p + +# Specify number of workers +./scripts/test-local.sh all -p -w 4 + +# Or directly with pytest +pytest -n auto +pytest -n 4 +``` + +## Writing Tests + +### Basic Test Structure + +```python +import pytest +from cachier import cachier + + +def test_basic_caching(): + """Test basic caching functionality.""" + + # Define a cached function local to this test + @cachier() + def expensive_computation(x): + return x**2 + + # First call - should compute + result1 = expensive_computation(5) + assert result1 == 25 + + # Second call - should return from cache + result2 = expensive_computation(5) + assert result2 == 25 + + # Clear cache for cleanup + expensive_computation.clear_cache() +``` + +### Backend-Specific Tests + +```python +@pytest.mark.mongo +def test_mongo_specific_feature(): + """Test MongoDB-specific functionality.""" + from tests.test_mongo_core import _test_mongetter + + @cachier(mongetter=_test_mongetter) + def mongo_cached_func(x): + return x * 2 + + # Test implementation + assert mongo_cached_func(5) == 10 +``` + +## Test Isolation + +### Critical Rule: Function Isolation + +**Never share cachier-decorated functions between test functions.** Each test must have its own decorated function to ensure proper isolation. + +#### Why This Matters + +Cachier identifies cached functions by their full module path and function name. When tests share decorated functions: + +- Cache entries can conflict between tests +- Parallel test execution may fail unpredictably +- Test results become non-deterministic + +#### Good Practice + +```python +def test_feature_one(): + @cachier() + def compute_one(x): # Unique to this test + return x * 2 + + assert compute_one(5) == 10 + + +def test_feature_two(): + @cachier() + def compute_two(x): # Different function for different test + return x * 2 + + assert compute_two(5) == 10 +``` + +#### Bad Practice + +```python +# DON'T DO THIS! +@cachier() +def shared_compute(x): # Shared between tests + return x * 2 + + +def test_feature_one(): + assert shared_compute(5) == 10 # May conflict with test_feature_two + + +def test_feature_two(): + assert shared_compute(5) == 10 # May conflict with test_feature_one +``` + +### Isolation Mechanisms + +1. **Pickle Backend**: Uses `isolated_cache_directory` fixture that creates unique directories per pytest-xdist worker +2. **External Backends**: Rely on function namespacing (module + function name) +3. **Clear Cache**: Always clear cache at test end for cleanup + +### Best Practices for Isolation + +1. Define cached functions inside test functions +2. Use unique, descriptive function names +3. Clear cache after each test +4. Avoid module-level cached functions in tests +5. Use fixtures for common setup/teardown + +## Backend-Specific Testing + +### MongoDB Tests + +```python +@pytest.mark.mongo +def test_mongo_feature(): + """Test with MongoDB backend.""" + + @cachier(mongetter=_test_mongetter, wait_for_calc_timeout=2) + def mongo_func(x): + return x + + # MongoDB-specific assertions + assert mongo_func.get_cache_mongetter() is not None +``` + +### Redis Tests + +```python +@pytest.mark.redis +def test_redis_feature(): + """Test with Redis backend.""" + + @cachier(backend="redis", redis_client=_test_redis_client) + def redis_func(x): + return x + + # Redis-specific testing + assert redis_func(5) == 5 +``` + +### SQL Tests + +```python +@pytest.mark.sql +def test_sql_feature(): + """Test with SQL backend.""" + + @cachier(backend="sql", sql_engine=test_engine) + def sql_func(x): + return x + + # SQL-specific testing + assert sql_func(5) == 5 +``` + +### Memory Tests + +```python +@pytest.mark.memory +def test_memory_feature(): + """Test with memory backend.""" + + @cachier(backend="memory") + def memory_func(x): + return x + + # Memory-specific testing + assert memory_func(5) == 5 +``` + +## Parallel Testing + +### How It Works + +1. pytest-xdist creates multiple worker processes +2. Each worker gets a subset of tests +3. Cachier's function identification ensures natural isolation +4. Pickle backend uses worker-specific cache directories + +### Running Parallel Tests + +```bash +# Automatic worker detection +./scripts/test-local.sh all -p + +# Specify workers +./scripts/test-local.sh all -p -w 4 + +# Direct pytest command +pytest -n auto +``` + +### Parallel Testing Considerations + +1. **Resource Usage**: More workers = more CPU/memory usage +2. **External Services**: Ensure Docker has sufficient resources +3. **Test Output**: May be interleaved; use `-v` for clarity +4. **Debugging**: Harder with parallel execution; use `-n 1` for debugging + +## CI/CD Integration + +### GitHub Actions + +The CI pipeline tests all backends: + +```yaml +# Local backends run in parallel +pytest -m "memory or pickle" -n auto + +# External backends run sequentially for stability +pytest -m mongo +pytest -m redis +pytest -m sql +``` + +### Environment Variables + +- `CACHIER_TEST_VS_DOCKERIZED_MONGO`: Use real MongoDB in CI +- `CACHIER_TEST_REDIS_HOST`: Redis connection details +- `SQLALCHEMY_DATABASE_URL`: SQL database connection + +## Troubleshooting + +### Common Issues + +1. **Import Errors**: Install backend-specific requirements + + ```bash + pip install -r tests/redis_requirements.txt + ``` + +2. **Docker Not Running**: Start Docker Desktop or daemon + + ```bash + docker ps # Check if Docker is running + ``` + +3. **Port Conflicts**: Stop conflicting services + + ```bash + docker stop cachier-test-mongo cachier-test-redis cachier-test-postgres + ``` + +4. **Flaky Tests**: Usually due to timing issues + + - Increase timeouts + - Add proper waits + - Check for race conditions + +5. **Cache Conflicts**: Ensure function isolation + + - Don't share decorated functions + - Clear cache after tests + - Use unique function names + +### Handling Flaky Tests + +Some tests, particularly in the pickle core module, may occasionally fail due to race conditions in multi-threaded scenarios. To handle these, we use the `pytest-rerunfailures` plugin. + +#### Marking Flaky Tests + +```python +@pytest.mark.flaky(reruns=5, reruns_delay=0.1) +def test_that_may_fail_intermittently(): + """This test will retry up to 5 times with 0.1s delay between attempts.""" + # Test implementation +``` + +#### Current Flaky Tests + +- `test_bad_cache_file`: Tests handling of corrupted cache files with concurrent access +- `test_delete_cache_file`: Tests handling of missing cache files during concurrent operations + +These tests involve race conditions between threads that are difficult to reproduce consistently, so they're configured to retry multiple times before being marked as failed. + +### Debugging Tips + +1. **Run Single Test**: + + ```bash + pytest -k test_name -v + ``` + +2. **Disable Parallel**: + + ```bash + pytest -n 1 + ``` + +3. **Check Logs**: + + ```bash + docker logs cachier-test-mongo + ``` + +4. **Interactive Debugging**: + + ```python + import pdb + + pdb.set_trace() + ``` + +### Performance Considerations + +1. **Test Speed**: Memory/pickle tests are fastest +2. **External Backends**: Add overhead for Docker/network +3. **Parallel Execution**: Speeds up test suite significantly +4. **Cache Size**: Large caches slow down tests + +## Best Practices Summary + +1. **Always** define cached functions inside test functions +2. **Never** share cached functions between tests +3. **Clear** cache after each test +4. **Use** appropriate markers for backend-specific tests +5. **Run** full test suite before submitting PRs +6. **Test** with parallel execution to catch race conditions +7. **Document** any special test requirements +8. **Follow** existing test patterns in the codebase + +## Adding New Tests + +When adding new tests: + +1. Follow existing naming conventions +2. Add appropriate backend markers +3. Ensure function isolation +4. Include docstrings explaining test purpose +5. Test both success and failure cases +6. Consider edge cases and error conditions +7. Run with all backends if applicable +8. Update this documentation if needed + +## Questions or Issues? + +- Check existing tests for examples +- Review the main README.rst +- Open an issue on GitHub +- Contact maintainers listed in README.rst diff --git a/tests/conftest.py b/tests/conftest.py index 045e292f..49dc3cbd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,100 @@ """Pytest configuration and shared fixtures for cachier tests.""" -from contextlib import suppress +import logging +import os +from urllib.parse import parse_qs, unquote, urlencode, urlparse, urlunparse import pytest +logger = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def inject_worker_schema_for_sql_tests(monkeypatch, request): + """Automatically inject worker-specific schema into SQL connection string. + + This fixture enables parallel SQL test execution by giving each pytest- + xdist worker its own PostgreSQL schema, preventing table creation + conflicts. + + """ + # Only apply to SQL tests + if "sql" not in request.node.keywords: + yield + return + + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + + if worker_id == "master": + # Not running in parallel, no schema isolation needed + yield + return + + # Get the original SQL connection string + original_url = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" + ) + + if "postgresql" in original_url: + # Create worker-specific schema name + schema_name = f"test_worker_{worker_id.replace('gw', '')}" + + # Parse the URL + parsed = urlparse(original_url) + + # Get existing query parameters + query_params = parse_qs(parsed.query) + + # Add or update the options parameter to set search_path + if "options" in query_params: + # Append to existing options + current_options = unquote(query_params["options"][0]) + new_options = f"{current_options} -csearch_path={schema_name}" + else: + # Create new options + new_options = f"-csearch_path={schema_name}" + + query_params["options"] = [new_options] + + # Rebuild the URL with updated query parameters + new_query = urlencode(query_params, doseq=True) + new_url = urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + new_query, + parsed.fragment, + ) + ) + + # Override both the environment variable and the module constant + monkeypatch.setenv("SQLALCHEMY_DATABASE_URL", new_url) + + # Also patch the SQL_CONN_STR constant used in tests + import tests.test_sql_core + + monkeypatch.setattr(tests.test_sql_core, "SQL_CONN_STR", new_url) + + # Ensure schema creation by creating it before tests run + try: + from sqlalchemy import create_engine, text + + # Use original URL to create schema (without search_path) + engine = create_engine(original_url) + with engine.connect() as conn: + conn.execute( + text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + ) + conn.commit() + engine.dispose() + except Exception as e: + # If we can't create the schema, the test will fail anyway + logger.debug(f"Failed to create schema {schema_name}: {e}") + + yield + @pytest.fixture(scope="session", autouse=True) def cleanup_mongo_clients(): @@ -16,19 +107,143 @@ def cleanup_mongo_clients(): yield # Cleanup after all tests + import contextlib + try: - from tests.mongo_tests.test_mongo_core import _test_mongetter - except ImportError: - return + from tests.test_mongo_core import _mongo_clients, _test_mongetter - client = getattr(_test_mongetter, "client", None) - if client is None: - return + # Close all tracked MongoDB clients + for client in _mongo_clients: + with contextlib.suppress(Exception): + client.close() + + # Clear the list for next test run + _mongo_clients.clear() + + # Also clean up _test_mongetter specifically + if hasattr(_test_mongetter, "client"): + # Remove the client attribute so future test runs start fresh + delattr(_test_mongetter, "client") + + # Clean up any _custom_mongetter functions that may have been created + import tests.test_mongo_core + + for attr_name in dir(tests.test_mongo_core): + attr = getattr(tests.test_mongo_core, attr_name) + if callable(attr) and hasattr(attr, "client"): + delattr(attr, "client") + + except (ImportError, AttributeError): + # If the module wasn't imported or client wasn't created, + # then there's nothing to clean up + pass + + +@pytest.fixture +def worker_id(request): + """Get the pytest-xdist worker ID.""" + return os.environ.get("PYTEST_XDIST_WORKER", "master") + + +@pytest.fixture(autouse=True) +def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): + """Ensure each test gets an isolated cache directory. + + This is especially important for pickle tests when running in parallel. + Each pytest-xdist worker gets its own cache directory to avoid conflicts. + + """ + if "pickle" in request.node.keywords: + # Create a unique cache directory for this test + if worker_id == "master": + # Not running in parallel mode + cache_dir = tmp_path / "cachier_cache" + else: + # Running with pytest-xdist - use worker-specific directory + cache_dir = tmp_path / f"cachier_cache_{worker_id}" + + cache_dir.mkdir(exist_ok=True, parents=True) + + # Monkeypatch the global cache directory for this test + import cachier.config + + monkeypatch.setattr( + cachier.config._global_params, "cache_dir", str(cache_dir) + ) + + # Also set environment variable as a backup + monkeypatch.setenv("CACHIER_TEST_CACHE_DIR", str(cache_dir)) + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_test_schemas(request): + """Clean up test schemas after all tests complete. + + This fixture ensures that worker-specific PostgreSQL schemas created during + parallel test execution are properly cleaned up. + + """ + yield # Let all tests run first + + # Cleanup after all tests + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + + if worker_id != "master": + # Clean up the worker-specific schema + original_url = os.environ.get("SQLALCHEMY_DATABASE_URL", "") + + if "postgresql" in original_url: + schema_name = f"test_worker_{worker_id.replace('gw', '')}" + + try: + from sqlalchemy import create_engine, text + + # Parse URL to remove any schema options for cleanup + parsed = urlparse(original_url) + query_params = parse_qs(parsed.query) + + # Remove options parameter if it exists + query_params.pop("options", None) + + # Rebuild clean URL + clean_query = ( + urlencode(query_params, doseq=True) if query_params else "" + ) + clean_url = urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + clean_query, + parsed.fragment, + ) + ) + + engine = create_engine(clean_url) + with engine.connect() as conn: + # Drop the schema and all its contents + conn.execute( + text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE") + ) + conn.commit() + engine.dispose() + except Exception as e: + # If cleanup fails, it's not critical + logger.debug(f"Failed to cleanup schema {schema_name}: {e}") - # pymongo_inmemory leaves an internal health-check client open. - with suppress(Exception): - client._mongod._client.close() # type: ignore[attr-defined] - client.close() - # Remove the client attribute so future test runs start fresh - delattr(_test_mongetter, "client") +def pytest_addoption(parser): + """Add custom command line options for parallel testing.""" + parser.addoption( + "--parallel", + action="store_true", + default=False, + help="Run tests in parallel using pytest-xdist", + ) + parser.addoption( + "--parallel-workers", + action="store", + default="auto", + help="Number of parallel workers (default: auto)", + ) diff --git a/tests/requirements_mongodb.txt b/tests/mongodb_requirements.txt similarity index 87% rename from tests/requirements_mongodb.txt rename to tests/mongodb_requirements.txt index 45e68808..ceac4be2 100644 --- a/tests/requirements_mongodb.txt +++ b/tests/mongodb_requirements.txt @@ -1,4 +1,3 @@ --r requirements.txt # to connect to the test mongodb server pymongo dnspython diff --git a/tests/requirements_redis.txt b/tests/redis_requirements.txt similarity index 66% rename from tests/requirements_redis.txt rename to tests/redis_requirements.txt index e734210b..9814a933 100644 --- a/tests/requirements_redis.txt +++ b/tests/redis_requirements.txt @@ -1,5 +1,3 @@ --r requirements.txt - redis>=4.0.0 pandas>=1.3.0 birch>=0.0.35 diff --git a/tests/requirements.txt b/tests/requirements.txt index 78297278..75e0e190 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,8 @@ # todo: add some version range or pinning latest versions # tests and coverages pytest +pytest-xdist # for parallel test execution +pytest-rerunfailures # for retrying flaky tests coverage pytest-cov pytest-asyncio diff --git a/tests/requirements_postgres.txt b/tests/requirements_postgres.txt deleted file mode 100644 index 382c48c7..00000000 --- a/tests/requirements_postgres.txt +++ /dev/null @@ -1,4 +0,0 @@ --r requirements.txt -# for SQL core tests -SQLAlchemy -psycopg[binary] # Psycopg3 diff --git a/tests/sql_requirements.txt b/tests/sql_requirements.txt new file mode 100644 index 00000000..81af8428 --- /dev/null +++ b/tests/sql_requirements.txt @@ -0,0 +1,3 @@ +# for SQL core tests +SQLAlchemy +psycopg2-binary