diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7d8fe1c26987..000000000000 --- a/.coveragerc +++ /dev/null @@ -1,53 +0,0 @@ -# .coveragerc for edx-platform -[run] -branch=true -data_file = reports/.coverage -source = - cms - common/djangoapps - lms - openedx - pavelib - scripts - xmodule - -omit = - cms/envs/* - cms/manage.py - cms/djangoapps/contentstore/views/dev.py - cms/djangoapps/*/migrations/* - cms/djangoapps/*/features/* - cms/lib/*/migrations/* - lms/debug/* - lms/envs/* - lms/djangoapps/*/migrations/* - lms/djangoapps/*/features/* - common/djangoapps/*/migrations/* - openedx/core/djangoapps/*/migrations/* - openedx/core/djangoapps/debug/* - openedx/envs/* - openedx/features/*/migrations/* - -concurrency=multiprocessing -parallel = true -relative_files = true - -[report] -ignore_errors = True - -exclude_lines = - pragma: no cover - raise NotImplementedError - -[html] -title = edx-platform Python Test Coverage Report -directory = reports/cover - -[xml] -output = reports/coverage.xml - -[paths] -jenkins_source = - /home/jenkins/workspace/$JOB_NAME - /home/jenkins/workspace/$SUBSET_JOB - /home/jenkins/edx-platform diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 89210e854f07..83528986575b 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -39,7 +39,7 @@ jobs: git fetch origin "$BASE_SHA" # The ^"? is because git may quote weird file paths - if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))'; then + if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/)|(scripts/[^/]+/pyproject\.toml)|(scripts/[^/]+/uv\.lock)|(pyproject\.toml)|(uv\.lock))'; then echo "RELEVANT=true" >> "$GITHUB_ENV" fi @@ -48,6 +48,17 @@ jobs: with: python-version: '3.12' + - name: install system requirements + if: ${{ env.RELEVANT == 'true' }} + run: | + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev + + - name: Install uv + if: ${{ env.RELEVANT == 'true' }} + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: "Recompile requirements" if: ${{ env.RELEVANT == 'true' }} run: | diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml index 664d50f0e545..8b4c8ab13ef1 100644 --- a/.github/workflows/check_python_dependencies.yml +++ b/.github/workflows/check_python_dependencies.yml @@ -1,12 +1,20 @@ +# Temporarily disabled: `find_python_dependencies` (from edx-repo-tools) only +# knows how to scan pip-compile's requirements/edx/{base,testing}.txt for +# undocumented git-hosted dependencies. Those files were removed when this +# repo switched to pyproject.toml + uv (see +# https://github.com/openedx/public-engineering/issues/543). Re-enable once +# edx-repo-tools can scan pyproject.toml/uv.lock instead — tracked at +# https://github.com/openedx/repo-tools/issues/725, fix proposed in +# https://github.com/openedx/repo-tools/pull/735. name: Check Python Dependencies on: - pull_request: - merge_group: + workflow_dispatch: jobs: check_dependencies: runs-on: ubuntu-latest + if: false strategy: matrix: diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index db004ee9b495..5634e2e2e694 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -24,23 +24,15 @@ jobs: - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev - - name: Install pip - run: make pre-requirements - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install python dependencies - run: make dev-requirements + run: | + make dev-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Static code analysis run: make check-types diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index c9ce6a808bed..b488019a88bc 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -28,6 +28,15 @@ jobs: with: python-version: "3.12" + - name: install system requirements + run: | + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Run make compile-requirements env: PACKAGE: "${{ inputs.package }}" @@ -48,7 +57,13 @@ jobs: with: branch: "${{ github.triggering_actor }}/compile-python-deps" branch-suffix: short-commit-hash - add-paths: requirements + add-paths: | + requirements + scripts/**/pyproject.toml + scripts/**/uv.lock + scripts/**/requirements* + pyproject.toml + uv.lock commit-message: | feat: Recompile Python dependencies diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 123934055bf9..903908d8ec84 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -48,18 +48,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install Required Python Dependencies run: | diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index eba3bf2bb0cc..4c3a27c22c4f 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -24,23 +24,15 @@ jobs: - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev - - name: Install pip - run: make pre-requirements - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install python dependencies - run: make dev-requirements + run: | + make dev-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" # As long there are sub-projects[1] in openedx-platform, we analyze each # project separately here, in order to make import-linting errors easier diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 0993fbdd0a4f..f01f8f9d4156 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -83,30 +83,23 @@ jobs: sudo apt-get update make ubuntu-requirements - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install Python dependencies run: | make dev-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken + uv pip install "django~=${{ matrix.django-version }}.0" + uv pip check # fail if this test-reqs/Django combination is broken fi - name: list installed package versions run: | - sudo pip freeze + uv pip freeze - name: Run Tests env: diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 6adf688c17f7..d810e62006b1 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -42,27 +42,20 @@ jobs: with: python-version: 3.12 - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install required Python dependencies run: | # dev-requirements is needed because the linter will otherwise # trip over some dev-only things like django-debug-toolbar - # (import debug_toolbar) that aren't in testing.txt. + # (import debug_toolbar) that aren't in the testing group. make dev-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" # After all requirements are installed, check that they're consistent with each other - pip check + uv pip check - name: Run quality tests run: | diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index a412e6015bea..a8caa69534f5 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -43,36 +43,23 @@ jobs: - name: Setup npm run: npm i -g npm@8.5.x - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - name: Install Required Python Dependencies env: PIP_SRC: ${{ runner.temp }} run: | make test-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Install npm env: PIP_SRC: ${{ runner.temp }} run: npm ci - - name: Install python packages - env: - PIP_SRC: ${{ runner.temp }} - run: | - pip install -e . - - name: Run Quality Tests env: PIP_SRC: ${{ runner.temp }} diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 2a971adae360..c46c341b6b21 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -31,10 +31,15 @@ jobs: with: python-version: "${{ matrix.python-version }}" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Install semgrep run: | - make pre-requirements - pip-sync requirements/edx/semgrep.txt + uv sync --only-group semgrep --frozen + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Run semgrep env: diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index a82b97d4e86f..c655f0963689 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -56,29 +56,15 @@ jobs: - name: Setup npm run: npm i -g npm@${{ matrix.npm-version }} - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true - - name: Install Limited Python Deps for Build + - name: Install Python Deps for Build and Collection run: | - # Install pip-tools.txt first to pin setuptools<82 before installing - # assets.txt. setuptools 82+ removed pkg_resources, which pyfilesystem2 - # (fs) still uses for namespace package declarations. The constraints.txt - # pin covers full installs, but this step only installs assets.txt so we - # pre-install pip-tools.txt to ensure setuptools 81.x is in place. - # See: https://github.com/PyFilesystem/pyfilesystem2/issues/577 - pip install -r requirements/pip-tools.txt - pip install -r requirements/edx/assets.txt + uv sync --group assets --frozen + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH @@ -97,10 +83,6 @@ jobs: npm clean-install npm run build - - name: Install Full Python Deps for Collection - run: | - pip install -r requirements/edx/base.txt -e . - - name: Check Assets Collection env: LMS_CFG: lms/envs/minimal.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e458c2bf103c..0d9d2346e328 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -91,17 +91,23 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: install requirements run: | make test-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken + uv pip install "django~=${{ matrix.django-version }}.0" + uv pip check # fail if this test-reqs/Django combination is broken fi - name: list installed package versions run: | - pip freeze + uv pip freeze - name: set settings path shell: bash @@ -176,9 +182,15 @@ jobs: run: | sudo apt-get update && sudo apt-get install libxmlsec1-dev + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: install requirements run: | make test-requirements + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: collect tests from all modules shell: bash @@ -311,7 +323,7 @@ jobs: - name: Install Python dependencies run: | - pip install -r requirements/edx/coverage.txt + pip install coverage diff-cover - name: Run coverage run: | diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index d0d55808a82f..baa0a62a1d34 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -41,12 +41,48 @@ jobs: with: python-version: "3.12" + - name: install system requirements + run: | + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Update any pinned dependencies env: NEW_VERSION: "${{ inputs.version }}" PACKAGE: "${{ inputs.package }}" run: | - sed 's/^\('$PACKAGE'[^#]*\)==[^ #]\+/\1=='$NEW_VERSION'/' -i requirements/constraints.txt + # Constraints now live in [tool.edx_lint].uv_constraints in pyproject.toml + # (previously requirements/constraints.txt), so this needs to edit that + # TOML array rather than sed-patching a plain-text file. --no-project + # avoids syncing the whole project just to edit a TOML file with tomlkit. + test -z "$NEW_VERSION" && exit 0 + uv run --no-project --with tomlkit python3 - <<'PYEOF' + import os + import re + + import tomlkit + + package = os.environ["PACKAGE"] + new_version = os.environ["NEW_VERSION"] + + + def normalized_name(spec): + return re.split(r"[<>=!~\[; ]", spec, maxsplit=1)[0].lower().replace("_", "-") + + + path = "pyproject.toml" + doc = tomlkit.parse(open(path, encoding="utf-8").read()) + constraints = doc["tool"]["edx_lint"]["uv_constraints"] + target = normalized_name(package) + for i, spec in enumerate(constraints): + if normalized_name(str(spec)) == target and "==" in str(spec): + constraints[i] = re.sub(r"==[^,]+", f"=={new_version}", str(spec)) + open(path, "w", encoding="utf-8").write(tomlkit.dumps(doc)) + PYEOF - name: Run make upgrade-package env: @@ -88,6 +124,8 @@ jobs: branch-suffix: short-commit-hash add-paths: | requirements + scripts/**/pyproject.toml + scripts/**/uv.lock scripts/**/requirements* commit-message: | feat: Upgrade Python dependency ${{ inputs.package }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fef9e3c2a9d2..339a61e63718 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,16 +4,23 @@ build: os: "ubuntu-lts-latest" tools: python: "3.12" + jobs: + post_create_environment: + - pip install uv + # Export [project.dependencies] *and* the `docs` group together (--group, not + # --only-group -- the latter excludes [project.dependencies] entirely, which + # would leave Django/XBlock/etc. for `pip install -e .` below to resolve on + # its own, unconstrained by uv.lock's [tool.uv].constraint-dependencies, e.g. + # picking setuptools>=82 and breaking fs/pyfilesystem2's pkg_resources import) + # to a plain requirements.txt, installed into RTD's own managed virtualenv + # below rather than a separate uv-managed one -- this keeps RTD's normal + # sphinx auto-build detection intact. + - uv export --frozen --no-hashes --group docs --no-emit-project -o doc-requirements.txt + post_install: + - python -m pip install -r doc-requirements.txt + # --no-deps: dependencies are already installed, pinned, from doc-requirements.txt + # above; this step only registers the local package itself for import. + - python -m pip install -e . --no-deps sphinx: configuration: docs/conf.py - -python: - install: - # Need to install this to set the correct version of steuptools for now - # because it is needed by fs - # See https://github.com/openedx/openedx-platform/issues/38068 for details. - - requirements: "requirements/pip-tools.txt" - - requirements: "requirements/edx/doc.txt" - - method: pip - path: . diff --git a/Makefile b/Makefile index 98931cb608d2..a24fc315d103 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ compile-requirements detect_changed_source_translations dev-requirements \ docs extract_translations \ guides help lint-imports local-requirements migrate migrate-lms migrate-cms \ - pre-requirements pull pull_xblock_translations pull_translations push_translations \ + pull pull_xblock_translations pull_translations push_translations \ requirements shell swagger \ technical-docs test-requirements ubuntu-requirements upgrade-package upgrade @@ -64,83 +64,110 @@ pull_translations: clean_translations ## pull translations via atlas detect_changed_source_translations: ## check if translation files are up-to-date i18n_tool changed -pre-requirements: ## install Python requirements for running pip-tools - pip install -r requirements/pip-tools.txt +local-requirements: ## no-op; `uv sync` (used by the targets below) already installs -e . itself + @true -local-requirements: -# edx-platform installs some Python projects from within the edx-platform repo itself. - pip install -e . +dev-requirements: ## install development environment requirements + uv sync --group default --frozen -dev-requirements: pre-requirements - @# The "$(wildcard..)" is to include private.txt if it exists, and make no mention - @# of it if it does not. Shell wildcarding can't do that with default options. - pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt) - make local-requirements +base-requirements: ## install only production/runtime dependencies + uv sync --no-default-groups --group non-core --frozen -base-requirements: pre-requirements - pip-sync requirements/edx/base.txt - make local-requirements - -test-requirements: pre-requirements - pip-sync --pip-args="--exists-action=w" requirements/edx/testing.txt - make local-requirements +test-requirements: ## install production dependencies plus the testing group (used by CI and tox) + uv sync --no-default-groups --group testing --frozen requirements: dev-requirements ## install development environment requirements -# Order is very important in this list: files must appear after everything they include! -REQ_FILES = \ - requirements/edx/coverage \ - requirements/edx-sandbox/base \ - requirements/edx/base \ - requirements/edx/doc \ - requirements/edx/testing \ - requirements/edx/assets \ - requirements/edx/development \ - requirements/edx/semgrep \ - scripts/xblock/requirements \ - scripts/user_retirement/requirements/base \ - scripts/user_retirement/requirements/testing \ - scripts/structures_pruning/requirements/base \ - scripts/structures_pruning/requirements/testing - -define COMMON_CONSTRAINTS_TEMP_COMMENT -# This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n -endef - -COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt -.PHONY: $(COMMON_CONSTRAINTS_TXT) -$(COMMON_CONSTRAINTS_TXT): - curl -L https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt > "$(@)" - printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) - -compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade -compile-requirements: pre-requirements ## Re-compile *.in requirements to *.txt - @# Bootstrapping: Rebuild pip and pip-tools first, and then install them - @# so that if there are any failures we'll know now, rather than the next - @# time someone tries to use the outputs. - sed 's/Django<5.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt - sed 's/pip<25.3//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt - - pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in - pip install -r requirements/pip-tools.txt - - @ export REBUILD='--rebuild'; \ - for f in $(REQ_FILES); do \ +# uv-managed sub-projects, each with their own pyproject.toml + uv.lock, +# independent of the root project's dependency graph: +# requirements/edx-sandbox, scripts/xblock, scripts/user_retirement, +# scripts/structures_pruning +# Their compatibility .txt exports (for anyone still installing via plain +# pip) don't follow a uniform naming/grouping scheme, so each is handled +# explicitly in compile-requirements below rather than through one generic +# loop over a shared list. + +compile-requirements: ## Regenerate uv.lock for the root project and all uv sub-projects + uv run --no-project --with edx-lint edx_lint write_uv_constraints pyproject.toml + uv lock ${UV_LOCK_OPTS} + + @# Compatibility exports for external tooling (e.g. tutor's Dockerfile) that + @# still does `pip install -r requirements/edx/.txt` directly. These are + @# GENERATED FILES -- see the header comment in each for what regenerates them. + @mkdir -p requirements/edx + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of [project.dependencies] plus the 'non-core' group"; \ + echo "# (optional third-party add-ons installed by default) for tools that still"; \ + echo "# 'pip install -r requirements/edx/base.txt' directly instead of using uv."; \ + echo "# Source of truth: [project.dependencies] / [dependency-groups].non-core in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --no-default-groups --group non-core --no-emit-project; \ + } > requirements/edx/base.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of the 'assets' dependency-group for tools that still"; \ + echo "# 'pip install -r requirements/edx/assets.txt' directly instead of using uv."; \ + echo "# Source of truth: [dependency-groups].assets in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --only-group assets --no-emit-project; \ + } > requirements/edx/assets.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of the 'default' dependency-group for tools that still"; \ + echo "# 'pip install -r requirements/edx/development.txt' directly instead of using uv."; \ + echo "# Source of truth: [dependency-groups].default in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --group default --no-emit-project; \ + } > requirements/edx/development.txt + + @# requirements/edx-sandbox and scripts/xblock: single compat export, no dependency-groups. + @for d in requirements/edx-sandbox scripts/xblock; do \ + echo ; \ + echo "== $$d ===============================" ; \ + uv run --no-project --with edx-lint edx_lint write_uv_constraints $$d/pyproject.toml && \ + (cd $$d && uv lock ${UV_LOCK_OPTS}) \ + || exit 1; \ + done + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r requirements/edx-sandbox/base.txt'"; \ + echo "# directly instead of using uv. Source of truth: requirements/edx-sandbox/pyproject.toml / uv.lock."; \ + (cd requirements/edx-sandbox && uv export --frozen --no-hashes --no-emit-project); \ + } > requirements/edx-sandbox/base.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r scripts/xblock/requirements.txt'"; \ + echo "# directly instead of using uv. Source of truth: scripts/xblock/pyproject.toml / uv.lock."; \ + (cd scripts/xblock && uv export --frozen --no-hashes --no-emit-project); \ + } > scripts/xblock/requirements.txt + + @# scripts/user_retirement and scripts/structures_pruning: base + testing (test group) compat exports. + @for d in scripts/user_retirement scripts/structures_pruning; do \ echo ; \ - echo "== $$f ===============================" ; \ - echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ - pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ - export REBUILD=''; \ + echo "== $$d ===============================" ; \ + uv run --no-project --with edx-lint edx_lint write_uv_constraints $$d/pyproject.toml && \ + (cd $$d && uv lock ${UV_LOCK_OPTS}) \ + || exit 1; \ + done + @for d in scripts/user_retirement scripts/structures_pruning; do \ + { \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/base.txt'"; \ + echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml / uv.lock."; \ + (cd $$d && uv export --frozen --no-hashes --no-emit-project); \ + } > $$d/requirements/base.txt; \ + { \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/testing.txt'"; \ + echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml (test group) / uv.lock."; \ + (cd $$d && uv export --frozen --no-hashes --group test --no-emit-project); \ + } > $$d/requirements/testing.txt; \ done -upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the pip requirements files to use the latest releases satisfying our constraints - $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" +upgrade: ## update all dependencies (uv.lock for the root project and all uv sub-projects) to the latest releases satisfying our constraints + $(MAKE) compile-requirements UV_LOCK_OPTS="--upgrade" upgrade-package: ## update just one package to the latest usable release @test -n "$(package)" || { echo "\nUsage: make upgrade-package package=...\n"; exit 1; } - $(MAKE) compile-requirements COMPILE_OPTS="--upgrade-package $(package)" + $(MAKE) compile-requirements UV_LOCK_OPTS="--upgrade-package $(package)" check-types: ## run static type-checking tests mypy diff --git a/README.rst b/README.rst index 3dd3930626f0..ab0ff56cdc3b 100644 --- a/README.rst +++ b/README.rst @@ -99,12 +99,12 @@ Language Packages: * Backend build: - - ``pip install -r requirements/edx/assets.txt`` + - ``uv sync --group assets`` * Backend application: - - ``pip install -r requirements/edx/base.txt`` (production) - - ``pip install -r requirements/edx/development.txt`` (development) + - ``uv sync --no-default-groups`` (production) + - ``uv sync --group default`` (development) Some Python packages have system dependencies. For example, installing these packages on Debian or Ubuntu will require first running ``sudo apt install python3-dev default-libmysqlclient-dev build-essential pkg-config`` to satisfy the requirements of the ``mysqlclient`` Python package. diff --git a/docs/concepts/testing/testing.rst b/docs/concepts/testing/testing.rst index 5e4248523c65..cd32f44eba8e 100644 --- a/docs/concepts/testing/testing.rst +++ b/docs/concepts/testing/testing.rst @@ -99,7 +99,8 @@ Running Python Unit tests ************************* The following commands need to be run within a Python environment in -which requirements/edx/testing.txt has been installed. If you are using a +which the ``testing`` dependency group (``uv sync --only-group testing``) has +been installed. If you are using a Docker-based Open edX distribution, then you probably will want to run these commands within the LMS and/or CMS Docker containers. diff --git a/docs/references/static-assets.rst b/docs/references/static-assets.rst index 9a3e3c25b663..c1e3970bb21d 100644 --- a/docs/references/static-assets.rst +++ b/docs/references/static-assets.rst @@ -72,7 +72,7 @@ Building frontend assets requires an active Node and Python environment with dependencies installed:: npm clean-install - pip install -r requirements/edx/assets.txt + uv sync --group assets Once your environment variables are set and build dependencies are installed, @@ -139,7 +139,7 @@ skip this section. First, ensure you have a Python enironment with all edx-platform dependencies installed:: - pip install -r requirements/edx/base.txt -e . + uv sync Next, download localized versions of edx-platform assets. Under the hood, this command uses the `Open edX Atlas`_ tool, which manages aggregated translations diff --git a/pyproject.toml b/pyproject.toml index 44488938ec95..da219b831b2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,409 @@ build-backend = "setuptools.build_meta" name = "openedx-platform" version = "0.13" requires-python = ">=3.12" -dependencies = ["setuptools"] +dependencies = [ + "acid-xblock", # This XBlock is used for unit tests as well as manual tests + "analytics-python", # Used for Segment analytics + "attrs", # Reduces boilerplate code involving class attributes + "Babel", # Internationalization utilities, used for date formatting in a few places + "boto3", # Amazon Web Services SDK for Python + "botocore", # via boto3, s3transfer + "bridgekeeper", # Used for determining permissions for courseware. Use of django-rules is preferred. + "celery", # Asynchronous task execution library + "chem", # A helper library for chemistry calculations + "codejail-includes", # CodeJail manages execution of untrusted code in secure sandboxes. + "cryptography", # Implementations of assorted cryptography algorithms + "defusedxml", + "Django", # Web application framework + "django-autocomplete-light", # Enhances Django admin with single-select autocomplete dropdowns for a better user experience. + "django-celery-results", # Only used for the CacheBackend for celery results + "django-config-models", # Configuration models for Django allowing config management with auditing + "django-cors-headers", # Used to allow to configure CORS headers for cross-domain requests + "django-countries", # Country data for Django forms and model fields + "django-crum", # Middleware that stores the current request and user in thread local storage + "django-filter", # Allows users to filter Django querysets dynamically + "django-ipware", # Get the client's real IP address + "django-method-override", # Allow use of HTTP methods normally not supported by browsers via the X-HTTP-Method-Override header + "django-model-utils", + "django-mptt", + "django-mysql", + "django-oauth-toolkit", # Provides oAuth2 capabilities for Django + "django-pipeline", + "django-ratelimit", + "django-sekizai", + "django-simple-history", + "django-statici18n", + "django-storages", + "django-user-tasks", + "django-waffle", + "django-webpack-loader", # Used to wire webpack bundles into the django asset pipeline + "djangorestframework", + "drf-spectacular", + "edx-ace", + "edx-api-doc-tools", + "edx-auth-backends", # Allow Studio to use LMS SSO + "edx-bulk-grades", # LMS REST API for managing bulk grading operations + "edx-ccx-keys", + "edx-celeryutils", + "edx-completion", + "edx-django-release-util", # Release utils for the edx release pipeline + "edx-django-sites-extensions", + "edx-codejail>=4.0.0", # Codejail 4 brings important safety improvements (no unsafe mode by default) + "edx-django-utils>=5.14.1", # edx-django-utils 5.14.1 adds FrontendMonitoringMiddleware; Utilities for cache, monitoring, and plugins + "edx-drf-extensions", + "edx-enterprise", + "edx-event-bus-kafka>=5.6.0", # edx-event-bus-kafka 5.6.0 adds support for putting client ids on event producers/consumers; Kafka implementation of event bus + "edx-event-bus-redis", + "edx-milestones", + "edx-opaque-keys>=2.12.0", + "edx-organizations", + "edx-proctoring>=2.0.1", + "edx-rest-api-client", # using hash to support django42 + "edx-search", + "edx-submissions", + "edx-toggles", # Feature toggles management + "edx-when", + "edxval", + "event-tracking", + "enmerkar-underscore", # Implements a underscore extractor for django-babel. + "fs", + "fs-s3fs", + "geoip2", # Python API for the GeoIP web services and databases + "glob2", # Enhanced glob module, used in openedx.core.lib.rooted_paths + "gunicorn", + "help-tokens", + "html5lib", # HTML parser, used for capa problems + "icalendar", # .ics generator, used by calendar_sync + "ipaddress", # Ip network support for Embargo feature + "importlib_metadata", # Used to access entry_points in i18n_api plugin + "jsonfield", # Django model field for validated JSON; used in several apps + "laboratory", # Library for testing that code refactors/infrastructure changes produce identical results + "lxml[html_clean]", # XML parser + "lti-consumer-xblock>=11.0.0", + "mako", # Primary template language used for server-side page rendering + "Markdown", # Convert text markup to HTML; used in capa problems, forums, and course wikis + "meilisearch", # Library to access Meilisearch search engine (will replace ElasticSearch) + "mongoengine", # Object-document mapper for MongoDB, used in the LMS dashboard + "mysqlclient", # Driver for the default production relational database + "nh3", # Python bindings to the ammonia (whitelist-based HTML sanitizing library); used for capa and LTI + "nodeenv", # Utility for managing Node.js environments; we use this for deployments and testing + "oauthlib", # OAuth specification support for authenticating via LTI or other Open edX services + "olxcleaner", + "openedx-atlas", # CLI tool to manage translations + "openedx-calc", # Library supporting mathematical calculations for Open edX + "openedx-core", + "openedx-django-require", + "openedx-events", # Open edX Events from Hooks Extension Framework (OEP-50) + "openedx-filters", # Open edX Filters from Hooks Extension Framework (OEP-50) + "openedx-forum", # Open edX forum v2 application + "openedx-django-wiki", + "path", + "piexif", # Exif image metadata manipulation, used in the profile_images app + "Pillow", # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. + "psutil", # Library for retrieving information on running processes and system utilization + "pycountry", + "pycryptodomex", + "PyJWT>=1.6.3", # PyJWT 1.6.3 contains PyJWTError, which is required by Apple auth in social-auth-core + "pylti1p3", # Required by content_libraries core library to support LTI 1.3 launches + "pymemcache", # Python interface to the memcached memory cache daemon + "pymongo", # MongoDB driver + "pynliner", # Inlines CSS styles into HTML for email notifications + "python-dateutil", + "python3-openid ; python_version>='3'", + "python3-saml", + "pyuca", # For more accurate sorting of translated country names in django-countries + "pysrt", # Support for SubRip subtitle files, used in the video XModule + "pytz", # Time zone information database + "PyYAML", # Used to parse XModule resource templates + "redis", # celery task broker + "requests-oauthlib", # Simplifies use of OAuth via the requests library, used for CCX and LTI + "random2", + "rules", # Django extension for rules-based authorization checks + "social-auth-core", + "simplejson", + "Shapely", # Geometry library, used for image click regions in capa + "six", # Utilities for supporting Python 2 & 3 in the same codebase + "slumber", # The following dependency is unsupported and used by the DeprecatedRestApiClient + "social-auth-app-django", + "sorl-thumbnail", + "sortedcontainers", # Provides SortedKeyList, used for lists of XBlock assets + "stevedore", # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins + "unicodecsv", # Easier support for CSV files with unicode text + "webob", + "web-fragments", # Provides the ability to render fragments of web pages + "wrapt", # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? + "XBlock[django]", # Courseware component architecture + "xss-utils", # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations + "unicodeit", # Converts mathjax equation to plain text by using unicode symbols + "psycopg2-binary", + "openedx-authz", # Authorization Framework for the Open edX Ecosystem +] + +[project.optional-dependencies] +openstack = [ + "django-storage-swift==1.2.19", +] +[dependency-groups] +coverage = [ + "coverage", # Code coverage testing for Python + "diff-cover", # Automatically find diff lines that need test coverage +] +non-core = [ + # Dependencies that are normally bundled with the platform, but are not core to + # the platform's functionality. They are still installed by default (this group is + # included from `testing`, and explicitly synced in `base-requirements`), but they + # should not be required for the platform to run nor for the test suite to pass. + "edx-i18n-tools>=0.4.6", # i18n_tool is needed at build time for pulling translations; Commands for developers and translators to extract, compile and validate translations + "django-ses", # Django email backend for Amazon’s Simple Email Service + "mailsnake", # MailChimp API; used for two management commands in the "mailing" djangoapp + "optimizely-sdk", # Optimizely provides A/B testing and other features, used by edx.org + "crowdsourcehinter-xblock", + "done-xblock", # a very simple XBlock that allows learners to mark an activity as Done + "recommender-xblock", # https://github.com/edx/RecommenderXBlock + "staff-graded-xblock", # https://github.com/openedx/staff_graded-xblock Allows off-site bulk scoring. + "edx-sga", # The more well known "staff graded assignment" XBlock, from MIT. + "ora2>=4.5.0", # Open Response Assessment XBlock + "xblock-poll", # Xblock for polling users + "xblock-drag-and-drop-v2", # Drag and Drop XBlock + "xblock-google-drive", # XBlock for google docs and calendar + "xblocks-contrib", # Package having multiple core XBlocks, https://github.com/openedx/xblocks-contrib?tab=readme-ov-file#xblocks-being-moved-here + "enterprise-integrated-channels", # Integrated Channels to transmit content metadata and learner data. +] +testing = [ + {include-group = "coverage"}, + {include-group = "non-core"}, + "beautifulsoup4", # Library for extracting data from HTML and XML files + "code-annotations", # Perform code annotation checking, such as for PII annotations + "cssselect", # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery + "ddt", # Run a test case multiple times with different input; used in many, many of our tests + "edx-lint", # pylint extensions for Open edX repositories + "factory-boy", # Library for creating test fixtures, used in many tests + "freezegun", # Pinning the freezegun version because 0.3.13 is causing failures which have also been reported on the git repo by public.; Allows tests to mock the output of assorted datetime module functions + "httpretty", # Library for mocking HTTP requests, used in many tests + "import-linter", # Tool for making assertions about which modules can import which others + "mock", # Deprecated alias to standard library `unittest.mock` + "ruff", # Fast Python linter and formatter + "polib", # Library for manipulating gettext translation files, used to test paver i18n commands + "pyquery", # jQuery-like API for retrieving fragments of HTML and XML files in tests + "pytest", # Testing framework + "pytest-attrib", # Select tests based on attributes + "pytest-cov", # pytest plugin for measuring code coverage + "pytest-django", # Django support for pytest + "pytest-json-report", # Output json formatted warnings after running pytest + "pytest-metadata", # To prevent 'make upgrade' failure, dependency of pytest-json-report + "pytest-randomly", # pytest plugin to randomly order tests + "pytest-reportlog", # Per-test timing data including setup/teardown (used for shard rebalancing) + "pytest-xdist[psutil]", # Parallel execution of tests on multiple CPU cores or hosts + "singledispatch", # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering + "testfixtures", # Provides a LogCapture utility used by several tests + "unidiff", # Required by coverage_pytest_plugin + "pylint-pytest", # A Pylint plugin to suppress pytest-related false positives. + "pact-python", # Library for contract testing + "py", # Needed for pytest configurations, was previously been fetched through tox +] +docs = [ + "code-annotations", # provides annotations for certain documentation + "sphinx-book-theme", # Common theme for all Open edX projects + "gitpython", # fetch git repo information + "Sphinx", # Documentation builder + "sphinx-design", # provides various responsive web-components + "sphinxcontrib-openapi", # Be able to render openapi schema in a sphinx project + "sphinxext-rediraffe", # Quickly and easily redirect when we move pages around. + "sphinx-reredirects", # Redirect from a sphinx project out to other places on the web including other sphinx projects + "sphinx-autoapi", +] +assets = [ + "click", + "libsass", + "nodeenv", +] +development = [ + {include-group = "testing"}, + {include-group = "docs"}, + {include-group = "assets"}, + "django-debug-toolbar", # A set of panels that display debug information about the current request/response + "django-stubs[compatible-mypy]", # Typing stubs for Django, so it works with mypy + "djangorestframework-stubs", # Typing stubs for DRF + "mypy", # static type checking + "pywatchman", # More efficient checking for runserver reload trigger events + "types-requests", # Typing stubs for requests + "vulture", # Detects possible dead/unused code, used in scripts/find-dead-code.sh + "watchdog", # Used by `npm run watch` to auto-recompile when assets are changed +] +semgrep = [ + "semgrep", # Semgrep performs structural code searches +] +ci = [ + "tox", + "tox-uv", +] +# The group installed by a bare `uv sync` with no --group/--only-group flags (see +# `default-groups` below). Named `default` (rather than `dev`, uv's own built-in +# default group name) to avoid confusion with the very similarly-named `development` +# group that it wraps. +default = [ + {include-group = "development"}, + {include-group = "ci"}, +] + +[tool.uv] +default-groups = ["default"] + +# DO NOT EDIT constraint-dependencies DIRECTLY. +# This list is managed by `edx_lint write_uv_constraints` +# and will be overwritten the next time `make upgrade` is run. +# - GLOBAL constraints: edit edx_lint/files/common_constraints.txt +# - REPO-SPECIFIC constraints: edit [tool.edx_lint].uv_constraints in this file +constraint-dependencies = [ + "Django<6.0", + "elasticsearch==7.9.1", + "celery>=5.2.2,!=5.6.1,<6.0.0", + "django-oauth-toolkit==1.7.1", + "django-stubs<6", + "edx-enterprise==8.7.2", + "libsass==0.10.0", + "numpy<2.0.0", + "openedx-core<2", + "openai<=0.28.1", + "path<16.12.0", + "py2neo<2022", + "pymongo<4.4.1", + "social-auth-app-django<=5.4.1", + "social-auth-core<5.0.0", + "lxml==5.3.2", + "xmlsec==1.3.14", + "django-debug-toolbar<6.0.0", + "sphinx-autoapi<3.6.1", + "setuptools<82", + "astroid==4.0.4", +] +[tool.edx_lint] +uv_constraints = [ + # Date: 2025-10-07 + # Stay on LTS version, remove once this is added to common constraint + "Django<6.0", + # Date: 2026-01-13 + # We would normally pin celery to <6.0.0 to avoid auto-updating across a major + # version boundary without more thorough testing. The reason it's currently also + # pinned to !=5.6.1 is because of a celery bug related to the eta and countdown + # parameters. This bug caused operational issues in MIT's deployment. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280 + "celery>=5.2.2,!=5.6.1,<6.0.0", + # Date: 2020-02-10 + # django-oauth-toolkit version >=2.0.0 has breaking changes. More details + # mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35277 + "django-oauth-toolkit==1.7.1", + # Date: 2024-07-19 + # Generally speaking, the major version of django-stubs must either match the major version + # of django, or exceed it by 1. So, we will need to perpetually constrain django-stubs and + # update it as we perform django upgrades. For more details, see: + # https://github.com/typeddjango/django-stubs?tab=readme-ov-file#version-compatibility + # including the note on "Partial Support". + # Issue: https://github.com/openedx/edx-platform/issues/35275 + "django-stubs<6", + # Date: 2019-08-16 + # The team that owns this package will manually bump this package rather than having it pulled in automatically. + # This is to allow them to better control its deployment and to do it in a process that works better + # for them. + "edx-enterprise==8.7.2", + # Date: 2023-07-26 + # Our legacy Sass code is incompatible with anything except this ancient libsass version. + # Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving + # away from legacy LMS/CMS frontends: + # https://github.com/openedx/edx-platform/issues/31616 + "libsass==0.10.0", + # Date: 2024-07-16 + # We need to upgrade the version of elasticsearch to at least 7.15 before we can upgrade to Numpy 2.0.0 + # Otherwise we see a failure while running the following command: + # export DJANGO_SETTINGS_MODULE=cms.envs.test; python manage.py cms check_reserved_keywords --override_file db_keyword_overrides.yml --report_path reports/reserved_keywords --report_file cms_reserved_keyword_report.csv + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 + "numpy<2.0.0", + # Date: 2023-09-18 + # Library is still in active development. Major versions require review + # from openedx-maintainers. Minor and patch versions can roll out automatically. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 + "openedx-core<2", + # Date: 2023-11-29 + # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35268 + "openai<=0.28.1", + # Date: 2024-04-26 + # path==16.12.0 breaks the unit test collections check + # needs to be investigated and fixed separately + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35267 + "path<16.12.0", + # Date: 2021-08-25 + # At the time of writing this comment, we do not know whether py2neo>=2022 + # will support our currently-deployed Neo4j version (3.5). + # Feel free to loosen this constraint if/when it is confirmed that a later + # version of py2neo will work with Neo4j 3.5. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35266 + "py2neo<2022", + # Date: 2020-04-08 + # Adding pin to avoid any major upgrade + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35265 + "pymongo<4.4.1", + # Date: 2024-08-06 + # social-auth-app-django 5.4.2 introduces a new migration that will not play nicely with large installations. This will touch + # user tables, which are quite large, especially on instances like edx.org. + # We are pinning this until after all the smaller migrations get handled and then we can migrate this all at once. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/37639 + "social-auth-app-django<=5.4.1", + # Date 2026-07-02 + # social-auth-core 5.x changes the OAuth pipeline's post-login redirect + # behavior, breaking common/djangoapps/third_party_auth's integration + # tests (AzureAD/Google/LinkedIn/Twitter full-pipeline specs). Keep on + # the 4.x line until that's investigated and fixed separately -- see the + # social-auth-app-django pin above for a related, already-deferred + # migration in this dependency family. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/38841 + "social-auth-core<5.0.0", + # Date 2025-01-08 + # elasticsearch==7.13.x is downgrading urllib3 from 2.2.3 to 1.26.20 + # https://github.com/elastic/elasticsearch-py/blob/v7.13.4/setup.py#L42 + # We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 + "elasticsearch==7.9.1", + # Date 2025-05-09 + # lxml and xmlsec need to be constrained because the latest version builds against a newer + # version of libxml2 than what we're running with. This leads to a version mismatch error + # at runtime. You can re-produce it by running any test. + # If lxml is pinned in the future and you see this error, it may be that the system libxml2 + # is now shipping the correct version and we can un-pin this. + # Issue: https://github.com/openedx/edx-platform/issues/36695 + "lxml==5.3.2", + "xmlsec==1.3.14", + # Date 2025-08-12 + # The newest version of the debug toolbar has a bug in it + # https://github.com/django-commons/django-debug-toolbar/issues/2172 + # Pin this back to the previous version until that bug is fixed. + "django-debug-toolbar<6.0.0", + # Date 2026-01-13 + # Sphinx-autoapi changed the version of astroid it needs + # but the newer version is not compatible with the current pylint version + # which wants a newer version of astroid. This can be removed once we're + # building requirements with Python 3.12 + # https://github.com/openedx/edx-platform/issues/37880 + "sphinx-autoapi<3.6.1", + # Date 2026-03-02 + # setuptools 82.0.0 removed pkg_resources from its distribution, but fs (pyfilesystem2) + # still uses pkg_resources for namespace package declarations. This constraint can be + # removed once pyfilesystem2 drops its pkg_resources usage. + # https://github.com/PyFilesystem/pyfilesystem2/issues/577 + # Issue for unpinning: https://github.com/openedx/openedx-platform/issues/38068 + "setuptools<82", + # Date 2026-03-02 + # The latest version of pylint pins back astroid to an older version. + # This holdback is not caught in the docs requirements file and since both the docs + # and testing file are required in the development.in file, we fail to compile + # development.txt because of conflicting dependencies. + # + # Holding astroid back until pylint releases a new version that works with the latest + # version of astroid. + # https://github.com/openedx/openedx-platform/issues/38066 + "astroid==4.0.4", +] [tool.setuptools] packages = ["cms", "common", "lms", "openedx", "xmodule"] @@ -206,6 +608,60 @@ norecursedirs = ". .* *.egg build conf dist node_modules test_root cms/envs lms/ python_classes = [] python_files = ["tests.py", "test_*.py", "tests_*.py", "*_tests.py", "__init__.py"] +[tool.coverage.run] +branch = true +data_file = "reports/.coverage" +source = [ + "cms", + "common/djangoapps", + "lms", + "openedx", + "pavelib", + "scripts", + "xmodule", +] +omit = [ + "cms/envs/*", + "cms/manage.py", + "cms/djangoapps/contentstore/views/dev.py", + "cms/djangoapps/*/migrations/*", + "cms/djangoapps/*/features/*", + "cms/lib/*/migrations/*", + "lms/debug/*", + "lms/envs/*", + "lms/djangoapps/*/migrations/*", + "lms/djangoapps/*/features/*", + "common/djangoapps/*/migrations/*", + "openedx/core/djangoapps/*/migrations/*", + "openedx/core/djangoapps/debug/*", + "openedx/envs/*", + "openedx/features/*/migrations/*", +] +concurrency = ["multiprocessing"] +parallel = true +relative_files = true + +[tool.coverage.report] +ignore_errors = true +exclude_lines = [ + "pragma: no cover", + "raise NotImplementedError", +] + +[tool.coverage.html] +title = "edx-platform Python Test Coverage Report" +directory = "reports/cover" + +[tool.coverage.xml] +output = "reports/coverage.xml" + +[tool.coverage.paths] +jenkins_source = [ + "/home/jenkins/workspace/$JOB_NAME", + "/home/jenkins/workspace/$SUBSET_JOB", + "/home/jenkins/edx-platform", +] + [tool.ruff] line-length = 120 exclude = [ diff --git a/requirements/README.rst b/requirements/README.rst index c602cc8905b1..8b53f9f691cb 100644 --- a/requirements/README.rst +++ b/requirements/README.rst @@ -1,18 +1,29 @@ Requirements/dependencies ######################### -These directories specify the Python (and system) dependencies for the LMS and Studio. - -- ``edx`` contains the normal Python requirements files -- ``edx-sandbox`` contains the requirements files for Codejail -- ``constraints.txt`` is shared between the two - -(In a normal `OEP-18`_-compliant repository, the ``*.in`` and ``*.txt`` files would be -directly in the requirements directory.) - -.. _OEP-18: https://github.com/openedx/open-edx-proposals/blob/master/oeps/oep-0018-bp-python-dependencies.rst - -While the ``*.in`` files are intended to be updated manually, the ``*.txt`` files should only be manipulated using Makefile targets in a Linux environment (to match our build and deploy systems). For developers on Mac, this can be achieved by using the GitHub workflows or by running Make targets from inside devstack's lms-shell or another Linux environment. +The main application's Python dependencies are declared in the root +``pyproject.toml`` (``[project.dependencies]`` for runtime deps, +``[dependency-groups]`` for testing/development/doc/assets/semgrep tooling, +and ``[tool.edx_lint].uv_constraints`` for repo-specific version pins) and +locked in the root ``uv.lock``, managed with `uv`_. + +This ``requirements/`` directory now only holds ``edx-sandbox``, its own +standalone ``uv``-managed project (``pyproject.toml`` + ``uv.lock``) for +Codejail's isolated sandbox environment. + +The three standalone script directories at the repo root (``scripts/xblock``, +``scripts/user_retirement``, ``scripts/structures_pruning``) each have their +own ``pyproject.toml`` + ``uv.lock`` too, independent of both the main app and +each other. This completes the migration from pip-compile tracked in +`public-engineering#543`_. + +All of these are manipulated using the Makefile targets below in a Linux +environment (to match our build and deploy systems); for developers on Mac, +this can be achieved by using the GitHub workflows or by running Make targets +from inside devstack's lms-shell or another Linux environment. + +.. _uv: https://docs.astral.sh/uv/ +.. _public-engineering#543: https://github.com/openedx/public-engineering/issues/543 If you don't have write permissions to openedx/edx-platform, you'll need to run these workflows on a fork. @@ -22,7 +33,9 @@ Workflows and Makefile targets Add a dependency ================ -To add a Python dependency, specify it in the appropriate ``requirements/edx/*.in`` file, push that up to a branch, and then use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. This will ensure the lockfiles are updated with any transitive dependencies and will ping you on a PR for updating your branch. +To add a Python dependency, add it to ``[project.dependencies]`` (or the +appropriate ``[dependency-groups]`` entry) in ``pyproject.toml``, push that up +to a branch, and then use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. This will ensure ``uv.lock`` is updated with any transitive dependencies and will ping you on a PR for updating your branch. Upgrade just one dependency =========================== @@ -31,17 +44,17 @@ Want to upgrade just *one* dependency without pulling in other upgrades? You can Or, if you need to do it locally, you can use the ``upgrade-package`` make target directly. For example, you could run ``make upgrade-package package=ecommerce``. -If your dependency is pinned in constraints.txt, you'll need to enter an explicit version number in the appropriate field when running the workflow; this will include an update to the constraint file in the resulting PR. +If your dependency is pinned in ``[tool.edx_lint].uv_constraints`` (in ``pyproject.toml``), you'll need to enter an explicit version number in the appropriate field when running the workflow; this will include an update to that constraint in the resulting PR. Downgrade a dependency ====================== If you instead need to surgically *downgrade* a dependency: -1. Add an exact-match or max-version constraint to ``constraints.txt`` with a comment explaining why (and ideally a ticket or issue link). Here's what it might look like:: +1. Add an exact-match or max-version constraint to ``[tool.edx_lint].uv_constraints`` in ``pyproject.toml`` with a comment explaining why (and ideally a ticket or issue link). Here's what it might look like:: # frobulator 2.x has breaking API changes; see https://github.com/openedx/edx-platform/issue/1234567 for fixing it - frobulator<2.0.0 + "frobulator<2.0.0", 2. After pushing that up to a branch, use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. @@ -78,7 +91,7 @@ Luckily, we have simple runbooks for upgrading or downgrading a single package, Is there an unpinned git dependency? ==================================== -If the diff relates to a dependency that is installed from git rather than from PyPI (such as being a transitive dependency of anything in github.in), check whether any of the dependencies in github.in has failed to pin a specific commit. We want to have as few of these dependencies as possible, as they're a maintenance and performance problem, and there are important instructions at the top of that file for how to manage them. +If the diff relates to a dependency that is installed from git rather than from PyPI, check ``[project.dependencies]`` in ``pyproject.toml`` for a direct reference (``name @ git+https://...@TAG-OR-SHA``) that has failed to pin a specific commit. We want to have as few of these dependencies as possible, as they're a maintenance and performance problem. Help, I didn't change any dependencies, and this is still failing! ================================================================== diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt deleted file mode 100644 index 748858b7015a..000000000000 --- a/requirements/common_constraints.txt +++ /dev/null @@ -1,24 +0,0 @@ -# This is a temporary solution to override the real common_constraints.txt -# In edx-lint, until the pyjwt constraint in edx-lint has been removed. -# See BOM-2721 for more details. -# Below is the copied and edited version of common_constraints -# A central location for most common version constraints -# (across edx repos) for pip-installation. -# -# Similar to other constraint files this file doesn't install any packages. -# It specifies version constraints that will be applied if a package is needed. -# When pinning something here, please provide an explanation of why it is a good -# idea to pin this package across all edx repos, Ideally, link to other information -# that will help people in the future to remove the pin when possible. -# Writing an issue against the offending project and linking to it here is good. -# -# Note: Changes to this file will automatically be used by other repos, referencing -# this file from Github directly. It does not require packaging in edx-lint. - -# using LTS django version -Django<6.0 - -# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. -# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html -# See https://github.com/openedx/edx-platform/issues/35126 for more info -elasticsearch<7.14.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt deleted file mode 100644 index 7a365e18aa74..000000000000 --- a/requirements/constraints.txt +++ /dev/null @@ -1,152 +0,0 @@ -# Version constraints for pip-installation. -# -# This file doesn't install any packages. It specifies version constraints -# that will be applied if a package is needed. -# -# When pinning something here, please provide an explanation of why. Ideally, -# link to other information that will help people in the future to remove the -# pin when possible. Writing an issue against the offending project and -# linking to it here is good. -# For further details on how to properly write constraints here please consult -# https://openedx.atlassian.net/wiki/spaces/COMM/pages/4400250883/Adding+pinned+dependencies+in+constraint+file - -# This file contains all common constraints for edx-repos --c common_constraints.txt - -# Date: 2025-10-07 -# Stay on LTS version, remove once this is added to common constraint -Django<6.0 - -# Date: 2026-01-13 -# We would normally pin celery to <6.0.0 to avoid auto-updating across a major -# version boundary without more thorough testing. The reason it's currently also -# pinned to !=5.6.1 is because of a celery bug related to the eta and countdown -# parameters. This bug caused operational issues in MIT's deployment. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280 -celery>=5.2.2,!=5.6.1,<6.0.0 - -# Date: 2020-02-10 -# django-oauth-toolkit version >=2.0.0 has breaking changes. More details -# mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35277 -django-oauth-toolkit==1.7.1 - -# Date: 2024-07-19 -# Generally speaking, the major version of django-stubs must either match the major version -# of django, or exceed it by 1. So, we will need to perpetually constrain django-stubs and -# update it as we perform django upgrades. For more details, see: -# https://github.com/typeddjango/django-stubs?tab=readme-ov-file#version-compatibility -# including the note on "Partial Support". -# Issue: https://github.com/openedx/edx-platform/issues/35275 -django-stubs<6 - -# Date: 2019-08-16 -# The team that owns this package will manually bump this package rather than having it pulled in automatically. -# This is to allow them to better control its deployment and to do it in a process that works better -# for them. -edx-enterprise==8.7.2 - -# Date: 2023-07-26 -# Our legacy Sass code is incompatible with anything except this ancient libsass version. -# Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving -# away from legacy LMS/CMS frontends: -# https://github.com/openedx/edx-platform/issues/31616 -libsass==0.10.0 - -# Date: 2024-07-16 -# We need to upgrade the version of elasticsearch to at least 7.15 before we can upgrade to Numpy 2.0.0 -# Otherwise we see a failure while running the following command: -# export DJANGO_SETTINGS_MODULE=cms.envs.test; python manage.py cms check_reserved_keywords --override_file db_keyword_overrides.yml --report_path reports/reserved_keywords --report_file cms_reserved_keyword_report.csv -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 -numpy<2.0.0 - -# Date: 2023-09-18 -# Library is still in active development. Major versions require review -# from openedx-maintainers. Minor and patch versions can roll out automatically. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 -openedx-core<2 - -# Date: 2023-11-29 -# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35268 -openai<=0.28.1 - -# Date: 2024-04-26 -# path==16.12.0 breaks the unit test collections check -# needs to be investigated and fixed separately -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35267 -path<16.12.0 - -# Date: 2021-08-25 -# At the time of writing this comment, we do not know whether py2neo>=2022 -# will support our currently-deployed Neo4j version (3.5). -# Feel free to loosen this constraint if/when it is confirmed that a later -# version of py2neo will work with Neo4j 3.5. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35266 -py2neo<2022 - -# Date: 2020-04-08 -# Adding pin to avoid any major upgrade -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35265 -pymongo<4.4.1 - -# Date: 2024-08-06 -# social-auth-app-django 5.4.2 introduces a new migration that will not play nicely with large installations. This will touch -# user tables, which are quite large, especially on instances like edx.org. -# We are pinning this until after all the smaller migrations get handled and then we can migrate this all at once. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/37639 -social-auth-app-django<=5.4.1 - -# # Date: 2024-10-14 -# # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first. -# edx-rest-api-client==5.7.1 - -# Date 2025-01-08 -# elasticsearch==7.13.x is downgrading urllib3 from 2.2.3 to 1.26.20 -# https://github.com/elastic/elasticsearch-py/blob/v7.13.4/setup.py#L42 -# We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 -elasticsearch==7.9.1 - -# Date 2025-05-09 -# lxml and xmlsec need to be constrained because the latest version builds against a newer -# version of libxml2 than what we're running with. This leads to a version mismatch error -# at runtime. You can re-produce it by running any test. -# If lxml is pinned in the future and you see this error, it may be that the system libxml2 -# is now shipping the correct version and we can un-pin this. -# Issue: https://github.com/openedx/edx-platform/issues/36695 -lxml==5.3.2 -xmlsec==1.3.14 - -# Date 2025-08-12 -# The newest version of the debug toolbar has a bug in it -# https://github.com/django-commons/django-debug-toolbar/issues/2172 -# Pin this back to the previous version until that bug is fixed. -django-debug-toolbar<6.0.0 - -# Date 2026-01-13 -# Sphinx-autoapi changed the version of astroid it needs -# but the newer version is not compatible with the current pylint version -# which wants a newer version of astroid. This can be removed once we're -# building requirements with Python 3.12 -# https://github.com/openedx/edx-platform/issues/37880 -sphinx-autoapi<3.6.1 - -# Date 2026-03-02 -# setuptools 82.0.0 removed pkg_resources from its distribution, but fs (pyfilesystem2) -# still uses pkg_resources for namespace package declarations. This constraint can be -# removed once pyfilesystem2 drops its pkg_resources usage. -# https://github.com/PyFilesystem/pyfilesystem2/issues/577 -# Issue for unpinning: https://github.com/openedx/openedx-platform/issues/38068 -setuptools<82 - -# Date 2026-03-02 -# The latest version of pylint pins back astroid to an older version. -# This holdback is not caught in the docs requirements file and since both the docs -# and testing file are required in the development.in file, we fail to compile -# development.txt because of conflicting dependencies. -# -# Holding astroid back until pylint releases a new version that works with the latest -# version of astroid. -# https://github.com/openedx/openedx-platform/issues/38066 -astroid==4.0.4 diff --git a/requirements/edx-sandbox/README.rst b/requirements/edx-sandbox/README.rst index 96c90ee911f4..2a3836563170 100644 --- a/requirements/edx-sandbox/README.rst +++ b/requirements/edx-sandbox/README.rst @@ -10,22 +10,27 @@ within ``