diff --git a/.dockerignore b/.dockerignore index e713043..c2e4622 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,7 @@ -tests/test_init_project.py \ No newline at end of file +tests/test_init_project.py + +**/__pycache__/ +**/*.pyc +**/*.pyo +**/*.pyd +.pytest_cache/ \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b755d0e..700b139 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -2,7 +2,7 @@ name: main on: push: - branches: [main, develop] + branches: [main] pull_request: jobs: @@ -10,10 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4.1.0 + uses: actions/setup-python@v6 with: python-version: 3.12 @@ -24,28 +24,31 @@ jobs: run: flake8 --count tests: - name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + name: tests - Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: true matrix: os: ["ubuntu-latest"] - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.13"] env: VENV: .venv steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' # cache option make the step fail if you don´t have requirements.txt or pyproject.toml on root. # https://github.com/actions/setup-python/issues/807. + - name: Install UV + run: make uv + - name: Create virtual environment - run: python -m venv $VENV + run: uv venv $VENV - name: Install package with test dependencies run: | @@ -60,19 +63,20 @@ jobs: - name: Upload coverage reports to Codecov if: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') && matrix.python-version == '3.12' - uses: codecov/codecov-action@v4.0.1 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} tests-in-docker: + name: tests - Python 3.12 on Docker runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4.1.0 + uses: actions/setup-python@v6 - name: Build docker image run: make docker-build - name: Run tests - run: make docker-ci-test + run: make docker-test diff --git a/.gitignore b/.gitignore index eb99da3..0838c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ terraform.tfstate.backup # project stuff scripts/config.sh test/ + +# UV +.python-version \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d55211d..d83c281 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,13 +65,18 @@ make docker-build make docker-gcp ``` +Install UV for faster installs (otherwise modify Makefile to use regular pip): +```shell +make uv +``` + 4. Create virtual environment and activate it: ```shell make venv ./.venv/bin/activate ``` -5. Install dependencies and the python package: +5. Install all dependencies for development and the python package in editable mode: ```shell make install ``` diff --git a/Dockerfile b/Dockerfile index 983b9f3..726671e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,60 @@ # --------------------------------------------------------------------------------------- -# BASE IMAGE +# BUILDER # --------------------------------------------------------------------------------------- -FROM python:3.12.10-slim-bookworm AS base +FROM python:3.12-slim-bookworm AS builder -# Setup a volume for configuration and authtentication. VOLUME ["/root/.config"] -# Update system and install build tools. Remove unneeded stuff afterwards. -# Upgrade PIP. -# Create working directory. -RUN apt-get update && \ - apt-get install -y --no-install-recommends gcc g++ build-essential && \ - rm -rf /var/lib/apt/lists/* && \ - pip install --upgrade pip && \ - mkdir -p /opt/project +# Use uv for high-speed installs +COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv -# Set working directory. -WORKDIR /opt/project +ENV UV_COMPILE_BYTECODE=1 -# --------------------------------------------------------------------------------------- -# DEPENDENCIES IMAGE (installed project dependencies) -# --------------------------------------------------------------------------------------- -# We do this first so when we modify code while development, this layer is reused -# from cache and only the layer installing the package executes again. -FROM base AS deps -COPY requirements.txt . -RUN pip install -r requirements.txt +COPY pyproject.toml requirements.txt README.md MANIFEST.in ./ +COPY src ./src -# --------------------------------------------------------------------------------------- -# Apache Beam integration IMAGE -# --------------------------------------------------------------------------------------- -FROM deps AS beam -# Copy files from official SDK image, including script/dependencies. -# IMPORTANT: This version must match the one in requirements.txt -COPY --from=apache/beam_python3.12_sdk:2.64.0 /opt/apache/beam /opt/apache/beam - -# Set the entrypoint to Apache Beam SDK launcher. -ENTRYPOINT ["/opt/apache/beam/boot"] +RUN uv pip install --system --upgrade pip && \ + uv pip install --system build && \ + uv pip install --system --prefix=/install -r requirements.txt && \ + uv pip install --system --prefix=/install --no-deps . # --------------------------------------------------------------------------------------- # PRODUCTION IMAGE # --------------------------------------------------------------------------------------- -# If you need Apache Beam integration, replace "deps" base image with "beam". -FROM deps AS prod +FROM python:3.12-slim-bookworm AS prod + +ENV PYTHONUNBUFFERED=1 -COPY . /opt/project -RUN pip install . && \ - rm -rf /root/.cache/pip && \ - rm -rf /opt/project/* +# Copy the pre-compiled packages from builder +COPY --from=builder /install /usr/local + +# APACHE BEAM INTEGRATION (Uncomment if needed) +# COPY --from=apache/beam_python3.12_sdk:2.71.0 /opt/apache/beam /opt/apache/beam +# ENTRYPOINT ["/opt/apache/beam/boot"] + +WORKDIR /opt/project # --------------------------------------------------------------------------------------- -# DEVELOPMENT IMAGE (editable install and development tools) +# DEVELOPMENT IMAGE # --------------------------------------------------------------------------------------- -# If you need Apache Beam integration, replace "deps" base image with "beam". -FROM deps AS dev +FROM builder AS dev -COPY . /opt/project -RUN make install +WORKDIR /opt/project + +COPY . . +RUN uv pip install --system -e .[lint,dev,build] && \ + uv pip install --system -r requirements-test.txt # --------------------------------------------------------------------------------------- -# TEST IMAGE (This one allows to check that package is properly installed in prod image) +# TEST IMAGE # --------------------------------------------------------------------------------------- FROM prod AS test -COPY ./tests /opt/project/tests -COPY ./requirements-test.txt /opt/project/ +COPY ./requirements-test.txt . +RUN pip install -r requirements-test.txt + +COPY ./tests ./tests -RUN pip install -r requirements-test.txt \ No newline at end of file +# Suppress all warnings during tests +# To see/address warnings, run tests in your development environment. +ENV PYTHONWARNINGS=ignore \ No newline at end of file diff --git a/Makefile b/Makefile index be8b58e..bbc8a5a 100644 --- a/Makefile +++ b/Makefile @@ -2,17 +2,25 @@ VENV_NAME:=.venv REQS_PROD:=requirements.txt +SETUP_FILE:=pyproject.toml +SOURCES = src + DOCKER_DEV_SERVICE:=dev -DOCKER_CI_TEST_SERVICE:=test -DOCKER_ISOLATED_SERVICE:=isolated +DOCKER_DEV_NO_GCP_SERVICE:=dev_no_gcp +DOCKER_PROD_SERVICE:=prod +DOCKER_TEST_SERVICE:=test GCP_PROJECT:=world-fishing-827 GCP_DOCKER_VOLUME:=gcp -sources = python_app_template +PYTHON_VERSION:=3.12 +UV_VERSION := 0.10.9 + +VENV:=uv venv +PIP:=uv pip +PIP_COMPILE:=uv pip compile + -PYTHON:=python -PIP:=${PYTHON} -m pip # --------------------- # DOCKER @@ -32,31 +40,36 @@ docker-gcp: docker-volume docker compose run gcloud config set project ${GCP_PROJECT} docker compose run gcloud auth application-default set-quota-project ${GCP_PROJECT} -.PHONY: docker-ci-test ## Runs tests using prod image, exporting coverage.xml report. -docker-ci-test: - docker compose run --rm ${DOCKER_CI_TEST_SERVICE} +.PHONY: docker-test ## Runs tests using prod image, exporting coverage.xml report. +docker-test: + docker compose run --rm ${DOCKER_TEST_SERVICE} .PHONY: docker-shell ## Enters to docker container shell. docker-shell: docker-volume docker compose run --rm -it ${DOCKER_DEV_SERVICE} -.PHONY: reqs ## Compiles requirements.txt with pip-tools. +.PHONY: docker-reqs ## Compiles requirements.txt with pip-tools. reqs: - docker compose run --rm ${DOCKER_ISOLATED_SERVICE} -c \ - 'pip-compile -o ${REQS_PROD} -v' + docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \ + '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -v' -.PHONY: reqs-upgrade ## Upgrades requirements.txt with pip-tools. +.PHONY: docker-reqs-upgrade ## Upgrades requirements.txt with pip-tools. reqs-upgrade: - docker compose run --rm ${DOCKER_ISOLATED_SERVICE} -c \ - 'pip-compile -o ${REQS_PROD} -U -v' + docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \ + '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -U -v' # --------------------- # VIRTUAL ENVIRONMENT # --------------------- +.PHONY: uv ## Installs UV +uv: + curl -LsSf https://astral.sh/uv/install.sh | UV_VERSION=$(UV_VERSION) sh + uv python pin ${PYTHON_VERSION} + .PHONY: venv ## Creates virtual environment. venv: - ${PYTHON} -m venv ${VENV_NAME} + ${VENV} ${VENV_NAME} .PHONY: upgrade-pip ## Upgrades pip. upgrade-pip: @@ -68,11 +81,12 @@ install-test: upgrade-pip .PHONY: install ## Install the package in editable mode & all dependencies for local development. install: upgrade-pip - ${PIP} install -e .[lint,dev,build,test] + ${PIP} install -e .[lint,dev,build] + make install-test .PHONY: test ## Run all unit tests exporting coverage.xml report. test: - ${PYTHON} -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(sources) + python -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(SOURCES) # --------------------- # QUALITY CHECKS @@ -80,36 +94,36 @@ test: .PHONY: hooks ## Install and pre-commit hooks. hooks: - ${PYTHON} -m pre_commit install --install-hooks - ${PYTHON} -m pre_commit install --hook-type commit-msg + python -m pre_commit install --install-hooks + python -m pre_commit install --hook-type commit-msg .PHONY: format ## Auto-format python source files according with PEP8. format: - ${PYTHON} -m black $(sources) - ${PYTHON} -m ruff check --fix $(sources) - ${PYTHON} -m ruff format $(sources) + python -m black $(SOURCES) + python -m ruff check --fix $(SOURCES) + python -m ruff format $(SOURCES) .PHONY: lint ## Lint python source files. lint: - ${PYTHON} -m ruff check $(sources) - ${PYTHON} -m ruff format --check $(sources) - ${PYTHON} -m black $(sources) --check --diff + python -m ruff check $(SOURCES) + python -m ruff format --check $(SOURCES) + python -m black $(SOURCES) --check --diff .PHONY: codespell ## Use Codespell to do spell checking. codespell: - ${PYTHON} -m codespell + python -m codespell .PHONY: typecheck ## Perform type-checking. typecheck: - ${PYTHON} -m mypy + python -m mypy .PHONY: audit ## Use pip-audit to scan for known vulnerabilities. audit: - ${PYTHON} -m pip_audit . + python -m pip_audit . .PHONY: pre-commit ## Run all pre-commit hooks. pre-commit: - ${PYTHON} -m pre_commit run --all-files + python -m pre_commit run --all-files .PHONY: all ## Run the standard set of checks performed in CI. all: lint codespell typecheck audit test @@ -121,7 +135,7 @@ all: lint codespell typecheck audit test .PHONY: build ## Build a source distribution and a wheel distribution. build: all clean - ${PYTHON} -m build + python -m build .PHONY: publish ## Publish the distribution to PyPI. publish: build diff --git a/README.md b/README.md index d6db6e2..d81548f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - Python versions + Python versions Last release diff --git a/docker-compose.yml b/docker-compose.yml index b734138..b155461 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,36 +6,39 @@ services: - 'gcp:/root/.config/' entrypoint: gcloud - dev: + dev: &dev + image: gfw/python-app-template-dev + platform: linux/amd64 build: - context: . target: dev - platform: linux/amd64 volumes: - '.:/opt/project' - 'gcp:/root/.config/' entrypoint: /bin/bash + dev_no_gcp: + image: gfw/python-app-template-dev + # Dev container without GCP credentials. + <<: *dev + volumes: + - ".:/opt/project" + + prod: + image: gfw/python-app-template-prod + build: + target: prod + platform: linux/amd64 + entrypoint: /bin/bash + test: + image: gfw/python-app-template-test platform: linux/amd64 # Runs tests using the production Docker image. # Intended to be executed in the GitHub CI environment. build: - context: . target: test entrypoint: 'pytest -v' - isolated: - # Minimal dev container without GCP credentials. - # Used for tasks that don't need external dependencies (e.g., compiling requirements). - build: - context: . - target: dev - platform: linux/amd64 - volumes: - - '.:/opt/project' - entrypoint: /bin/bash - volumes: gcp: external: true diff --git a/init_project.py b/init_project.py index 4562d9f..42607ea 100644 --- a/init_project.py +++ b/init_project.py @@ -7,6 +7,7 @@ FILES_TO_UPDATE = ( "cloudbuild.yaml", "CONTRIBUTING.md", + "docker-compose.yml", "Makefile", "pyproject.toml", "README.md", diff --git a/requirements.txt b/requirements.txt index 240e028..b403fac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --output-file=requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile -o requirements.txt pyproject.toml markdown-it-py==4.0.0 # via rich mdurl==0.1.2 @@ -11,10 +7,10 @@ mdurl==0.1.2 pygments==2.19.2 # via rich pyyaml==6.0.3 - # via python-app-template (setup.py) + # via python-app-template (pyproject.toml) rich==14.2.0 # via - # python-app-template (setup.py) + # python-app-template (pyproject.toml) # rich-argparse rich-argparse==1.7.2 - # via python-app-template (setup.py) + # via python-app-template (pyproject.toml)