infra!: added CI/CD pipeline and made various improvements to Makefile and Dockerfile

2025-12-21 11:59:55 +01:00
parent ffd2678c91
commit f969724ee4
5 changed files with 133 additions and 18 deletions
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -0,0 +1,54 @@
+name: CI/CD Awesome Pipeline
+
+on:
+  push:
+    branches: [main]
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches: [main]
+
+env:
+  REGISTRY_URL: ${{ vars.REGISTRY_URL || 'gitea.iswearihadsomethingforthis.net' }}
+  REGISTRY_USER: ${{ vars.REGISTRY_USER || 'francwa' }}
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build and run tests
+        run: make _ci-run-tests
+
+  build-and-push:
+    name: Build & Push to Registry
+    runs-on: ubuntu-latest
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Load config from Makefile
+        id: config
+        run: |
+          eval "$(make _ci-image-name)"
+          echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
+
+      - name: Extract version from tag
+        id: version
+        run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+
+      - name: Build production image
+        run: make build
+
+      - name: Tag and push to registry
+        run: |
+          docker tag ${{ steps.config.outputs.image_name }}:latest ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_USER }}/${{ steps.config.outputs.image_name }}:${{ steps.version.outputs.version }}
+          docker tag ${{ steps.config.outputs.image_name }}:latest ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_USER }}/${{ steps.config.outputs.image_name }}:latest
+          echo "${{ secrets.GITEA_TOKEN }}" | docker login ${{ env.REGISTRY_URL }} -u ${{ env.REGISTRY_USER }} --password-stdin
+          docker push ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_USER }}/${{ steps.config.outputs.image_name }}:${{ steps.version.outputs.version }}
+          docker push ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_USER }}/${{ steps.config.outputs.image_name }}:latest
--- a/47
+++ b/47
@@ -3,9 +3,15 @@
 .DEFAULT_GOAL := help

 # --- SETTINGS ---
+PYTHON_VERSION = 3.12.7
+PYTHON_VERSION_SHORT = $(shell echo $(PYTHON_VERSION) | cut -d. -f1,2)
 # Change to 'uv' when ready.
 RUNNER ?= poetry
+
+export PYTHON_VERSION
+export PYTHON_VERSION_SHORT
 export RUNNER
+export IMAGE_NAME

 # --- VARIABLES ---
 CORE_DIR     = brain
@@ -20,7 +26,12 @@ INSTALL_CMD = $(if $(filter uv,$(RUNNER)),sync,install)
 ARGS           = $(filter-out $@,$(MAKECMDGOALS))
 BUMP_CMD       = cd $(CORE_DIR) && $(RUNNER) run bump-my-version bump
 COMPOSE_CMD    = docker-compose
-DOCKER_CMD     = cd $(CORE_DIR) && docker build --build-arg RUNNER=$(RUNNER) -t $(IMAGE_NAME):latest .
+DOCKER_CMD     = docker build \
+ --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+ --build-arg PYTHON_VERSION_SHORT=$(PYTHON_VERSION_SHORT) \
+ --build-arg RUNNER=$(RUNNER) \
+ -f $(CORE_DIR)/Dockerfile \
+ -t $(IMAGE_NAME):latest .

 RUNNER_ADD     = cd $(CORE_DIR) && $(RUNNER) add
 RUNNER_HOOKS   = cd $(CORE_DIR) && $(RUNNER) run pre-commit install -c ../.pre-commit-config.yaml
@@ -35,7 +46,7 @@ T = \033[36m
 R = \033[0m

 # --- TARGETS ---
-.PHONY: add build check-docker check-runner clean coverage down format help init-env install install-hooks lint logs major minor patch prune ps restart run shell test up update _check_branch
+.PHONY: add build build-test check-docker check-runner clean coverage down format help init-dotenv install install-hooks lint logs major minor patch prune ps restart run shell test up update _check_branch _ci-image-name _ci-run-tests

 # Catch-all for args
 %:
@@ -50,6 +61,17 @@ build: check-docker
 	$(DOCKER_CMD)
 	@echo "✅ Image $(IMAGE_NAME):latest ready."

+build-test: check-docker
+	@echo "$(T)🐳 Building test image (with dev deps)...$(R)"
+	docker build \
+		--build-arg RUNNER=$(RUNNER) \
+		--build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+		--build-arg PYTHON_VERSION_SHORT=$(PYTHON_VERSION_SHORT) \
+		-f $(CORE_DIR)/Dockerfile \
+		--target test \
+		-t $(IMAGE_NAME):test .
+	@echo "✅ Test image $(IMAGE_NAME):test ready."
+
 check-docker:
 	@command -v docker >/dev/null 2>&1 || { echo "$(R)❌ Docker not installed$(R)"; exit 1; }
 	@docker info >/dev/null 2>&1 || { echo "$(R)❌ Docker daemon not running$(R)"; exit 1; }
@@ -87,12 +109,13 @@ help:
 	@echo "$(G)Setup:$(R)"
 	@echo "  $(T)check-docker 		$(R)  Verify Docker is installed and running."
 	@echo "  $(T)check-runner 		$(R)  Verify package manager ($(RUNNER))."
-	@echo "  $(T)init-env     		$(R)  Create .env from .env.example with generated secrets."
+	@echo "  $(T)init-dotenv  		$(R)  Create .env from .env.example with generated secrets."
 	@echo "  $(T)install      		$(R)  Install ALL dependencies (Prod + Dev)."
 	@echo "  $(T)install-hooks		$(R)  Install git pre-commit hooks."
 	@echo ""
 	@echo "$(G)Docker:$(R)"
-	@echo "  $(T)build        		$(R)  Build the docker image."
+	@echo "  $(T)build        		$(R)  Build the docker image (production)."
+	@echo "  $(T)build-test   		$(R)  Build the docker image (with dev deps for testing)."
 	@echo "  $(T)down         		$(R)  Stop and remove containers."
 	@echo "  $(T)logs         		$(R)  Follow logs."
 	@echo "  $(T)prune        		$(R)  Clean Docker system."
@@ -107,13 +130,13 @@ help:
 	@echo "  $(T)coverage     		$(R)  Run tests with coverage."
 	@echo "  $(T)format       		$(R)  Format code (Ruff)."
 	@echo "  $(T)lint         		$(R)  Lint code without fixing."
-	@echo "  $(T)test ...     		$(R)  Run tests."
+	@echo "  $(T)test ...     		$(R)  Run tests (local with $(RUNNER))."
 	@echo "  $(T)update       		$(R)  Update dependencies."
 	@echo ""
 	@echo "$(G)Versioning:$(R)"
 	@echo "  $(T)major/minor/patch	$(R)  Bump version."

-init-env:
+init-dotenv:
 	@echo "$(T)🔑 Initializing .env file...$(R)"
 	@if [ -f .env ]; then \
 		echo "$(R)⚠️  .env already exists. Skipping.$(R)"; \
@@ -175,7 +198,7 @@ patch: _check_branch

 prune: check-docker
 	@echo "$(T)🗑️  Pruning Docker resources...$(R)"
-	docker system prune -af --volumes
+	docker system prune -af
 	@echo "✅ Docker cleaned."

 ps: check-docker
@@ -196,7 +219,7 @@ shell: check-docker

 test: check-runner
 	@echo "$(T)🧪 Running tests...$(R)"
-	$(RUNNER_RUN) pytest -n auto --dist=loadscope $(ARGS)
+	$(RUNNER_RUN) pytest $(ARGS)

 up: check-docker
 	@echo "$(T)🚀 Starting Agent Media...$(R)"
@@ -213,3 +236,11 @@ _check_branch:
 	if [ "$$curr" != "main" ]; then \
 	   echo "❌ Error: not on the main branch"; exit 1; \
 	fi
+
+_ci-image-name:
+	@echo "IMAGE_NAME=$(IMAGE_NAME)"
+
+_ci-run-tests: build-test
+	@echo "$(T)🧪 Running tests in Docker...$(R)"
+	docker run --rm $(IMAGE_NAME):test pytest
+	@echo "✅ Tests passed."
--- a/brain/Dockerfile
+++ b/brain/Dockerfile
@@ -1,12 +1,16 @@
 # Dockerfile for Agent Media
 # Multi-stage build for smaller image size
-ARG PYTHON_VERSION=3.14.2
+ARG PYTHON_VERSION
+ARG PYTHON_VERSION_SHORT
 ARG RUNNER
 # ===========================================
 # Stage 1: Builder
 # ===========================================
 FROM python:${PYTHON_VERSION}-slim-bookworm as builder

+# Re-declare ARGs after FROM to make them available in this stage
+ARG RUNNER
+
 # STFU - No need - Write logs asap
 ENV DEBIAN_FRONTEND=noninteractive \
    PYTHONDONTWRITEBYTECODE=1 \
@@ -25,7 +29,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 WORKDIR /tmp

 # Copy dependency files
-COPY pyproject.toml poetry.lock* uv.lock* Makefile ./
+COPY brain/pyproject.toml brain/poetry.lock* brain/uv.lock* Makefile ./

 # Install dependencies as root (to avoid permission issues with system packages)
 RUN --mount=type=cache,target=/root/.cache/pip \
@@ -39,10 +43,35 @@ RUN --mount=type=cache,target=/root/.cache/pip \
    fi

 # ===========================================
-# Stage 2: Runtime
+# Stage 2: Testing
+# ===========================================
+FROM builder as test
+
+ARG RUNNER
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=cache,target=/root/.cache/pypoetry \
+    --mount=type=cache,target=/root/.cache/uv \
+    if [ "$RUNNER" = "poetry" ]; then \
+        poetry install --no-root; \
+    elif [ "$RUNNER" = "uv" ]; then \
+        uv pip install --system -e .[dev]; \
+    fi
+
+COPY brain/agent/ ./agent/
+COPY brain/application/ ./application/
+COPY brain/domain/ ./domain/
+COPY brain/infrastructure/ ./infrastructure/
+COPY brain/tests/ ./tests/
+COPY brain/app.py .
+
+# ===========================================
+# Stage 3: Runtime
 # ===========================================
 FROM python:${PYTHON_VERSION}-slim-bookworm as runtime

+ARG PYTHON_VERSION_SHORT
+
 ENV LLM_PROVIDER=deepseek \
    MEMORY_STORAGE_DIR=/data/memory \
    PYTHONDONTWRITEBYTECODE=1 \
@@ -69,15 +98,15 @@ USER appuser
 WORKDIR /home/appuser/app

 # Copy Python packages from builder stage
-COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=builder /usr/local/lib/python${PYTHON_VERSION_SHORT}/site-packages /usr/local/lib/python${PYTHON_VERSION_SHORT}/site-packages
 COPY --from=builder /usr/local/bin /usr/local/bin

 # Copy application code (already owned by appuser)
-COPY --chown=appuser:appuser agent/ ./agent/
-COPY --chown=appuser:appuser application/ ./application/
-COPY --chown=appuser:appuser domain/ ./domain/
-COPY --chown=appuser:appuser infrastructure/ ./infrastructure/
-COPY --chown=appuser:appuser app.py .
+COPY --chown=appuser:appuser brain/agent/ ./agent/
+COPY --chown=appuser:appuser brain/application/ ./application/
+COPY --chown=appuser:appuser brain/domain/ ./domain/
+COPY --chown=appuser:appuser brain/infrastructure/ ./infrastructure/
+COPY --chown=appuser:appuser brain/app.py .

 # Create volumes for persistent data
 VOLUME ["/data/memory", "/data/logs"]
--- a/brain/poetry.lock
+++ b/brain/poetry.lock
@@ -1220,4 +1220,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "6204ac4c938e73f59f5bffef08e8e3cdbbc5c307a693578b1183d6140d4e8f31"
+content-hash = "b6cec0647accef2c235cededb06cab49f30033fb5e5ce1f6b589a4da5d2a2d8d"
--- a/brain/pyproject.toml
+++ b/brain/pyproject.toml
@@ -47,6 +47,7 @@ addopts = [
    #"--cov-report=xml",            # Génère un rapport XML (pour CI/CD)
    #"--cov-fail-under=80",         # Échoue si coverage < 80%
    "-n=auto",                     # --numprocesses=auto : parallélise les tests (pytest-xdist)
+    "--dist=loadscope",            # Distribution strategy: group tests by module
    "--strict-markers",            # Erreur si un marker non déclaré est utilisé
    "--disable-warnings",          # Désactive l'affichage des warnings (sauf erreurs)
 ]