feature/ASSIST-1596-playwright-e2e-test

.github/workflows/build-test-suite.yml

@@ -0,0 +1,20 @@
+name: Build image for test-suite
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build:
+    name: Build Image
+    uses: uktrade/platform-github/.github/workflows/platform-build-docker-ecr-image.yml@latest
+    secrets:
+      AWS_OIDC_ROLE_ARN: ${{ secrets.OIDC_IAM_ROLE }}
+    with:
+      ECR_REPOSITORY: "redbox/test-suite"
+      CPU_ARCHITECTURE: "linux/arm64" # Can also be 'linux/x86_64' or 'linux/amd64'
+      GITHUB_RUNNER: "ubuntu-24.04-arm" # Need to support the CPU_ARCHITECTURE above. For 'x86' or 'amd64' use 'ubuntu-24.04'
+      CONTEXT: "./django_app/tests/e2e" # The folder which the Dockerfile lives in.

.gitignore

@@ -238,6 +238,5 @@ infrastructure/aws/cleanup_lambda/layer/*
 /django_app/.local.env
 
 # Playwright
-/test-results
-/playwright
-/django_app/test-results
+test-results/
+playwright/

Makefile

@@ -43,7 +43,7 @@ test-redbox: ## Test redbox
 
 .PHONY: test-django
 test-django: ## Test django-app
-	cd django_app && poetry install && poetry run pytest --ignore=tests/playwright --cov=redbox_app -v --cov-report=term-missing --cov-report=xml --cov-fail-under=80 --ds redbox_app.settings --envfile ../tests/.env.test $(TEST)
+	cd django_app && poetry install && poetry run pytest --ignore=tests/playwright --ignore=tests/e2e --cov=redbox_app -v --cov-report=term-missing --cov-report=xml --cov-fail-under=80 --ds redbox_app.settings --envfile ../tests/.env.test $(TEST)
 
 .PHONY: test-django-single
 test-django-single: ## Test django-app with specified test file/case
@@ -54,6 +54,19 @@ build-django-static: ## Build django-app static files
 	cd django_app/frontend/ && npm install && npm run build
 	cd django_app/ && poetry run python manage.py collectstatic --noinput
 
+
+.PHONY: test-e2e
+test-e2e:
+	# Does this need to be separate?
+	docker compose down opensearch db sso minio
+	docker compose up -d --wait opensearch db sso minio
+	docker compose up -d --wait redbox-django-app
+	cd django_app && \
+	poetry install --only e2e && \
+	poetry run playwright install --with-deps chromium && \
+	docker exec -it $$(docker ps -q --filter "name=django_app") venv/bin/django-admin loaddata tests/fixtures/chatllmbackend.json && \
+	BASE_URL=http://localhost:8080 DJANGO_ALLOW_ASYNC_UNSAFE=1 poetry run pytest tests/e2e/test_e2e.py --confcutdir=tests/e2e --tracing retain-on-failure --video on --screenshot on -k test_user_journey
+
 .PHONY: test-integration
 test-integration:
 	docker compose down opensearch db sso minio

django_app/tests/e2e/.dockerignore

@@ -0,0 +1,3 @@
+node_modules/
+__pycache__
+.env

django_app/tests/e2e/Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.12-slim AS build
+
+ENV POETRY_VERSION=2.2.1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1 \
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+RUN pip install "poetry==$POETRY_VERSION"
+
+WORKDIR $PYSETUP_PATH
+
+COPY poetry.lock pyproject.toml ./
+
+RUN --mount=type=cache,target=/root/.cache/pypoetry \
+    poetry install --no-root
+
+COPY . .
+
+FROM python:3.12-slim AS runtime
+
+ENV PLAYWRIGHT_BROWSERS_PATH="/opt/playwright-browsers" \
+    VENV_PATH="/opt/pysetup/.venv" \
+    PATH="/opt/pysetup/.venv/bin:$PATH"
+
+RUN groupadd -g 1001 appgroup && \
+    useradd -u 1001 -g appgroup -m -d /home/appuser -s /bin/bash appuser
+
+WORKDIR /app
+
+COPY --from=build --chown=appuser:appgroup /opt/pysetup/.venv /opt/pysetup/.venv
+COPY --from=build --chown=appuser:appgroup /opt/pysetup/ ./
+
+RUN playwright install --with-deps chromium && \
+    chmod -R o+rx $PLAYWRIGHT_BROWSERS_PATH
+
+USER appuser
+
+ENTRYPOINT ["python", "-m","pytest","/app"]

django_app/tests/e2e/pages.py

@@ -0,0 +1,234 @@
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Collection, Sequence
+from dataclasses import dataclass, field
+from time import sleep
+
+from playwright.sync_api import Locator, Page, expect
+from yarl import URL
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class PageError(ValueError):
+    pass
+
+
+class BasePage(ABC):
+    def __init__(self, page: Page):
+        self.page = page
+        self.check_title()
+
+    def check_title(self):
+        expected_page_title = self.expected_page_title
+        expect(self.page).to_have_title(expected_page_title)
+
+    def navigate_to_privacy_page(self) -> "PrivacyPage":
+        self.page.get_by_role("link", name="Privacy", exact=True).click()
+        return PrivacyPage(self.page)
+
+    def navigate_to_accessibility_page(self) -> "AccessibilityPage":
+        self.page.get_by_role("link", name="Accessibility", exact=True).click()
+        return AccessibilityPage(self.page)
+
+    def navigate_to_support_page(self) -> "SupportPage":
+        self.page.get_by_role("link", name="Support", exact=True).click()
+        return SupportPage(self.page)
+
+    @property
+    @abstractmethod
+    def expected_page_title(self) -> str: ...
+
+    @property
+    def title(self) -> str:
+        return self.page.title()
+
+    @property
+    def url(self) -> URL:
+        return URL(self.page.url)
+
+    def __str__(self) -> str:
+        return f'"{self.title}" at {self.url}'
+
+
+class SSOLoginPage:
+    def __init__(self, page: Page, base_url: URL):
+        self.page = page
+        page.goto(str(base_url))
+
+    def login(self, username, password):
+        self.page.get_by_label("Email:").fill(username)
+
+        self.page.get_by_label("Password:").fill(password)
+
+        self.page.get_by_role("button").click()
+
+
+class SignedInBasePage(BasePage, ABC):
+    def navigate_to_chats(self) -> "ChatsPage":
+        self.page.get_by_role("link", name="Chats", exact=True).click()
+
+
+class LandingPage(BasePage):
+    def __init__(self, page, base_url: URL):
+        page.goto(str(base_url))
+        super().__init__(page)
+
+    @property
+    def expected_page_title(self) -> str:
+        return "Assist at DBT"
+
+    def sign_in(self) -> "ChatsPage":
+        self.page.get_by_role("link", name="Sign in", exact=True).click()
+        return ChatsPage(self.page)
+
+
+@dataclass
+class ChatMessage:
+    status: str | None
+    text: str
+    sources: Sequence[str]
+    element: Locator = field(repr=False)
+    chats_page: "ChatsPage" = field(repr=False)
+
+    @classmethod
+    def from_element(cls, element: Locator, page: "ChatsPage") -> "ChatMessage":
+        status = element.get_attribute("data-status")
+        text = element.locator(".ids-chat-message__text").inner_text()
+        sources = element.locator("sources-list").get_by_role("listitem").all_inner_texts()
+        return cls(status=status, text=text, sources=sources, element=element, chats_page=page)
+
+
+class ChatsPage(SignedInBasePage):
+    @property
+    def expected_page_title(self) -> str:
+        return "New chat - Chats - Assist at DBT"
+
+    @property
+    def selected_llm(self) -> str:
+        return self.page.locator("#llm-selector").inner_text()
+
+    @property
+    def write_message(self) -> str:
+        return self.page.locator("#message").input_value()
+
+    @write_message.setter
+    def write_message(self, value: str):
+        self.page.locator("#message").fill(value)
+
+    @property
+    def available_file_names(self) -> Sequence[str]:
+        return self.page.locator("document-selector .govuk-checkboxes__label").all_inner_texts()
+
+    @property
+    def selected_file_names(self) -> Collection[str]:
+        return {file_name for file_name in self.available_file_names if self.page.get_by_label(file_name).is_checked()}
+
+    @selected_file_names.setter
+    def selected_file_names(self, file_names_to_select: Collection[str]):
+        for file_name in self.available_file_names:
+            checkbox = self.page.get_by_label(file_name)
+            if file_name in file_names_to_select:
+                checkbox.check()
+            else:
+                checkbox.uncheck()
+
+    def feedback_stars(self, rating: int):
+        self.page.locator(".feedback-container").get_by_role("button").nth(rating - 1).click()
+
+    feedback_stars = property(fset=feedback_stars)
+
+    @property
+    def feedback_text(self) -> str:
+        return self.page.locator(".feedback-container").get_by_role("textbox").inner_text()
+
+    @feedback_text.setter
+    def feedback_text(self, text: str):
+        self.page.locator(".feedback-container").get_by_role("textbox").fill(text)
+
+    def feedback_chips(self, chips: Collection[str]):
+        for chip in chips:
+            self.page.locator(".feedback-container").get_by_test_id(chip).check()
+
+    feedback_chips = property(fset=feedback_chips)
+
+    @property
+    def chat_title(self) -> str:
+        return self.page.locator(".ids-chat-title__heading").inner_text()
+
+    @chat_title.setter
+    def chat_title(self, title: str):
+        self.page.locator(".chat-title-edit-button").click()
+        input_ = self.page.get_by_label("Chat Title")
+        input_.fill(title)
+        input_.press("Enter")
+
+    def start_new_chat(self) -> "ChatsPage":
+        self.page.get_by_role("button", name="New chat").click()
+        return ChatsPage(self.page)
+
+    def delete_first_chat(self) -> "ChatsPage":
+        self.page.locator(".rb-chat-history__actions-button").first.click()
+        self.page.locator('[data-action="delete"]').first.click()
+        self.page.locator('[data-action="delete-confirm"]').first.click()
+        return ChatsPage(self.page)
+
+    def count_chats(self) -> "ChatsPage":
+        chat_links = self.page.locator(".chat-item-link").all()
+        return len(chat_links)
+
+    def send(self) -> "ChatsPage":
+        self.page.keyboard.press("Enter")
+        return ChatsPage(self.page)
+
+    def improve(self):
+        self.page.get_by_role("button", name="Help improve the response").click()
+
+    def submit_feedback(self):
+        self.page.locator(".feedback-container").get_by_role("button", name="Submit").click()
+
+    @property
+    def all_messages(self) -> Sequence[ChatMessage]:
+        return [ChatMessage.from_element(element, self) for element in self.page.locator("chat-message").all()]
+
+    def get_all_messages_once_streaming_has_completed(
+        self, retry_interval: int = 1, max_tries: int = 120
+    ) -> Sequence[ChatMessage]:
+        tries = 0
+        while True:
+            messages = self.all_messages
+            if not any(m.status == "streaming" for m in messages):
+                logger.info("messages: %s", messages)
+                return messages
+            if tries >= max_tries:
+                logger.error("messages: %s", messages)
+                error_message = "Too many retries waiting for response"
+                raise PageError(error_message)
+            tries += 1
+            sleep(retry_interval)
+
+    def wait_for_latest_message(self, role="Redbox") -> ChatMessage:
+        return [m for m in self.get_all_messages_once_streaming_has_completed() if m.role == role][-1]
+
+    def navigate_to_titled_chat(self, title: str) -> "ChatsPage":
+        self.page.get_by_role("link", name=title).click()
+        return ChatsPage(self.page)
+
+
+class PrivacyPage(BasePage):
+    @property
+    def expected_page_title(self) -> str:
+        return "Privacy notice - Assist at DBT"
+
+
+class AccessibilityPage(BasePage):
+    @property
+    def expected_page_title(self) -> str:
+        return "Accessibility statement - Assist at DBT"
+
+
+class SupportPage(BasePage):
+    @property
+    def expected_page_title(self) -> str:
+        return "Support - Assist at DBT"