Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .github/workflows/ci-tests-e2e-forks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,25 @@ jobs:
fi
echo "PR author: $PR_AUTHOR"

- name: Download new-test video comment
if: github.event.action == 'completed'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
run_id: ${{ github.event.workflow_run.id }}
name: playwright-video-comment
path: video-comment
if_no_artifact_found: warn

- name: Upsert new-test video comment
if: github.event.action == 'completed' && hashFiles('video-comment/playwright-video-comment.md') != ''
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr.outputs.number }}
section-name: new-test-videos
section-content-file: video-comment/playwright-video-comment.md
comment-marker: '<!-- playwright-video-new-tests -->'
token: ${{ github.token }}

- name: Handle Test Start — upsert playwright starting section
if: steps.route.outputs.handle == 'true' && github.event.action == 'requested'
uses: ./.github/actions/upsert-comment-section
Expand Down
135 changes: 83 additions & 52 deletions .github/workflows/ci-tests-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ jobs:
with:
source_sha: ${{ github.event.pull_request.head.sha || github.sha }}

# Records video only for spec files newly added in this PR, to keep cost bounded.
# Records video only for tests newly added in this PR, to keep cost bounded.
playwright-video-new-tests:
needs: setup
if: ${{ github.event_name == 'pull_request' }}
Expand All @@ -283,39 +283,57 @@ jobs:
with:
fetch-depth: 0

- name: Detect newly-added test spec files
- name: Detect newly-added tests
id: detect
shell: bash
env:
BASE_REF: ${{ github.base_ref }}
IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# HEAD is refs/pull/N/merge, so its first parent is the base tip the PR was merged
# onto. pull_request.base.sha is recorded when the PR opens and does not follow the
# base branch, so diffing against it attributes every spec the base gained since
# then to this PR.
if [ "$(git rev-parse --verify --quiet HEAD^2)" = "${{ github.event.pull_request.head.sha }}" ]; then
DIFF_RANGE="$(git rev-parse HEAD^1)..HEAD"
else
DIFF_RANGE="${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}"
if [ "$IS_FORK" = "true" ]; then
echo '::notice::Skipping video recording for a fork pull request.'
echo 'has-new-tests=false' >> "$GITHUB_OUTPUT"
exit 0
fi
# git diff and grep are separate statements (not piped) so a git diff failure
# (e.g. the fallback sha isn't in the local object store) surfaces as a build
# error instead of silently reading as "no new specs" and no-oping the video job.
ADDED_FILES=$(git diff --name-only --diff-filter=A "$DIFF_RANGE" -- browser_tests/tests)
NEW_FILES=$(printf '%s\n' "$ADDED_FILES" | grep '\.spec\.ts$' || true)

if [ -z "$NEW_FILES" ]; then
echo "has-new-tests=false" >> "$GITHUB_OUTPUT"

git fetch --no-tags origin "$BASE_REF"
DIFF_RANGE="origin/${BASE_REF}...HEAD"
NEW_FILES=$(git diff --name-only --diff-filter=AMR "$DIFF_RANGE" -- 'browser_tests/**/*.spec.ts')
: > /tmp/fallback-specs.txt

while IFS= read -r file; do
[ -z "$file" ] && continue
# A leaf title is not a stable Playwright identity: duplicate or
# short titles can select unrelated tests through --grep. Run every
# changed spec as a whole so assertion-only improvements are also
# recorded.
printf '%s\n' "$file" >> /tmp/fallback-specs.txt
done <<< "$NEW_FILES"

sort -u -o /tmp/fallback-specs.txt /tmp/fallback-specs.txt
if [ ! -s /tmp/fallback-specs.txt ]; then
echo '::notice::No changed Playwright specs found.'
echo 'has-new-tests=false' >> "$GITHUB_OUTPUT"
else
echo "has-new-tests=true" >> "$GITHUB_OUTPUT"
echo 'has-new-tests=true' >> "$GITHUB_OUTPUT"
fi

{
echo "files<<EOF_NEW_SPEC_FILES"
echo 'files<<EOF_NEW_SPEC_FILES'
echo "$NEW_FILES"
echo "EOF_NEW_SPEC_FILES"
echo 'EOF_NEW_SPEC_FILES'
echo 'titles<<EOF_NEW_TEST_TITLES'
echo 'EOF_NEW_TEST_TITLES'
echo 'fallback-files<<EOF_FALLBACK_SPEC_FILES'
cat /tmp/fallback-specs.txt
echo 'EOF_FALLBACK_SPEC_FILES'
} >> "$GITHUB_OUTPUT"

echo "Newly-added spec files:"
echo 'Changed spec files:'
echo "${NEW_FILES:-<none>}"
echo 'Whole-file fallbacks:'
cat /tmp/fallback-specs.txt

- name: Download built frontend
if: steps.detect.outputs.has-new-tests == 'true'
Expand All @@ -337,42 +355,25 @@ jobs:
- name: Find safely recordable new tests
if: steps.detect.outputs.has-new-tests == 'true'
id: recordable
shell: bash
env:
NEW_SPEC_FILES: ${{ steps.detect.outputs.files }}
FALLBACK_SPEC_FILES: ${{ steps.detect.outputs.fallback-files }}
run: |
FILES=$(echo "$NEW_SPEC_FILES" | tr '\n' ' ')
mapfile -t FILES < <(printf '%s\n' "$FALLBACK_SPEC_FILES" | sed '/^$/d' | sort -u)
AUDIT_EXIT=0
AUDIT_OUTPUT=$(pnpm exec playwright test --project=audit --list --pass-with-no-tests $FILES 2>&1) || AUDIT_EXIT=$?
AUDIT_OUTPUT=$(pnpm exec playwright test --project=audit --list --pass-with-no-tests "${FILES[@]}" 2>&1) || AUDIT_EXIT=$?
echo "$AUDIT_OUTPUT"

if [ "$AUDIT_EXIT" -ne 0 ]; then
exit "$AUDIT_EXIT"
fi

if ! echo "$AUDIT_OUTPUT" | grep -Fxq 'Total: 0 tests in 0 files'; then
{
echo '## New-test video walkthrough'
echo
echo '### Cannot skip safely: audit tests have no regular CI coverage'
echo
echo 'At least one newly added test is routed to the `audit` project. Unlike the other projects excluded by Chromium, no regular CI job executes audit tests.'
echo 'This video job only records the `chromium` project, so it fails rather than allowing the PR to pass without executing the new audit test.'
echo
echo 'Detected files:'
echo '```text'
echo "$NEW_SPEC_FILES"
echo '```'
echo
echo 'Audit discovery output:'
echo '```text'
echo "$AUDIT_OUTPUT"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
exit 1
echo '::notice::Some new tests are audit-routed; continuing Chromium discovery.'
fi

LIST_EXIT=0
LIST_OUTPUT=$(pnpm exec playwright test --project=chromium --list --pass-with-no-tests $FILES 2>&1) || LIST_EXIT=$?
LIST_OUTPUT=$(pnpm exec playwright test --project=chromium --list --pass-with-no-tests "${FILES[@]}" 2>&1) || LIST_EXIT=$?
echo "$LIST_OUTPUT"

if [ "$LIST_EXIT" -ne 0 ]; then
Expand All @@ -390,13 +391,13 @@ jobs:
echo
echo '### Skipped safely: no Chromium-eligible tests'
echo
echo 'The workflow detected newly added Playwright spec files, but none of their tests are eligible for the `chromium` project used to record walkthroughs.'
echo 'This commonly happens when every test uses a project-routing tag excluded by Chromium: `@perf`, `@cloud`, or `@mobile`.'
echo 'The workflow detected newly added Playwright spec files, but none of their tests are eligible for the Chromium project used to record walkthroughs.'
echo 'This commonly happens when every test uses a project-routing tag excluded by Chromium: @perf, @cloud, or @mobile.'
echo 'The regular project-specific E2E jobs still run these tests. This video job succeeds without creating an empty report or video artifact.'
echo
echo 'Detected files:'
echo '```text'
echo "$NEW_SPEC_FILES"
echo "$FALLBACK_SPEC_FILES"
echo '```'
echo
echo 'Playwright discovery output:'
Expand All @@ -407,23 +408,53 @@ jobs:

- name: Run new test(s) with video recording
if: steps.recordable.outputs.has-tests == 'true'
shell: bash
env:
RECORD_VIDEO: 'true'
SLOW_MO: '1000'
NEW_SPEC_FILES: ${{ steps.detect.outputs.files }}
FALLBACK_SPEC_FILES: ${{ steps.detect.outputs.fallback-files }}
run: |
FILES=$(echo "$NEW_SPEC_FILES" | tr '\n' ' ')
pnpm exec playwright test --project=chromium --timeout=60000 $FILES
mapfile -t FILES < <(printf '%s\n' "$FALLBACK_SPEC_FILES" | sed '/^$/d' | sort -u)
pnpm exec playwright test --project=chromium --timeout=60000 "${FILES[@]}"

- name: Upload new-test report (with embedded video)
if: ${{ !cancelled() && steps.recordable.outputs.has-tests == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: playwright-report-new-tests
path: ./playwright-report/
path: |
./playwright-report/
./test-results/**/*.webm
retention-days: 30
if-no-files-found: warn

- name: Write video comment
if: ${{ !cancelled() && steps.recordable.outputs.has-tests == 'true' }}
env:
NEW_TEST_TITLES: ${{ steps.detect.outputs.titles }}
FALLBACK_SPEC_FILES: ${{ steps.detect.outputs.fallback-files }}
run: |
{
echo '## Playwright videos for new tests'
echo
echo "[Download the recorded videos and HTML report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"
echo
echo 'Recorded test titles:'
if [ -n "$NEW_TEST_TITLES" ]; then
printf '%s\n' "$NEW_TEST_TITLES" | sed 's/^/- `/' | sed 's/$/`/'
else
printf '%s\n' "$FALLBACK_SPEC_FILES" | sed 's/^/- all tests in `/' | sed 's/$/`/'
fi
} > /tmp/playwright-video-comment.md

- name: Upload video comment
if: ${{ !cancelled() && steps.recordable.outputs.has-tests == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: playwright-video-comment
path: /tmp/playwright-video-comment.md
retention-days: 1

#### BEGIN Deployment and commenting (PRs with direct secret access)
# when using pull_request event, we have permission to comment directly
# otherwise, we use workflow_run in ci-tests-e2e-forks.yaml
Expand Down
14 changes: 14 additions & 0 deletions browser_tests/tests/agent/videoRecordingDemo.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import {
comfyPageFixture as test,
comfyExpect as expect
} from '@e2e/fixtures/ComfyPage'

test('opens the settings menu for the video recording demo', async ({
comfyPage
}) => {
await comfyPage.settingDialog.open()
await expect(comfyPage.settingDialog.root).toBeVisible()

await comfyPage.settingDialog.close()
await expect(comfyPage.settingDialog.root).toBeHidden()
})
Loading