diff --git a/.github/workflows/server-test.yaml b/.github/workflows/server-test.yaml index f65ae214..f1141d8f 100644 --- a/.github/workflows/server-test.yaml +++ b/.github/workflows/server-test.yaml @@ -7,6 +7,10 @@ on: branches: ["main"] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build-headful: uses: ./.github/workflows/chromium-headful-image.yaml @@ -16,7 +20,27 @@ jobs: uses: ./.github/workflows/chromium-headless-image.yaml secrets: inherit - test: + test-server-unit: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: "server/go.mod" + cache: true + cache-dependency-path: server/go.sum + + - name: Run server unit tests + run: make test-unit + working-directory: server + + test-server-e2e: runs-on: ubuntu-latest needs: [build-headful, build-headless] permissions: @@ -39,11 +63,21 @@ jobs: with: version: 10 + - name: Add pnpm to PATH + run: | + echo "$PNPM_HOME" >> "$GITHUB_PATH" + export PATH="$PNPM_HOME:$PATH" + pnpm --version + + - name: Install Playwright e2e dependencies + run: pnpm install --frozen-lockfile + working-directory: server/e2e/playwright + - name: Set up Go uses: actions/setup-go@v5 with: go-version-file: "server/go.mod" - cache: true + cache: false - name: Compute short SHA for images id: vars @@ -56,8 +90,8 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Run server Makefile tests - run: make test + - name: Run server e2e tests + run: make test-e2e working-directory: server env: E2E_CHROMIUM_HEADFUL_IMAGE: onkernel/chromium-headful:${{ steps.vars.outputs.short_sha }} diff --git a/server/e2e/backend_docker.go b/server/e2e/backend_docker.go index fc944e04..b14543d5 100644 --- a/server/e2e/backend_docker.go +++ b/server/e2e/backend_docker.go @@ -118,7 +118,7 @@ func (c *dockerBackend) Stop(ctx context.Context) error { if c.ctr == nil { return nil } - return testcontainers.TerminateContainer(c.ctr) + return testcontainers.TerminateContainer(c.ctr, testcontainers.StopTimeout(0)) } // APIBaseURL returns the URL for the container's API server. diff --git a/server/e2e/container.go b/server/e2e/container.go index 11077263..82c17833 100644 --- a/server/e2e/container.go +++ b/server/e2e/container.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" instanceoapi "github.com/kernel/kernel-images/server/lib/oapi" ) @@ -44,15 +45,23 @@ func NewTestContainer(tb testing.TB, image string) *TestContainer { // from the runner's loopback that a remote instance cannot reach. This keeps the // hypeman CI job green while preserving coverage on the Docker backend. func (c *TestContainer) Start(ctx context.Context, cfg ContainerConfig) error { + c.tb.Helper() + start := time.Now() if cfg.HostAccess && !c.backend.SupportsHostAccess() { c.tb.Skipf("skipping host-access test: %s backend has no host-loopback bridge for the instance", backendKindFromEnv()) } - return c.backend.Start(ctx, cfg) + err := c.backend.Start(ctx, cfg) + c.logTiming("start", start, err) + return err } // Stop stops and removes the instance. func (c *TestContainer) Stop(ctx context.Context) error { - return c.backend.Stop(ctx) + c.tb.Helper() + start := time.Now() + err := c.backend.Stop(ctx) + c.logTiming("stop", start, err) + return err } // APIBaseURL returns the URL for the instance's API server. @@ -112,18 +121,30 @@ func (c *TestContainer) APIClientNoKeepAlive() (*instanceoapi.ClientWithResponse // WaitReady waits for the instance's API to become ready. func (c *TestContainer) WaitReady(ctx context.Context) error { - return c.backend.WaitReady(ctx) + c.tb.Helper() + start := time.Now() + err := c.backend.WaitReady(ctx) + c.logTiming("wait_ready", start, err) + return err } // WaitDevTools waits for the CDP WebSocket endpoint to be ready. func (c *TestContainer) WaitDevTools(ctx context.Context) error { - return c.backend.WaitDevTools(ctx) + c.tb.Helper() + start := time.Now() + err := c.backend.WaitDevTools(ctx) + c.logTiming("wait_devtools", start, err) + return err } // WaitChromeDriver waits for the ChromeDriver proxy (and upstream ChromeDriver) // to be ready. func (c *TestContainer) WaitChromeDriver(ctx context.Context) error { - return c.backend.WaitChromeDriver(ctx) + c.tb.Helper() + start := time.Now() + err := c.backend.WaitChromeDriver(ctx) + c.logTiming("wait_chromedriver", start, err) + return err } // Exec executes a command inside the instance and returns the exit code and @@ -136,3 +157,18 @@ func (c *TestContainer) Exec(ctx context.Context, cmd []string) (int, string, er func (c *TestContainer) ExitCh() <-chan error { return c.backend.ExitCh() } + +func (c *TestContainer) logTiming(phase string, start time.Time, err error) { + status := "ok" + if err != nil { + status = "error" + } + c.tb.Logf("[e2e-timing] test=%q phase=%s backend=%s image=%s duration=%s status=%s", + c.tb.Name(), + phase, + backendKindFromEnv(), + c.Image, + time.Since(start).Truncate(time.Millisecond), + status, + ) +} diff --git a/server/e2e/e2e_chromium_configure_powerset_test.go b/server/e2e/e2e_chromium_configure_powerset_test.go index eda65b54..fcba4c6a 100644 --- a/server/e2e/e2e_chromium_configure_powerset_test.go +++ b/server/e2e/e2e_chromium_configure_powerset_test.go @@ -31,6 +31,7 @@ const ( // TestChromiumConfigureMultipartPowerset runs a representative matrix by default. // Set E2E_CHROMIUM_CONFIGURE_POWERSET=1 to run every non-empty combination. func TestChromiumConfigureMultipartPowerset(t *testing.T) { + t.Parallel() if _, err := exec.LookPath("docker"); err != nil { t.Skipf("docker not available: %v", err) @@ -57,6 +58,7 @@ func TestChromiumConfigureMultipartPowerset(t *testing.T) { for _, bits := range matrix { bits := bits t.Run(chromiumConfigurePowersetLabel(bits), func(t *testing.T) { + t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) defer cancel() diff --git a/server/e2e/e2e_chromium_restart_bench_test.go b/server/e2e/e2e_chromium_restart_bench_test.go index 170c189d..78ddd7e2 100644 --- a/server/e2e/e2e_chromium_restart_bench_test.go +++ b/server/e2e/e2e_chromium_restart_bench_test.go @@ -269,6 +269,8 @@ func doChromiumRestart(ctx context.Context, client *instanceoapi.ClientWithRespo // Useful for quick iteration without the full benchmark harness. // Run with: go test -v -run TestChromiumRestartTiming ./e2e/... func TestChromiumRestartTiming(t *testing.T) { + t.Parallel() + if _, err := exec.LookPath("docker"); err != nil { t.Skip("docker not available") } @@ -284,7 +286,10 @@ func TestChromiumRestartTiming(t *testing.T) { const iterations = 3 for _, img := range images { + img := img t.Run(img.name, func(t *testing.T) { + t.Parallel() + c := NewTestContainer(t, img.image) env := map[string]string{ diff --git a/server/e2e/e2e_enterprise_extension_test.go b/server/e2e/e2e_enterprise_extension_test.go index a5a78a19..386efe55 100644 --- a/server/e2e/e2e_enterprise_extension_test.go +++ b/server/e2e/e2e_enterprise_extension_test.go @@ -80,49 +80,24 @@ func runEnterpriseExtensionTest(t *testing.T, image string) { t.Log("[test] uploading kernel-like extension first (to simulate prod)") uploadKernelLikeExtension(t, ctx, c) - // Wait for Chrome to restart with the new flags - time.Sleep(3 * time.Second) - require.NoError(t, c.WaitDevTools(ctx), "devtools not ready after kernel extension") + downloadLogBaseline := extensionDownloadLogSnapshot(t, ctx, c) // Upload the enterprise test extension (with update.xml and .crx) t.Log("[test] uploading enterprise test extension (with update.xml and .crx)") uploadEnterpriseTestExtension(t, ctx, c) - // Wait a bit for Chrome to process the enterprise policy - t.Log("[test] waiting for Chrome to process enterprise policy") - time.Sleep(5 * time.Second) - // Check what files were extracted on the server t.Log("[test] checking extracted extension files on server") checkExtractedFiles(t, ctx, c) - // Check the kernel-images-api logs for extension download requests - t.Log("[test] checking if Chrome fetched the extension") - checkExtensionDownloadLogs(t, ctx, c) - // Verify enterprise policy was configured correctly t.Log("[test] verifying enterprise policy configuration") - verifyEnterprisePolicy(t, ctx, c) + waitForEnterprisePolicy(t, ctx, c, 10*time.Second) - // Wait longer and check again if Chrome has downloaded the extension t.Log("[test] waiting for Chrome to download extension via enterprise policy") - time.Sleep(30 * time.Second) - - // Check logs again - checkExtensionDownloadLogs(t, ctx, c) + waitForExtensionDownload(t, ctx, c, downloadLogBaseline, 30*time.Second) // Check Chrome's extension installation logs - t.Log("[test] checking Chrome stderr for extension-related logs") - checkChromiumLogs(t, ctx, c) - - // Try to trigger extension installation by restarting Chrome - t.Log("[test] restarting Chrome to trigger policy refresh") - restartChrome(t, ctx, c) - - time.Sleep(15 * time.Second) - - // Check logs one more time - checkExtensionDownloadLogs(t, ctx, c) checkChromiumLogs(t, ctx, c) // Check Chrome's policy state @@ -135,7 +110,7 @@ func runEnterpriseExtensionTest(t *testing.T, image string) { // Verify the extension is installed t.Log("[test] checking if extension is installed in Chrome's user-data") - verifyExtensionInstalled(t, ctx, c) + waitForExtensionInstalled(t, ctx, c, 30*time.Second) t.Log("[test] enterprise extension installation test completed") } @@ -240,31 +215,64 @@ func uploadEnterpriseTestExtension(t *testing.T, ctx context.Context, c *TestCon t.Logf("[extension] uploaded elapsed=%s", elapsed.String()) } -// verifyEnterprisePolicy checks that the enterprise policy was configured correctly. -func verifyEnterprisePolicy(t *testing.T, ctx context.Context, c *TestContainer) { +func waitForEnterprisePolicy(t *testing.T, ctx context.Context, c *TestContainer, timeout time.Duration) { t.Helper() + deadline := time.Now().Add(timeout) + var lastContent string + var lastErr error + for { + policyContent, err := enterprisePolicyContent(ctx, c) + lastContent, lastErr = policyContent, err + if err == nil { + lastErr = assertEnterprisePolicy(policyContent) + if lastErr == nil { + t.Logf("[policy] configured content=%s", policyContent) + return + } + } + if time.Now().After(deadline) { + require.NoError(t, lastErr, "enterprise policy did not become ready within %s; last_content=%s", timeout, lastContent) + return + } + select { + case <-ctx.Done(): + require.NoError(t, ctx.Err(), "context cancelled waiting for enterprise policy") + return + case <-time.After(500 * time.Millisecond): + } + } +} +func enterprisePolicyContent(ctx context.Context, c *TestContainer) (string, error) { // Read policy.json policyContent, err := execCombinedOutputWithClient(ctx, c, "cat", []string{"/etc/chromium/policies/managed/policy.json"}) - require.NoError(t, err, "failed to read policy.json") - t.Logf("[policy] content=%s", policyContent) + if err != nil { + return "", fmt.Errorf("failed to read policy.json: %w", err) + } + return policyContent, nil +} +func assertEnterprisePolicy(policyContent string) error { var policy map[string]interface{} - err = json.Unmarshal([]byte(policyContent), &policy) - require.NoError(t, err, "failed to parse policy.json") + if err := json.Unmarshal([]byte(policyContent), &policy); err != nil { + return fmt.Errorf("failed to parse policy.json: %w", err) + } maxConnectionsPerProxy, ok := policy["MaxConnectionsPerProxy"].(float64) - require.True(t, ok, "MaxConnectionsPerProxy not found in policy.json") - require.Equal(t, float64(16), maxConnectionsPerProxy, "unexpected MaxConnectionsPerProxy value") + if !ok { + return fmt.Errorf("MaxConnectionsPerProxy not found in policy.json") + } + if maxConnectionsPerProxy != float64(16) { + return fmt.Errorf("unexpected MaxConnectionsPerProxy value: %v", maxConnectionsPerProxy) + } // Check ExtensionInstallForcelist exists and contains our extension extensionInstallForcelist, ok := policy["ExtensionInstallForcelist"].([]interface{}) - require.True(t, ok, "ExtensionInstallForcelist not found in policy.json") - require.GreaterOrEqual(t, len(extensionInstallForcelist), 1, "ExtensionInstallForcelist should have at least 1 entry") - - // Log all entries - for i, entry := range extensionInstallForcelist { - t.Logf("[policy] forcelist_entry=%d value=%v", i, entry) + if !ok { + return fmt.Errorf("ExtensionInstallForcelist not found in policy.json") + } + if len(extensionInstallForcelist) < 1 { + return fmt.Errorf("ExtensionInstallForcelist should have at least 1 entry") } // Find the enterprise-test entry @@ -272,17 +280,13 @@ func verifyEnterprisePolicy(t *testing.T, ctx context.Context, c *TestContainer) for _, entry := range extensionInstallForcelist { if entryStr, ok := entry.(string); ok && strings.Contains(entryStr, "enterprise-test") { found = true - t.Logf("[policy] found_entry=%s", entryStr) break } } - require.True(t, found, "enterprise-test entry not found in ExtensionInstallForcelist") - - // Check ExtensionSettings - extensionSettings, ok := policy["ExtensionSettings"].(map[string]interface{}) - if ok { - t.Logf("[policy] extension_settings=%+v", extensionSettings) + if !found { + return fmt.Errorf("enterprise-test entry not found in ExtensionInstallForcelist") } + return nil } // checkExtractedFiles checks what files were extracted on the server side. @@ -326,8 +330,7 @@ func checkExtractedFiles(t *testing.T, ctx context.Context, c *TestContainer) { func checkExtensionDownloadLogs(t *testing.T, ctx context.Context, c *TestContainer) { t.Helper() - // Check kernel-images-api log for requests to update.xml and .crx - apiLog, err := execCombinedOutputWithClient(ctx, c, "cat", []string{"/var/log/supervisord/kernel-images-api"}) + apiLog, err := extensionDownloadLog(ctx, c) if err != nil { t.Logf("[logs] error=%v", err) return @@ -355,6 +358,91 @@ func checkExtensionDownloadLogs(t *testing.T, ctx context.Context, c *TestContai } } +func extensionDownloadLogSnapshot(t *testing.T, ctx context.Context, c *TestContainer) string { + t.Helper() + apiLog, err := extensionDownloadLog(ctx, c) + if err != nil { + t.Logf("[logs] baseline_error=%v", err) + return "" + } + return apiLog +} + +func waitForExtensionDownload(t *testing.T, ctx context.Context, c *TestContainer, baseline string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastLog string + var lastErr error + for { + apiLog, err := extensionDownloadLog(ctx, c) + lastLog, lastErr = extensionDownloadLogSince(apiLog, baseline), err + if err == nil && extensionDownloadObserved(lastLog) { + t.Log("[logs] Chrome made GET requests to fetch the enterprise extension") + checkExtensionDownloadLogs(t, ctx, c) + return + } + if time.Now().After(deadline) { + require.NoError(t, lastErr, "extension download was not observed within %s; last_log=%s", timeout, lastLog) + require.True(t, extensionDownloadObserved(lastLog), "extension download was not observed within %s", timeout) + return + } + select { + case <-ctx.Done(): + require.NoError(t, ctx.Err(), "context cancelled waiting for extension download") + return + case <-time.After(1 * time.Second): + } + } +} + +func extensionDownloadLog(ctx context.Context, c *TestContainer) (string, error) { + return execCombinedOutputWithClient(ctx, c, "cat", []string{"/var/log/supervisord/kernel-images-api"}) +} + +func extensionDownloadLogSince(apiLog, baseline string) string { + if baseline != "" && strings.HasPrefix(apiLog, baseline) { + return apiLog[len(baseline):] + } + return apiLog +} + +func extensionDownloadObserved(apiLog string) bool { + var sawUpdateXML, sawCRX bool + for _, line := range strings.Split(apiLog, "\n") { + if !strings.Contains(line, "GET") || !strings.Contains(line, "enterprise-test") { + continue + } + if strings.Contains(line, "update.xml") { + sawUpdateXML = true + } + if strings.Contains(line, ".crx") { + sawCRX = true + } + } + return sawUpdateXML && sawCRX +} + +func TestExtensionDownloadObservedRequiresUpdateXMLAndCRX(t *testing.T) { + t.Parallel() + + require.False(t, extensionDownloadObserved("")) + require.False(t, extensionDownloadObserved(`GET http://127.0.0.1/extensions/enterprise-test/update.xml HTTP/1.1`)) + require.False(t, extensionDownloadObserved(`GET http://127.0.0.1/extensions/enterprise-test/extension.crx HTTP/1.1`)) + require.False(t, extensionDownloadObserved(`GET http://127.0.0.1/extensions/kernel/update.xml HTTP/1.1 +GET http://127.0.0.1/extensions/kernel/extension.crx HTTP/1.1`)) + require.True(t, extensionDownloadObserved(`GET http://127.0.0.1/extensions/enterprise-test/update.xml HTTP/1.1 +GET http://127.0.0.1/extensions/enterprise-test/extension.crx HTTP/1.1`)) +} + +func TestExtensionDownloadLogSince(t *testing.T) { + t.Parallel() + + baseline := "line 1\nline 2\n" + require.Equal(t, "line 3\n", extensionDownloadLogSince(baseline+"line 3\n", baseline)) + require.Equal(t, "line 3\n", extensionDownloadLogSince("line 3\n", baseline)) + require.Equal(t, "line 3\n", extensionDownloadLogSince("line 3\n", "")) +} + // checkChromePolicies checks how Chrome sees the policies. func checkChromePolicies(t *testing.T, ctx context.Context, c *TestContainer) { t.Helper() @@ -422,18 +510,6 @@ func checkChromiumLogs(t *testing.T, ctx context.Context, c *TestContainer) { } } -// restartChrome restarts Chrome via supervisorctl. -func restartChrome(t *testing.T, ctx context.Context, c *TestContainer) { - t.Helper() - - output, err := execCombinedOutputWithClient(ctx, c, "supervisorctl", []string{"-c", "/etc/supervisor/supervisord.conf", "restart", "chromium"}) - if err != nil { - t.Logf("[restart] error=%v output=%s", err, output) - } else { - t.Logf("[restart] result=%s", output) - } -} - // takeChromePolicyScreenshot takes a screenshot of chrome://policy to debug what Chrome sees func takeChromePolicyScreenshot(t *testing.T, ctx context.Context, c *TestContainer) { t.Helper() @@ -496,28 +572,42 @@ const { chromium } = require('playwright-core'); } } -// verifyExtensionInstalled checks if the extension was installed by Chrome. -func verifyExtensionInstalled(t *testing.T, ctx context.Context, c *TestContainer) { +func waitForExtensionInstalled(t *testing.T, ctx context.Context, c *TestContainer, timeout time.Duration) { t.Helper() + expectedExtensionName := "Minimal Enterprise Test Extension" + deadline := time.Now().Add(timeout) + var lastOut []byte + var lastErr error + for { + remaining := time.Until(deadline) + if remaining <= 0 { + t.Logf("[verify] last_output=%s", string(lastOut)) + require.NoError(t, lastErr, "extension %q was not installed within %s", expectedExtensionName, timeout) + return + } - // Check the extension directory - extDir, err := execCombinedOutputWithClient(ctx, c, "ls", []string{"-la", "/home/kernel/extensions/"}) - if err != nil { - t.Logf("[verify] error=%v", err) - } else { - t.Logf("[verify] extensions_dir=%s", extDir) - } - - // Check if Chrome installed the extension using Playwright to inspect chrome://extensions - // Note: When loaded via --load-extension, Chrome generates a NEW extension ID based on the - // directory path, which differs from the ID in update.xml (which is for the packed .crx file). - // So we verify by extension name instead. + attemptTimeout := 15 * time.Second + if remaining < attemptTimeout { + attemptTimeout = remaining + } + attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + lastOut, lastErr = chromeExtensionsCheckOutput(attemptCtx, c, expectedExtensionName) + cancel() + if lastErr == nil { + t.Logf("[verify] extension installed output=%s", string(lastOut)) + return + } - expectedExtensionName := "Minimal Enterprise Test Extension" - t.Logf("[verify] expected_extension_name=%s", expectedExtensionName) + select { + case <-ctx.Done(): + require.NoError(t, ctx.Err(), "context cancelled waiting for extension installation") + return + case <-time.After(1 * time.Second): + } + } +} - // Use playwright to navigate to chrome://extensions and verify extension is loaded - t.Log("[verify] checking chrome://extensions via playwright") +func chromeExtensionsCheckOutput(ctx context.Context, c *TestContainer, expectedExtensionName string) ([]byte, error) { cmd := exec.CommandContext(ctx, "pnpm", "exec", "tsx", "-e", fmt.Sprintf(` const { chromium } = require('playwright-core'); @@ -527,52 +617,69 @@ const { chromium } = require('playwright-core'); const ctx = contexts[0] || await browser.newContext(); const pages = ctx.pages(); const page = pages[0] || await ctx.newPage(); - + await page.goto('chrome://extensions'); await page.waitForLoadState('networkidle'); - await page.waitForTimeout(2000); - - const extensionInfo = await page.evaluate(() => { + + const expectedName = %q; + + const readExtensions = async () => await page.evaluate(() => { const manager = document.querySelector('extensions-manager'); if (!manager || !manager.shadowRoot) return { error: 'no extensions-manager' }; - + const itemList = manager.shadowRoot.querySelector('extensions-item-list'); if (!itemList || !itemList.shadowRoot) return { error: 'no item-list' }; - + const items = itemList.shadowRoot.querySelectorAll('extensions-item'); const extensions = []; - + for (const item of items) { if (!item.shadowRoot) continue; const nameEl = item.shadowRoot.querySelector('#name'); const name = nameEl?.textContent?.trim() || 'unknown'; extensions.push(name); } - + return { extensions }; }); - - if (extensionInfo.error) { - console.log('ERROR: ' + extensionInfo.error); - process.exit(1); - } - - const expectedName = %q; - if (extensionInfo.extensions.includes(expectedName)) { - console.log('SUCCESS: Extension "' + expectedName + '" found'); + + try { + await page.waitForFunction((expectedName) => { + const manager = document.querySelector('extensions-manager'); + if (!manager || !manager.shadowRoot) return false; + + const itemList = manager.shadowRoot.querySelector('extensions-item-list'); + if (!itemList || !itemList.shadowRoot) return false; + + const items = itemList.shadowRoot.querySelectorAll('extensions-item'); + for (const item of items) { + if (!item.shadowRoot) continue; + const nameEl = item.shadowRoot.querySelector('#name'); + const name = nameEl?.textContent?.trim() || 'unknown'; + if (name === expectedName) return true; + } + return false; + }, expectedName, { timeout: 2000 }); + + const extensionInfo = await readExtensions(); + console.log('SUCCESS: Extension "' + expectedName + '" found. Extensions: ' + extensionInfo.extensions.join(', ')); + await browser.close(); process.exit(0); - } else { - console.log('FAIL: Extension "' + expectedName + '" not found. Extensions: ' + extensionInfo.extensions.join(', ')); + } catch (err) { + const extensionInfo = await readExtensions(); + if (extensionInfo.error) { + console.log('ERROR: ' + extensionInfo.error); + } else { + console.log('FAIL: Extension "' + expectedName + '" not found. Extensions: ' + extensionInfo.extensions.join(', ')); + } + console.log('wait_error=' + err.message); + await browser.close(); process.exit(1); } - - await browser.close(); })(); `, c.CDPURL(), expectedExtensionName)) cmd.Dir = getPlaywrightPath() - out, err := cmd.CombinedOutput() - t.Logf("[playwright] output=%s", string(out)) - require.NoError(t, err, "extension verification failed: expected extension %q to be installed in chrome://extensions", expectedExtensionName) + return cmd.CombinedOutput() } // execCombinedOutputWithClient executes a command in the container via the API. diff --git a/server/e2e/e2e_playwright_test.go b/server/e2e/e2e_playwright_test.go index 27c29def..14ee03de 100644 --- a/server/e2e/e2e_playwright_test.go +++ b/server/e2e/e2e_playwright_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "encoding/json" + "fmt" "net/http" "os/exec" "testing" @@ -101,17 +102,20 @@ func TestPlaywrightDaemonRecovery(t *testing.T) { client, err := c.APIClient() require.NoError(t, err) - // Helper to execute playwright code and verify success - executeAndVerify := func(description string) { - t.Logf("action: %s", description) - + executeUserAgent := func() error { code := `return await page.evaluate(() => navigator.userAgent);` req := instanceoapi.ExecutePlaywrightCodeJSONRequestBody{Code: code} rsp, err := client.ExecutePlaywrightCodeWithResponse(ctx, req) - require.NoError(t, err, "%s: request error: %v", description, err) - require.Equal(t, http.StatusOK, rsp.StatusCode(), "%s: unexpected status: %s body=%s", description, rsp.Status(), string(rsp.Body)) - require.NotNil(t, rsp.JSON200, "%s: expected JSON200 response", description) + if err != nil { + return fmt.Errorf("request error: %w", err) + } + if rsp.StatusCode() != http.StatusOK { + return fmt.Errorf("unexpected status: %s body=%s", rsp.Status(), string(rsp.Body)) + } + if rsp.JSON200 == nil { + return fmt.Errorf("expected JSON200 response") + } if !rsp.JSON200.Success { var errorMsg, stderr string @@ -121,13 +125,47 @@ func TestPlaywrightDaemonRecovery(t *testing.T) { if rsp.JSON200.Stderr != nil { stderr = *rsp.JSON200.Stderr } - t.Fatalf("%s: execution failed. Error: %s, Stderr: %s", description, errorMsg, stderr) + return fmt.Errorf("execution failed. Error: %s, Stderr: %s", errorMsg, stderr) } - require.NotNil(t, rsp.JSON200.Result, "%s: expected result to be non-nil", description) + if rsp.JSON200.Result == nil { + return fmt.Errorf("expected result to be non-nil") + } + return nil + } + + executeAndVerify := func(description string) { + t.Logf("action: %s", description) + require.NoError(t, executeUserAgent(), "%s", description) t.Logf("%s: success", description) } + waitForExecution := func(description string, timeout time.Duration) { + t.Logf("action: %s", description) + deadline := time.Now().Add(timeout) + var lastErr error + for attempt := 1; ; attempt++ { + if err := executeUserAgent(); err != nil { + lastErr = err + } else { + t.Logf("%s: success after %d attempt(s)", description, attempt) + return + } + + if time.Now().After(deadline) { + require.NoError(t, lastErr, "%s did not recover within %s", description, timeout) + return + } + + select { + case <-ctx.Done(): + require.NoError(t, ctx.Err(), "%s context cancelled while waiting for recovery", description) + return + case <-time.After(500 * time.Millisecond): + } + } + } + // Step 1: Execute playwright code to start the daemon and establish CDP connection executeAndVerify("initial execution (starts daemon)") @@ -151,12 +189,12 @@ func TestPlaywrightDaemonRecovery(t *testing.T) { } } - // Step 3: Wait for chromium to be ready again + // Step 3: Wait for chromium and the playwright daemon to be ready again t.Log("waiting for chromium to be ready after restart") - time.Sleep(2 * time.Second) + require.NoError(t, c.WaitDevTools(ctx), "DevTools not ready after chromium restart") // Step 4: Execute playwright code again - daemon should recover - executeAndVerify("execution after chromium restart (daemon should recover)") + waitForExecution("execution after chromium restart (daemon should recover)", 30*time.Second) t.Log("playwright daemon recovery test passed") } diff --git a/server/e2e/e2e_recording_audio_test.go b/server/e2e/e2e_recording_audio_test.go index 74651437..35ea9e79 100644 --- a/server/e2e/e2e_recording_audio_test.go +++ b/server/e2e/e2e_recording_audio_test.go @@ -20,6 +20,8 @@ import ( ) func TestReplayRecordingIncludesAudioTrack(t *testing.T) { + t.Parallel() + if _, err := exec.LookPath("docker"); err != nil { t.Skipf("docker not available: %v", err) } @@ -30,8 +32,8 @@ func TestReplayRecordingIncludesAudioTrack(t *testing.T) { c := NewTestContainer(t, headfulImage) require.NoError(t, c.Start(ctx, ContainerConfig{ Env: map[string]string{ - "WIDTH": "1280", - "HEIGHT": "720", + "WIDTH": "1280", + "HEIGHT": "720", }, }), "failed to start container") defer c.Stop(ctx) @@ -77,8 +79,8 @@ func TestReplayRecordingZombocomArchiveAudio(t *testing.T) { c := NewTestContainer(t, headfulImage) require.NoError(t, c.Start(ctx, ContainerConfig{ Env: map[string]string{ - "WIDTH": "1280", - "HEIGHT": "720", + "WIDTH": "1280", + "HEIGHT": "720", }, }), "failed to start container") defer c.Stop(ctx) @@ -199,9 +201,10 @@ func recordReplayAudio(t *testing.T, ctx context.Context, c *TestContainer, play require.NoError(t, os.WriteFile(outputPath, downloadResp.Body, 0o644), "failed to write downloaded recording") } + recordingPath := writeContainerRecording(t, ctx, c, downloadResp.Body) require.True(t, mp4HasAudioTrack(downloadResp.Body), "downloaded recording does not contain an audio track") - require.Greater(t, mp4AudioPeakLevel(t, downloadResp.Body), minPeakLevel, "downloaded recording audio track is silent") - formatDuration, audioDuration := mp4Durations(t, downloadResp.Body) + require.Greater(t, mp4AudioPeakLevel(t, ctx, c, recordingPath), minPeakLevel, "downloaded recording audio track is silent") + formatDuration, audioDuration := mp4Durations(t, ctx, c, recordingPath) require.GreaterOrEqual(t, audioDuration, formatDuration-2, "downloaded recording audio track ends before the recording does") } @@ -392,6 +395,20 @@ func writeContainerAudioFixture(t *testing.T, ctx context.Context, c *TestContai return "file://" + fixturePath } +func writeContainerRecording(t *testing.T, ctx context.Context, c *TestContainer, data []byte) string { + t.Helper() + + client, err := c.APIClient() + require.NoError(t, err, "failed to create API client") + + const recordingPath = "/tmp/e2e-recording-audio.mp4" + params := &instanceoapi.WriteFileParams{Path: recordingPath} + rsp, err := client.WriteFileWithBodyWithResponse(ctx, params, "video/mp4", bytes.NewReader(data)) + require.NoError(t, err, "write recording for audio analysis") + require.Equal(t, http.StatusCreated, rsp.StatusCode(), "unexpected write status: %s body=%s", rsp.Status(), string(rsp.Body)) + return recordingPath +} + func mp4HasAudioTrack(data []byte) bool { for i := 0; i+16 <= len(data); i++ { if !bytes.Equal(data[i:i+4], []byte("hdlr")) { @@ -415,24 +432,22 @@ func stringValue(v *string) string { return *v } -func mp4AudioPeakLevel(t *testing.T, data []byte) float64 { +func mp4AudioPeakLevel(t *testing.T, ctx context.Context, c *TestContainer, recordingPath string) float64 { t.Helper() - recordingPath := filepath.Join(t.TempDir(), "recording.mp4") - require.NoError(t, os.WriteFile(recordingPath, data, 0o644), "failed to write recording for audio analysis") - - out, err := exec.Command( - "docker", "run", "--rm", - "-v", recordingPath+":/tmp/recording.mp4:ro", - "--entrypoint", "ffmpeg", - headfulImage, - "-hide_banner", - "-i", "/tmp/recording.mp4", - "-map", "0:a:0", - "-af", "astats=metadata=1:reset=0", - "-f", "null", - "-", - ).CombinedOutput() + out, err := execCombinedOutput( + ctx, + c, + "ffmpeg", + []string{ + "-hide_banner", + "-i", recordingPath, + "-map", "0:a:0", + "-af", "astats=metadata=1:reset=0", + "-f", "null", + "-", + }, + ) require.NoError(t, err, "failed to analyze recording audio: %s", string(out)) matches := regexp.MustCompile(`Max level: ([0-9.]+)`).FindStringSubmatch(string(out)) @@ -443,23 +458,21 @@ func mp4AudioPeakLevel(t *testing.T, data []byte) float64 { return peak } -func mp4Durations(t *testing.T, data []byte) (float64, float64) { +func mp4Durations(t *testing.T, ctx context.Context, c *TestContainer, recordingPath string) (float64, float64) { t.Helper() - recordingPath := filepath.Join(t.TempDir(), "recording.mp4") - require.NoError(t, os.WriteFile(recordingPath, data, 0o644), "failed to write recording for duration analysis") - - out, err := exec.Command( - "docker", "run", "--rm", - "-v", recordingPath+":/tmp/recording.mp4:ro", - "--entrypoint", "ffprobe", - headfulImage, - "-v", "error", - "-show_entries", "format=duration", - "-show_entries", "stream=codec_type,duration", - "-of", "json", - "/tmp/recording.mp4", - ).CombinedOutput() + out, err := execCombinedOutput( + ctx, + c, + "ffprobe", + []string{ + "-v", "error", + "-show_entries", "format=duration", + "-show_entries", "stream=codec_type,duration", + "-of", "json", + recordingPath, + }, + ) require.NoError(t, err, "failed to probe recording durations: %s", string(out)) var probe struct { @@ -471,7 +484,7 @@ func mp4Durations(t *testing.T, data []byte) (float64, float64) { Duration string `json:"duration"` } `json:"format"` } - require.NoError(t, json.Unmarshal(out, &probe), "failed to parse ffprobe output") + require.NoError(t, json.Unmarshal([]byte(out), &probe), "failed to parse ffprobe output") formatDuration, err := strconv.ParseFloat(probe.Format.Duration, 64) require.NoError(t, err, "failed to parse format duration") diff --git a/server/e2e/e2e_zip_transfer_bench_test.go b/server/e2e/e2e_zip_transfer_bench_test.go index 299337d5..dc72fce6 100644 --- a/server/e2e/e2e_zip_transfer_bench_test.go +++ b/server/e2e/e2e_zip_transfer_bench_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/require" ) +const transferFixtureURL = "https://public-ping-bucket-kernel.s3.us-east-1.amazonaws.com/index.html" + // TestZipTransferTiming measures the time to download a directory as a zip and re-upload it. // This is useful for understanding the performance characteristics of the zip transfer endpoints // and evaluating whether alternative compression methods (like zstd) would be beneficial. @@ -110,17 +112,104 @@ func TestZipTransferTiming(t *testing.T) { t.Logf(" Upload throughput: %.1f MB/s (uncompressed)", float64(dirSize)/1024/1024/avgUpload.Seconds()) } -// populateUserData creates some realistic content in the user-data directory -// by executing a playwright script that navigates to a page. +// populateUserData creates browser profile content using a static page origin. func populateUserData(ctx context.Context, client *instanceoapi.ClientWithResponses) error { - // Navigate to example.com to generate some browser state code := ` - await page.goto('https://example.com'); - await page.waitForTimeout(500); - // Visit another page to generate more cache/state - await page.goto('https://www.google.com'); - await page.waitForTimeout(500); - return 'done'; + await page.goto('` + transferFixtureURL + `', { waitUntil: 'load', timeout: 10000 }); + await page.evaluate(async () => { + const payload = 'kernel-transfer-fixture-' + 'x'.repeat(256 * 1024); + localStorage.setItem('transfer-payload', payload); + document.cookie = 'transfer_fixture=ok; path=/; max-age=3600'; + + await new Promise((resolve, reject) => { + const req = indexedDB.open('transfer-fixture-db', 1); + req.onupgradeneeded = () => req.result.createObjectStore('entries', { keyPath: 'id' }); + req.onerror = () => reject(req.error); + req.onsuccess = () => { + const db = req.result; + const tx = db.transaction('entries', 'readwrite'); + tx.oncomplete = () => { + db.close(); + resolve(); + }; + tx.onerror = () => { + db.close(); + reject(tx.error); + }; + tx.onabort = () => { + db.close(); + reject(tx.error); + }; + tx.objectStore('entries').put({ id: 'payload', value: payload }); + }; + }); + + if ('caches' in window) { + const cache = await caches.open('transfer-fixture-cache'); + await cache.put('/fixture-cache-entry', new Response(payload)); + } + }); + + await page.goto('` + transferFixtureURL + `', { waitUntil: 'load', timeout: 10000 }); + const result = await page.evaluate(async () => { + const localStorageValue = localStorage.getItem('transfer-payload') ?? ''; + const cookiePresent = document.cookie.includes('transfer_fixture=ok'); + const indexedDBValue = await new Promise((resolve, reject) => { + const req = indexedDB.open('transfer-fixture-db', 1); + req.onerror = () => reject(req.error); + req.onsuccess = () => { + const db = req.result; + const tx = db.transaction('entries', 'readonly'); + let value = ''; + tx.oncomplete = () => { + db.close(); + resolve(value); + }; + tx.onerror = () => { + db.close(); + reject(tx.error); + }; + const getReq = tx.objectStore('entries').get('payload'); + getReq.onsuccess = () => { + value = getReq.result?.value ?? ''; + }; + getReq.onerror = () => { + db.close(); + reject(getReq.error); + }; + }; + }); + + let cacheValue = ''; + if ('caches' in window) { + const match = await caches.match('/fixture-cache-entry'); + cacheValue = match ? await match.text() : ''; + } + + const expectedBytes = 'kernel-transfer-fixture-'.length + 256 * 1024; + return { + title: document.title, + expectedBytes, + localStorageBytes: localStorageValue.length, + indexedDBBytes: indexedDBValue.length, + cacheBytes: cacheValue.length, + cookiePresent, + }; + }); + + if ( + result.localStorageBytes !== result.expectedBytes || + result.indexedDBBytes !== result.expectedBytes || + result.cacheBytes !== result.expectedBytes || + !result.cookiePresent + ) { + throw new Error('transfer fixture did not persist before snapshot: ' + JSON.stringify(result)); + } + + await context.storageState(); + await page.goto('about:blank', { waitUntil: 'domcontentloaded', timeout: 10000 }); + + return result; ` req := instanceoapi.ExecutePlaywrightCodeJSONRequestBody{Code: code} rsp, err := client.ExecutePlaywrightCodeWithResponse(ctx, req) diff --git a/server/e2e/playwright/index.ts b/server/e2e/playwright/index.ts index 93fcd72a..2010d360 100644 --- a/server/e2e/playwright/index.ts +++ b/server/e2e/playwright/index.ts @@ -322,49 +322,53 @@ class CDPClient { console.log('[cdp] action: verify-mv3-service-worker'); this.page.setDefaultTimeout(timeout); - // Step 1: Navigate to chrome://extensions console.log('[cdp] navigating to chrome://extensions'); await this.page.goto('chrome://extensions'); - await this.page.waitForTimeout(2000); - // Step 2: Enable developer mode by clicking the toggle console.log('[cdp] enabling developer mode'); const devMode = this.page.getByRole('button', { name: 'Developer mode' }); await devMode.click(); - await this.page.waitForTimeout(1000); - // Step 3: Find the extension and extract the ID - // chrome://extensions uses shadow DOM, so we need to use evaluate to pierce it console.log('[cdp] checking for MV3 Service Worker Test extension'); + await this.page.waitForFunction(() => { + const manager = document.querySelector('extensions-manager'); + if (!manager?.shadowRoot) return false; + + const itemList = manager.shadowRoot.querySelector('extensions-item-list'); + if (!itemList?.shadowRoot) return false; + + for (const item of itemList.shadowRoot.querySelectorAll('extensions-item')) { + if (!item.shadowRoot) continue; + + const name = item.shadowRoot.querySelector('#name')?.textContent?.trim() || ''; + const inspectViews = item.shadowRoot.querySelector('#inspect-views')?.textContent || ''; + if (name === 'MV3 Service Worker Test' && inspectViews.includes('service worker')) { + return true; + } + } + + return false; + }, undefined, { timeout }); const extensionInfo = await this.page.evaluate(() => { - // Get the extensions-manager element const manager = document.querySelector('extensions-manager'); if (!manager || !manager.shadowRoot) return null; - // Get the item list const itemList = manager.shadowRoot.querySelector('extensions-item-list'); if (!itemList || !itemList.shadowRoot) return null; - // Find all extension items const items = itemList.shadowRoot.querySelectorAll('extensions-item'); for (const item of items) { if (!item.shadowRoot) continue; - // Get the extension name const nameEl = item.shadowRoot.querySelector('#name'); const name = nameEl?.textContent?.trim() || ''; if (name === 'MV3 Service Worker Test') { - // Get the extension ID from the item's id attribute const id = item.getAttribute('id'); - - // Check if service worker is inactive const inspectViews = item.shadowRoot.querySelector('#inspect-views'); const isInactive = inspectViews?.textContent?.includes('(Inactive)') || false; - - // Check if service worker link exists const hasServiceWorker = inspectViews?.textContent?.includes('service worker') || false; return { id, name, isInactive, hasServiceWorker }; @@ -387,27 +391,30 @@ class CDPClient { throw new Error('Extension does not have a service worker registered'); } - if (extensionInfo.isInactive) { - await this.captureScreenshot({ filename: 'mv3-service-worker-inactive.png' }); - throw new Error('Service worker is marked as (Inactive)'); + if (!extensionInfo.id) { + await this.captureScreenshot({ filename: 'mv3-extension-missing-id.png' }); + throw new Error('MV3 Service Worker Test extension did not expose an extension ID'); } - console.log('[cdp] service worker is active'); + if (extensionInfo.isInactive) { + console.log('[cdp] service worker is inactive before ping; verifying message handling wakes it'); + } else { + console.log('[cdp] service worker is active before ping'); + } - // Step 4: Navigate to the extension's popup const extensionId = extensionInfo.id; const popupUrl = `chrome-extension://${extensionId}/popup.html`; console.log(`[cdp] navigating to popup: ${popupUrl}`); await this.page.goto(popupUrl); - await this.page.waitForTimeout(1000); - // Step 5: Click the "Ping Service Worker" button console.log('[cdp] clicking Ping Service Worker button'); const pingButton = this.page.getByRole('button', { name: 'Ping Service Worker' }); await pingButton.click(); - await this.page.waitForTimeout(2000); + await this.page.waitForFunction(() => { + const status = document.querySelector('#status'); + return status?.classList.contains('success') || status?.classList.contains('error'); + }, undefined, { timeout }); - // Step 6: Verify the status shows success const statusElement = this.page.locator('#status'); const statusText = await statusElement.textContent(); console.log(`[cdp] status text: ${statusText}`); diff --git a/server/lib/cdpmonitor/handlers_test.go b/server/lib/cdpmonitor/handlers_test.go index 3af2afe6..c95a3594 100644 --- a/server/lib/cdpmonitor/handlers_test.go +++ b/server/lib/cdpmonitor/handlers_test.go @@ -314,13 +314,17 @@ func TestTabOpened(t *testing.T) { } func TestBindingAndTimeline(t *testing.T) { - srv := newTestServer(t) - defer srv.close() - - _, ec, cleanup := startMonitor(t, srv, nil) - defer cleanup() + withMonitor := func(t *testing.T) (*testServer, *eventCollector) { + t.Helper() + srv := newTestServer(t) + t.Cleanup(srv.close) + _, ec, cleanup := startMonitor(t, srv, nil) + t.Cleanup(cleanup) + return srv, ec + } t.Run("interaction_click", func(t *testing.T) { + srv, ec := withMonitor(t) srv.sendToMonitor(t, map[string]any{ "method": "Runtime.bindingCalled", "params": map[string]any{ @@ -334,6 +338,7 @@ func TestBindingAndTimeline(t *testing.T) { }) t.Run("interaction_scroll_settled", func(t *testing.T) { + srv, ec := withMonitor(t) srv.sendToMonitor(t, map[string]any{ "method": "Runtime.bindingCalled", "params": map[string]any{ @@ -349,6 +354,7 @@ func TestBindingAndTimeline(t *testing.T) { }) t.Run("layout_shift", func(t *testing.T) { + srv, ec := withMonitor(t) srv.sendToMonitor(t, map[string]any{ "method": "PerformanceTimeline.timelineEventAdded", "params": map[string]any{ @@ -379,6 +385,7 @@ func TestBindingAndTimeline(t *testing.T) { }) t.Run("unknown_binding_ignored", func(t *testing.T) { + srv, ec := withMonitor(t) srv.sendToMonitor(t, map[string]any{ "method": "Runtime.bindingCalled", "params": map[string]any{ @@ -390,6 +397,7 @@ func TestBindingAndTimeline(t *testing.T) { }) t.Run("rate_limited_per_session", func(t *testing.T) { + srv, ec := withMonitor(t) // Send two binding events back-to-back within the 50ms window. // Only the first should produce a published event. before := func() int {