Re-architect iOS XCUITest suite: per-test isolation + domain organization
Migrate the XCUITest suite off the legacy shared-account model (and the prior Django-style auth assumptions) to a parallel-safe, domain-organized architecture, validated end-to-end against the live Kratos stack. Isolation (parallel-safe by construction): - Core/Fixtures/TestAccount.swift: each test mints its own pre-verified Kratos identity (uit_<domain>_<uuid>@test.honeydue.local), logs in, seeds under its own token, and deletes the identity in teardown (cascading all data + clearing Kratos). No shared testuser; parallel workers no longer race. - AuthenticatedUITestCase rewritten to that model (member surface preserved); adds requiresResidence / seedAccountPreconditions to seed UI-gated data BEFORE login (a fresh account is empty at login). Organization (255 tests preserved, none dropped): - 21 domain suites under Auth/ Onboarding/ Residence/ Task/ Contractor/ Document/ Sharing/ Navigation/ Smoke/ CrossCutting/ E2E/, consistent <Domain>UITests naming. Removes the Suite1..11 / AAA_ / ZZ_ / Tests/Rebuild naming chaos and the overlapping task/residence/auth suites. Runner + test plans: - run_ui_tests.sh: Smoke gate -> Seed -> Parallel(8 workers) -> Sweep. The parallel phase runs the whole target minus phase-managed suites via -skip-testing, so new suites auto-include (no hand-maintained list to drift). Drops the 2-worker cap and Suite6 isolation (isolation made them moot). - HoneyDueUITests.xctestplan skips the 4 phase-managed suites; adds Smoke.xctestplan. Kratos auth fixes folded in (login/verify/reset endpoints removed under Kratos): real Mailpit verification codes replace the obsolete fixed "123456"; teardown deletes Kratos identities; admin-panel login uses the correct seeded password. Build green; isolation, parallelism, and the precondition/sharing migrations validated against the live stack (0 leaked accounts). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
+87
-149
@@ -1,201 +1,139 @@
|
||||
#!/bin/bash
|
||||
# run_ui_tests.sh — Three-phase UI test runner with parallel middle phase
|
||||
# run_ui_tests.sh — Phased UI test runner.
|
||||
#
|
||||
# Architecture: every test mints its OWN isolated Kratos account (see
|
||||
# Core/Fixtures/TestAccount.swift + AuthenticatedUITestCase), so suites are
|
||||
# fully independent and the parallel phase scales to many workers with no
|
||||
# cross-suite data races. There is no per-suite ordering and no Suite6
|
||||
# special-casing anymore.
|
||||
#
|
||||
# Phases:
|
||||
# 0. Smoke gate — fast launch/login sanity. Abort the run if it fails.
|
||||
# 1. Seed — ensure baseline accounts exist (AAA_SeedTests).
|
||||
# 2. Parallel — the WHOLE target minus the four phase-managed suites, via
|
||||
# -skip-testing. New suites are auto-included (no hand-
|
||||
# maintained list to drift), run at $WORKERS workers.
|
||||
# 3. Sweep — clear-all-data + delete leaked uit_* Kratos identities
|
||||
# (SuiteZZ_CleanupTests). Non-blocking.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_ui_tests.sh # Default: iPhone 17 Pro, 4 workers
|
||||
# ./run_ui_tests.sh "iPhone Air" 3 # Custom device and worker count
|
||||
# ./run_ui_tests.sh --skip-seed # Skip seeding (already done)
|
||||
# ./run_ui_tests.sh --skip-cleanup # Skip cleanup at end
|
||||
# ./run_ui_tests.sh --only-parallel # Only run parallel phase
|
||||
# ./run_ui_tests.sh # iPhone 17 Pro, 8 workers
|
||||
# ./run_ui_tests.sh "iPhone Air" 6 # custom device + worker count
|
||||
# ./run_ui_tests.sh --skip-seed # skip phase 1
|
||||
# ./run_ui_tests.sh --skip-cleanup # skip phase 3
|
||||
# ./run_ui_tests.sh --only-parallel # only phase 2
|
||||
# ./run_ui_tests.sh --smoke # only phase 0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT="$SCRIPT_DIR/honeyDue.xcodeproj"
|
||||
SCHEME="HoneyDueUITests"
|
||||
TARGET="HoneyDueUITests"
|
||||
DESTINATION="platform=iOS Simulator,name=iPhone 17 Pro"
|
||||
# 2 workers avoids simulator contention that caused intermittent XCUITest
|
||||
# typing / UI-update races (Suite5/7/8 flakes under 4-worker load). Phase 2b
|
||||
# isolates Suite6 further.
|
||||
WORKERS=2
|
||||
WORKERS=8
|
||||
|
||||
# Suites that run in their own phases — excluded from the parallel phase.
|
||||
PHASE_MANAGED=(
|
||||
"$TARGET/AAA_SeedTests"
|
||||
"$TARGET/SuiteZZ_CleanupTests"
|
||||
"$TARGET/SmokeUITests"
|
||||
"$TARGET/AppLaunchUITests"
|
||||
)
|
||||
|
||||
SKIP_SEED=false
|
||||
SKIP_CLEANUP=false
|
||||
ONLY_PARALLEL=false
|
||||
ONLY_SMOKE=false
|
||||
|
||||
# Parse flags (positional args for device/workers, flags for skip options)
|
||||
POSITIONAL_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--skip-seed) SKIP_SEED=true ;;
|
||||
--skip-cleanup) SKIP_CLEANUP=true ;;
|
||||
--only-parallel) ONLY_PARALLEL=true; SKIP_SEED=true; SKIP_CLEANUP=true ;;
|
||||
--smoke) ONLY_SMOKE=true ;;
|
||||
*) POSITIONAL_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ${#POSITIONAL_ARGS[@]} -ge 1 ]; then
|
||||
DESTINATION="platform=iOS Simulator,name=${POSITIONAL_ARGS[0]}"
|
||||
fi
|
||||
if [ ${#POSITIONAL_ARGS[@]} -ge 2 ]; then
|
||||
WORKERS="${POSITIONAL_ARGS[1]}"
|
||||
fi
|
||||
[ ${#POSITIONAL_ARGS[@]} -ge 1 ] && DESTINATION="platform=iOS Simulator,name=${POSITIONAL_ARGS[0]}"
|
||||
[ ${#POSITIONAL_ARGS[@]} -ge 2 ] && WORKERS="${POSITIONAL_ARGS[1]}"
|
||||
|
||||
RESULTS_DIR="$SCRIPT_DIR/build/test-results"
|
||||
DERIVED_DATA="$SCRIPT_DIR/build/DerivedData"
|
||||
mkdir -p "$RESULTS_DIR" "$DERIVED_DATA"
|
||||
|
||||
BOLD='\033[1m'
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[0;33m'
|
||||
RESET='\033[0m'
|
||||
|
||||
phase_header() {
|
||||
echo ""
|
||||
echo -e "${BOLD}════════════════════════════════════════════════════${RESET}"
|
||||
echo -e "${BOLD} $1${RESET}"
|
||||
echo -e "${BOLD}════════════════════════════════════════════════════${RESET}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Seed tests — must run first, sequentially
|
||||
SEED_TESTS=(
|
||||
"-only-testing:HoneyDueUITests/AAA_SeedTests"
|
||||
)
|
||||
|
||||
# All parallelizable test classes
|
||||
PARALLEL_TESTS=(
|
||||
"-only-testing:HoneyDueUITests/AuthCriticalPathTests"
|
||||
"-only-testing:HoneyDueUITests/NavigationCriticalPathTests"
|
||||
"-only-testing:HoneyDueUITests/SmokeTests"
|
||||
"-only-testing:HoneyDueUITests/SimpleLoginTest"
|
||||
"-only-testing:HoneyDueUITests/Suite0_OnboardingRebuildTests"
|
||||
"-only-testing:HoneyDueUITests/Suite1_RegistrationTests"
|
||||
"-only-testing:HoneyDueUITests/Suite2_AuthenticationRebuildTests"
|
||||
"-only-testing:HoneyDueUITests/Suite3_ResidenceRebuildTests"
|
||||
"-only-testing:HoneyDueUITests/Suite4_ComprehensiveResidenceTests"
|
||||
"-only-testing:HoneyDueUITests/Suite5_TaskTests"
|
||||
"-only-testing:HoneyDueUITests/Suite7_ContractorTests"
|
||||
"-only-testing:HoneyDueUITests/Suite8_DocumentWarrantyTests"
|
||||
"-only-testing:HoneyDueUITests/Suite9_IntegrationE2ETests"
|
||||
"-only-testing:HoneyDueUITests/Suite10_ComprehensiveE2ETests"
|
||||
)
|
||||
|
||||
# Suite6 runs in a smaller-parallel phase of its own. Under 4-worker contention
|
||||
# with 14 other classes, SwiftUI's TextField binding intermittently lags behind
|
||||
# XCUITest typing, leaving the Add-Task form un-submittable. Isolating Suite6
|
||||
# to 2 workers gives the binding enough time to flush reliably.
|
||||
SUITE6_TESTS=(
|
||||
"-only-testing:HoneyDueUITests/Suite6_ComprehensiveTaskTests"
|
||||
)
|
||||
|
||||
# Cleanup tests — must run last, sequentially
|
||||
CLEANUP_TESTS=(
|
||||
"-only-testing:HoneyDueUITests/SuiteZZ_CleanupTests"
|
||||
)
|
||||
|
||||
run_phase() {
|
||||
local phase_name="$1"
|
||||
local result_path="$RESULTS_DIR/${phase_name}.xcresult"
|
||||
shift
|
||||
local extra_args=("$@")
|
||||
BOLD='\033[1m'; GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m'
|
||||
phase_header() { echo ""; echo -e "${BOLD}════════════════════════════════════════════════════${RESET}"; echo -e "${BOLD} $1${RESET}"; echo -e "${BOLD}════════════════════════════════════════════════════${RESET}"; echo ""; }
|
||||
|
||||
run_xcodebuild() {
|
||||
local result_path="$RESULTS_DIR/$1.xcresult"; shift
|
||||
rm -rf "$result_path"
|
||||
|
||||
xcodebuild test \
|
||||
-project "$PROJECT" \
|
||||
-scheme "$SCHEME" \
|
||||
-destination "$DESTINATION" \
|
||||
-derivedDataPath "$DERIVED_DATA" \
|
||||
-resultBundlePath "$result_path" \
|
||||
"${extra_args[@]}" \
|
||||
2>&1 | tail -30
|
||||
|
||||
-project "$PROJECT" -scheme "$SCHEME" -destination "$DESTINATION" \
|
||||
-derivedDataPath "$DERIVED_DATA" -resultBundlePath "$result_path" \
|
||||
"$@" 2>&1 | tail -40
|
||||
return ${PIPESTATUS[0]}
|
||||
}
|
||||
|
||||
OVERALL_START=$(date +%s)
|
||||
|
||||
# ── Phase 1: Seed ──────────────────────────────────────────────
|
||||
if [ "$SKIP_SEED" = false ]; then
|
||||
phase_header "Phase 1/3: Seeding test data (sequential)"
|
||||
SEED_START=$(date +%s)
|
||||
|
||||
if run_phase "SeedTests" "${SEED_TESTS[@]}"; then
|
||||
SEED_END=$(date +%s)
|
||||
echo -e "\n${GREEN}✓ Seed phase passed ($(( SEED_END - SEED_START ))s)${RESET}"
|
||||
# ── Phase 0: Smoke gate ────────────────────────────────────────
|
||||
if [ "$ONLY_PARALLEL" = false ]; then
|
||||
phase_header "Phase 0: Smoke gate"
|
||||
if run_xcodebuild "Smoke" \
|
||||
-only-testing:"$TARGET/SmokeUITests" \
|
||||
-only-testing:"$TARGET/AppLaunchUITests"; then
|
||||
echo -e "${GREEN}✓ Smoke passed${RESET}"
|
||||
else
|
||||
SEED_END=$(date +%s)
|
||||
echo -e "\n${RED}✗ Seed phase FAILED ($(( SEED_END - SEED_START ))s)${RESET}"
|
||||
echo -e "${RED} Cannot proceed without seeded data. Aborting.${RESET}"
|
||||
echo -e "${RED}✗ Smoke FAILED — aborting (app can't launch/log in).${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
[ "$ONLY_SMOKE" = true ] && exit 0
|
||||
fi
|
||||
|
||||
# ── Phase 2: Parallel Tests ───────────────────────────────────
|
||||
phase_header "Phase 2/3: Running tests in parallel ($WORKERS workers)"
|
||||
PARALLEL_START=$(date +%s)
|
||||
|
||||
if run_phase "ParallelTests" \
|
||||
-parallel-testing-enabled YES \
|
||||
-parallel-testing-worker-count "$WORKERS" \
|
||||
"${PARALLEL_TESTS[@]}"; then
|
||||
PARALLEL_END=$(date +%s)
|
||||
echo -e "\n${GREEN}✓ Parallel phase passed ($(( PARALLEL_END - PARALLEL_START ))s)${RESET}"
|
||||
PARALLEL_PASSED=true
|
||||
else
|
||||
PARALLEL_END=$(date +%s)
|
||||
echo -e "\n${RED}✗ Parallel phase FAILED ($(( PARALLEL_END - PARALLEL_START ))s)${RESET}"
|
||||
PARALLEL_PASSED=false
|
||||
fi
|
||||
|
||||
# ── Phase 2b: Suite6 (isolated parallel) ──────────────────────
|
||||
phase_header "Phase 2b: Suite6 task tests (2 workers, isolated)"
|
||||
SUITE6_START=$(date +%s)
|
||||
|
||||
if run_phase "Suite6Tests" \
|
||||
-parallel-testing-enabled YES \
|
||||
-parallel-testing-worker-count 2 \
|
||||
"${SUITE6_TESTS[@]}"; then
|
||||
SUITE6_END=$(date +%s)
|
||||
echo -e "\n${GREEN}✓ Suite6 phase passed ($(( SUITE6_END - SUITE6_START ))s)${RESET}"
|
||||
SUITE6_PASSED=true
|
||||
else
|
||||
SUITE6_END=$(date +%s)
|
||||
echo -e "\n${RED}✗ Suite6 phase FAILED ($(( SUITE6_END - SUITE6_START ))s)${RESET}"
|
||||
SUITE6_PASSED=false
|
||||
fi
|
||||
|
||||
# ── Phase 3: Cleanup ──────────────────────────────────────────
|
||||
if [ "$SKIP_CLEANUP" = false ]; then
|
||||
phase_header "Phase 3/3: Cleaning up test data (sequential)"
|
||||
CLEANUP_START=$(date +%s)
|
||||
|
||||
if run_phase "CleanupTests" "${CLEANUP_TESTS[@]}"; then
|
||||
CLEANUP_END=$(date +%s)
|
||||
echo -e "\n${GREEN}✓ Cleanup phase passed ($(( CLEANUP_END - CLEANUP_START ))s)${RESET}"
|
||||
# ── Phase 1: Seed ──────────────────────────────────────────────
|
||||
if [ "$SKIP_SEED" = false ]; then
|
||||
phase_header "Phase 1: Seed baseline accounts"
|
||||
if run_xcodebuild "Seed" -only-testing:"$TARGET/AAA_SeedTests"; then
|
||||
echo -e "${GREEN}✓ Seed passed${RESET}"
|
||||
else
|
||||
CLEANUP_END=$(date +%s)
|
||||
echo -e "\n${YELLOW}⚠ Cleanup phase failed ($(( CLEANUP_END - CLEANUP_START ))s) — non-blocking${RESET}"
|
||||
echo -e "${RED}✗ Seed FAILED — aborting.${RESET}"; exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────
|
||||
OVERALL_END=$(date +%s)
|
||||
TOTAL_TIME=$(( OVERALL_END - OVERALL_START ))
|
||||
# ── Phase 2: Parallel (whole target minus phase-managed) ───────
|
||||
phase_header "Phase 2: Parallel suite ($WORKERS workers)"
|
||||
SKIP_ARGS=()
|
||||
for t in "${PHASE_MANAGED[@]}"; do SKIP_ARGS+=( -skip-testing:"$t" ); done
|
||||
PARALLEL_START=$(date +%s)
|
||||
if run_xcodebuild "Parallel" \
|
||||
-only-testing:"$TARGET" "${SKIP_ARGS[@]}" \
|
||||
-parallel-testing-enabled YES -parallel-testing-worker-count "$WORKERS"; then
|
||||
PARALLEL_PASSED=true; echo -e "${GREEN}✓ Parallel phase passed${RESET}"
|
||||
else
|
||||
PARALLEL_PASSED=false; echo -e "${RED}✗ Parallel phase FAILED${RESET}"
|
||||
fi
|
||||
PARALLEL_END=$(date +%s)
|
||||
|
||||
# ── Phase 3: Sweep ─────────────────────────────────────────────
|
||||
if [ "$SKIP_CLEANUP" = false ]; then
|
||||
phase_header "Phase 3: Sweep leaked accounts + data"
|
||||
if run_xcodebuild "Sweep" -only-testing:"$TARGET/SuiteZZ_CleanupTests"; then
|
||||
echo -e "${GREEN}✓ Sweep passed${RESET}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Sweep failed (non-blocking)${RESET}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Summary ────────────────────────────────────────────────────
|
||||
phase_header "Summary"
|
||||
echo " Total time: ${TOTAL_TIME}s"
|
||||
echo " Workers: $WORKERS"
|
||||
echo " Total time: $(( $(date +%s) - OVERALL_START ))s"
|
||||
echo " Parallel: $(( PARALLEL_END - PARALLEL_START ))s @ $WORKERS workers"
|
||||
echo " Results: $RESULTS_DIR/"
|
||||
echo ""
|
||||
|
||||
if [ "$PARALLEL_PASSED" = true ] && [ "${SUITE6_PASSED:-true}" = true ]; then
|
||||
echo -e " ${GREEN}${BOLD}ALL TESTS PASSED${RESET}"
|
||||
exit 0
|
||||
if [ "${PARALLEL_PASSED:-false}" = true ]; then
|
||||
echo -e " ${GREEN}${BOLD}ALL TESTS PASSED${RESET}"; exit 0
|
||||
else
|
||||
echo -e " ${RED}${BOLD}TESTS FAILED${RESET}"
|
||||
echo -e " Check results: open $RESULTS_DIR/"
|
||||
exit 1
|
||||
echo -e " ${RED}${BOLD}TESTS FAILED${RESET} — open $RESULTS_DIR/"; exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user