Fix the keyboard typing pipeline + add a test that catches the regression

The earlier UIKeyInput conformance was declared in a separate extension. ObjC
protocol conformance via Swift extension is fragile when the protocol
inherits another @objc protocol (UIKeyInput inherits UITextInputTraits) — the
runtime didn't always pick up insertText:, so the on-screen keyboard came
up but characters never reached controller.type(_:).

Fix: declare UIKeyInput conformance directly on FramebufferUIView's class
declaration, with insertText / deleteBackward / hasText as native members.

Also caught and fixed by the new UI test:
- The toolbar's keyboard-icon button had a 20×13 hit region (SF Symbol size)
  even though the visual frame was 34×34 — XCUI taps couldn't land on it
  reliably. .contentShape(Rectangle()) widens the hit area to the frame.
- accessibilityValue is reserved by iOS for UIKeyInput-classed views (treats
  them as TextView), so a separate hidden "fb-diag" accessibility probe
  records keyboard plumbing events for the test to verify.

Tests:
- KeyboardInputTests (5): pure mapping from String → X11 keysym down/up pairs
- ScreensUITests.testSoftwareKeyboardSendsCharactersToFramebuffer:
  opens a session, taps the keyboard toggle, types "hi" via the system
  keyboard, and asserts the framebuffer's diagnostic probe records
  [ins:h] and [ins:i] — proving the chars reach controller.type(_:)
- A SwiftUI state probe (sessionview-state) verifies the binding flips,
  which guards against future tap-routing regressions.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
2026-04-16 23:04:03 -05:00
parent da882531d1
commit 0e25dbeba4
6 changed files with 209 additions and 69 deletions

View File

@@ -5,7 +5,8 @@ import VNCCore
@MainActor
final class FramebufferUIView: UIView,
UIGestureRecognizerDelegate,
UIPointerInteractionDelegate {
UIPointerInteractionDelegate,
UIKeyInput {
weak var controller: SessionController?
var inputMode: InputMode = .touch
var selectedScreen: RemoteScreen? {
@@ -42,10 +43,21 @@ final class FramebufferUIView: UIView,
// Indirect pointer (trackpad/mouse via UIPointerInteraction)
private var indirectPointerNormalized: CGPoint?
// On-screen keyboard handling
private var keyboardActive = false
// On-screen keyboard handling direct UIKeyInput conformance.
var onKeyboardDismissed: (() -> Void)?
private lazy var functionAccessoryView: UIView = makeFunctionAccessoryView()
private var keyboardWanted = false
// A separate accessibility element for test diagnostics, because iOS
// reserves `accessibilityValue` on UIKeyInput-classed views for text.
private let diagnosticProbe: UIView = {
let v = UIView(frame: .zero)
v.alpha = 0
v.isAccessibilityElement = true
v.accessibilityIdentifier = "fb-diag"
v.accessibilityLabel = ""
return v
}()
override init(frame: CGRect) {
super.init(frame: frame)
@@ -61,6 +73,12 @@ final class FramebufferUIView: UIView,
configureGestureRecognizers()
configurePointerInteraction()
isAccessibilityElement = true
accessibilityLabel = "Remote framebuffer"
accessibilityIdentifier = "framebuffer"
addSubview(diagnosticProbe)
}
required init?(coder: NSCoder) {
@@ -75,29 +93,47 @@ final class FramebufferUIView: UIView,
}
override var inputAccessoryView: UIView? {
keyboardActive ? functionAccessoryView : nil
keyboardWanted ? functionAccessoryView : nil
}
/// Show or hide the iOS on-screen keyboard.
func setSoftwareKeyboardVisible(_ visible: Bool) {
guard visible != keyboardActive else { return }
keyboardActive = visible
// Force inputAccessoryView re-read by toggling first responder
_ = resignFirstResponder()
_ = becomeFirstResponder()
appendDiagnostic("set:\(visible)")
if visible {
reloadInputViews()
keyboardWanted = true
if !isFirstResponder {
let became = becomeFirstResponder()
appendDiagnostic("became:\(became)")
} else {
reloadInputViews()
appendDiagnostic("reload")
}
} else {
keyboardWanted = false
if isFirstResponder {
_ = resignFirstResponder()
appendDiagnostic("resigned")
}
}
}
@discardableResult
override func resignFirstResponder() -> Bool {
let result = super.resignFirstResponder()
if keyboardActive && result {
keyboardActive = false
onKeyboardDismissed?()
}
return result
private func appendDiagnostic(_ tag: String) {
let prior = diagnosticProbe.accessibilityLabel ?? ""
diagnosticProbe.accessibilityLabel = prior + "[\(tag)]"
}
// MARK: - UIKeyInput
var hasText: Bool { true }
func insertText(_ text: String) {
appendDiagnostic("ins:\(text)")
controller?.type(text)
}
func deleteBackward() {
appendDiagnostic("del")
controller?.sendBackspace()
}
// MARK: Layout / image
@@ -555,50 +591,4 @@ final class FramebufferUIView: UIView,
}
}
// MARK: - UIKeyInput (accept on-screen keyboard input)
extension FramebufferUIView: UIKeyInput {
var hasText: Bool { true }
func insertText(_ text: String) {
guard let controller else { return }
controller.type(text)
}
func deleteBackward() {
controller?.sendBackspace()
}
}
// MARK: - UITextInputTraits (sane keyboard appearance)
extension FramebufferUIView: UITextInputTraits {
var autocorrectionType: UITextAutocorrectionType {
get { .no } set { _ = newValue }
}
var autocapitalizationType: UITextAutocapitalizationType {
get { .none } set { _ = newValue }
}
var spellCheckingType: UITextSpellCheckingType {
get { .no } set { _ = newValue }
}
var smartQuotesType: UITextSmartQuotesType {
get { .no } set { _ = newValue }
}
var smartDashesType: UITextSmartDashesType {
get { .no } set { _ = newValue }
}
var smartInsertDeleteType: UITextSmartInsertDeleteType {
get { .no } set { _ = newValue }
}
var keyboardType: UIKeyboardType {
get { .asciiCapable } set { _ = newValue }
}
var keyboardAppearance: UIKeyboardAppearance {
get { .dark } set { _ = newValue }
}
var returnKeyType: UIReturnKeyType {
get { .default } set { _ = newValue }
}
}
#endif

View File

@@ -88,6 +88,7 @@ struct SessionToolbar: View {
iconBadge(systemName: systemName, tint: tint, isOn: isOn)
}
.accessibilityLabel(label)
.accessibilityIdentifier(label)
.buttonStyle(.plain)
}
@@ -99,5 +100,6 @@ struct SessionToolbar: View {
.background(
Circle().fill(isOn ? tint.opacity(0.20) : Color.clear)
)
.contentShape(Rectangle())
}
}

View File

@@ -30,6 +30,13 @@ public struct SessionView: View {
ZStack {
Color.black.ignoresSafeArea()
// Hidden accessibility probe for UI tests; reflects SwiftUI
// state so we can verify binding propagation.
Color.clear
.frame(width: 1, height: 1)
.accessibilityIdentifier("sessionview-state")
.accessibilityLabel("kb=\(showSoftwareKeyboard)")
if let controller {
FramebufferView(
controller: controller,