Support semantics traversal and sort

compose/ui/ui-uikit/src/uikitMain/objc/CMPUIKitUtils/CMPUIKitUtils/CMPAccessibilityElement.h

@@ -31,6 +31,8 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (UIAccessibilityTraits)accessibilityTraits;
 
+- (UIAccessibilityContainerType)accessibilityContainerType;
+
 - (NSString *__nullable)accessibilityIdentifier;
 
 - (NSString *__nullable)accessibilityHint;

compose/ui/ui-uikit/src/uikitMain/objc/CMPUIKitUtils/CMPUIKitUtils/CMPAccessibilityElement.m

@@ -41,6 +41,10 @@ - (UIAccessibilityTraits)accessibilityTraits {
     return [super accessibilityTraits];
 }
 
+- (UIAccessibilityContainerType)accessibilityContainerType {
+    return [super accessibilityContainerType];
+}
+
 - (NSString *__nullable)accessibilityIdentifier {
     return [super accessibilityIdentifier];
 }

compose/ui/ui/src/uikitMain/kotlin/androidx/compose/ui/platform/Accessibility.uikit.kt

@@ -23,8 +23,6 @@ import androidx.compose.ui.platform.accessibility.accessibilityCustomActions
 import androidx.compose.ui.platform.accessibility.accessibilityLabel
 import androidx.compose.ui.platform.accessibility.accessibilityTraits
 import androidx.compose.ui.platform.accessibility.accessibilityValue
-import androidx.compose.ui.platform.accessibility.canBeAccessibilityElement
-import androidx.compose.ui.platform.accessibility.isRTL
 import androidx.compose.ui.platform.accessibility.isScreenReaderFocusable
 import androidx.compose.ui.platform.accessibility.scrollIfPossible
 import androidx.compose.ui.platform.accessibility.scrollToCenterRectIfNeeded
@@ -34,13 +32,18 @@ import androidx.compose.ui.semantics.SemanticsActions
 import androidx.compose.ui.semantics.SemanticsNode
 import androidx.compose.ui.semantics.SemanticsOwner
 import androidx.compose.ui.semantics.SemanticsProperties
+import androidx.compose.ui.semantics.getAllUncoveredSemanticsNodesToIntObjectMap
 import androidx.compose.ui.semantics.getOrNull
+import androidx.compose.ui.semantics.isImportantForAccessibility
+import androidx.compose.ui.semantics.sortByGeometryGroupings
 import androidx.compose.ui.uikit.density
 import androidx.compose.ui.uikit.utils.CMPAccessibilityElement
+import androidx.compose.ui.unit.IntRect
 import androidx.compose.ui.unit.asCGRect
 import androidx.compose.ui.unit.asDpRect
 import androidx.compose.ui.unit.toDpRect
 import androidx.compose.ui.unit.toRect
+import androidx.compose.ui.util.fastForEach
 import androidx.compose.ui.viewinterop.InteropWrappingView
 import androidx.compose.ui.viewinterop.NativeAccessibilityViewSemanticsKey
 import kotlin.coroutines.CoroutineContext
@@ -70,6 +73,9 @@ import platform.CoreGraphics.CGRectGetMinY
 import platform.CoreGraphics.CGRectIsEmpty
 import platform.Foundation.NSNotFound
 import platform.UIKit.NSStringFromCGRect
+import platform.UIKit.UIAccessibilityContainerType
+import platform.UIKit.UIAccessibilityContainerTypeNone
+import platform.UIKit.UIAccessibilityContainerTypeSemanticGroup
 import platform.UIKit.UIAccessibilityCustomAction
 import platform.UIKit.UIAccessibilityFocusedElement
 import platform.UIKit.UIAccessibilityLayoutChangedNotification
@@ -125,6 +131,8 @@ private sealed interface AccessibilityNode {
     val accessibilityHint: String? get() = null
     val accessibilityValue: String? get() = null
     val accessibilityTraits: UIAccessibilityTraits get() = UIAccessibilityTraitNone
+    val accessibilityContainerType: UIAccessibilityContainerType
+        get() = UIAccessibilityContainerTypeNone
     val accessibilityIdentifier: String? get() = null
     val accessibilityInteropView: InteropWrappingView? get() = null
     val accessibilityCustomActions: List<UIAccessibilityCustomAction> get() = emptyList()
@@ -145,7 +153,8 @@ private sealed interface AccessibilityNode {
     class Semantics(
         private val semanticsNode: SemanticsNode,
         private val mediator: AccessibilityMediator,
-        private val ignoreSemanticChildren: Boolean = false
+        private val ignoreSemanticChildren: Boolean = false,
+        private val frame: IntRect?,
     ) : AccessibilityNode {
         private val cachedConfig = semanticsNode.copyWithMergingEnabled().config
 
@@ -156,7 +165,7 @@ private sealed interface AccessibilityNode {
             get() = semanticsNode.isScreenReaderFocusable()
 
         override val accessibilityFrame: CValue<CGRect>
-            get() = mediator.convertToAppWindowCGRect(semanticsNode.boundsInWindow)
+            get() = mediator.convertToAppWindowCGRect(frame?.toRect() ?: semanticsNode.unclippedBoundsInWindow)
 
         override val accessibilityInteropView: InteropWrappingView?
             get() = if (ignoreSemanticChildren) {
@@ -216,7 +225,7 @@ private sealed interface AccessibilityNode {
         }
 
         override fun accessibilityElementDidBecomeFocused() {
-            mediator.debugLogger?.apply {
+            accessibilityDebugLogger?.apply {
                 log(null)
                 log("Focused on:")
                 log(cachedConfig)
@@ -276,15 +285,25 @@ private sealed interface AccessibilityNode {
      * semantic node with all its children.
      */
     class Container(
-        containerNode: SemanticsNode,
-        mediator: AccessibilityMediator
+        private val containerNode: SemanticsNode,
+        mediator: AccessibilityMediator,
+        frame: IntRect?,
     ) : AccessibilityNode {
         override val key: AccessibilityElementKey = containerNode.containerKey
 
         override val isAccessibilityElement = false
 
         override val accessibilityFrame: CValue<CGRect> =
-            mediator.convertToAppWindowCGRect(containerNode.boundsInWindow)
+            mediator.convertToAppWindowCGRect(
+                frame?.toRect() ?: containerNode.unclippedBoundsInWindow
+            )
+
+        override val accessibilityContainerType: UIAccessibilityContainerType
+            get() = if (containerNode.isTraversalGroup) {
+                UIAccessibilityContainerTypeSemanticGroup
+            } else {
+                UIAccessibilityContainerTypeNone
+            }
     }
 }
 
@@ -572,15 +591,16 @@ internal interface AccessibilityDebugLogger {
 
 private val accessibilityDebugLogger: AccessibilityDebugLogger? = null
 // Uncomment for debugging:
-// private val accessibilityDebugLogger = object : AccessibilityDebugLogger {
-//     override fun log(message: Any?) {
-//         if (message == null) {
-//             println()
-//         } else {
-//             println("[a11y]: $message")
+// private val accessibilityDebugLogger: AccessibilityDebugLogger? =
+//     object : AccessibilityDebugLogger {
+//         override fun log(message: Any?) {
+//             if (message == null) {
+//                 println()
+//             } else {
+//                 println("[a11y]: $message")
+//             }
 //         }
 //     }
-// }
 
 private sealed interface AccessibilityElementFocusMode {
     val targetElementKey: AccessibilityElementKey?
@@ -620,7 +640,7 @@ internal class AccessibilityMediator(
     private var focusMode: AccessibilityElementFocusMode = AccessibilityElementFocusMode.Initial
         set(value) {
             field = value
-            debugLogger?.log("Focus mode: $focusMode")
+            accessibilityDebugLogger?.log("Focus mode: $focusMode")
         }
 
     /**
@@ -638,9 +658,6 @@ internal class AccessibilityMediator(
      */
     private val invalidationChannel = Channel<Unit>(1, onBufferOverflow = BufferOverflow.DROP_LATEST)
 
-    var debugLogger: AccessibilityDebugLogger? = null
-        private set
-
     /**
      * Job to cancel tree syncing when the mediator is disposed.
      */
@@ -697,15 +714,13 @@ internal class AccessibilityMediator(
                     // Workaround for the channel buffering two invalidations despite the capacity of 1
                 }
 
-                debugLogger = accessibilityDebugLogger.takeIf { isEnabled }
-
                 if (isEnabled) {
                     if (isAccessibilityActive) {
                         scheduleAccessibilityDisablingAndCleanup()
                         val time = measureTime {
                             sync(invalidationKind)
                         }
-                        debugLogger?.log("AccessibilityMediator.sync took $time")
+                        accessibilityDebugLogger?.log("AccessibilityMediator.sync took $time")
                     }
                 } else if (root.element != null) {
                     root.element = null
@@ -773,11 +788,11 @@ internal class AccessibilityMediator(
                 scrollResult.announceMessage()
             )
 
-            debugLogger?.log("PageScrolled")
+            accessibilityDebugLogger?.log("PageScrolled")
 
             if (accessibilityElementsMap[focusedNode.semanticsKey] == null) {
                 val element = findClosestElementToRect(rect = focusedRectInWindow)
-                debugLogger?.log("LayoutChanged, result: $element")
+                accessibilityDebugLogger?.log("LayoutChanged, result: $element")
 
                 (element as? AccessibilityElement)?.let {
                     focusMode = AccessibilityElementFocusMode.KeepFocus(element.key)
@@ -789,14 +804,14 @@ internal class AccessibilityMediator(
     }
 
     fun onSemanticsChange() {
-        debugLogger?.log("onSemanticsChange")
+        accessibilityDebugLogger?.log("onSemanticsChange")
 
         invalidationKind = SemanticsTreeInvalidationKind.COMPLETE
         invalidationChannel.trySend(Unit)
     }
 
     fun onLayoutChange(nodeId: Int) {
-        debugLogger?.log("onLayoutChange (nodeId=$nodeId)")
+        accessibilityDebugLogger?.log("onLayoutChange (nodeId=$nodeId)")
 
         // TODO: Properly implement layout invalidation, taking into account that semantics
         //  can also change after the `onLayoutChange` event.
@@ -851,50 +866,77 @@ internal class AccessibilityMediator(
     private fun traverseSemanticsTree(rootNode: SemanticsNode): AccessibilityElement {
         val presentIds = mutableSetOf<AccessibilityElementKey>()
 
-        fun traverseSemanticsNode(node: SemanticsNode): AccessibilityElement {
-            presentIds.add(node.semanticsKey)
-            val semanticsChildren = node
-                .replacedChildren
-                .filter { it.isValid }
-                .sortedByAccessibilityOrder(node.isRTL)
-
-            val children = semanticsChildren.map(::traverseSemanticsNode)
+        val nodes = owner.getAllUncoveredSemanticsNodesToIntObjectMap(rootNode.id)
+
+        // Flatten all children except nodes inside traversal groups to:
+        // - have the same traversal order as on Android
+        // - allow navigation between semantic containers on iOS
+        fun flattenChildrenInsideTraversalGroup(
+            node: SemanticsNode, semanticsChildren: ArrayList<SemanticsNode>
+        ) {
+            node.children.fastForEach { child ->
+                if (child.isValid && (nodes.contains(child.id) || child.isTraversalGroup)) {
+                    semanticsChildren.add(child)
+                }
+                if (!child.isTraversalGroup) {
+                    flattenChildrenInsideTraversalGroup(child, semanticsChildren)
+                }
+            }
+        }
 
-            return if (node.canBeAccessibilityElement() && semanticsChildren.isNotEmpty()) {
-                presentIds.add(node.containerKey)
+        fun traverseGroup(node: SemanticsNode): AccessibilityElement {
+            presentIds.add(node.semanticsKey)
 
-                // Unlike Android, iOS Accessibility engine does not traverse inside accessibility
-                // elements which marked as accessible (focusable).
-                // To align behavior, flatting the node with all its children and arranging them
-                // inside the synthetic container node.
-                val containerElement = createOrUpdateAccessibilityElement(
+            if (!node.isTraversalGroup && node.id != rootNode.id) {
+                return createOrUpdateAccessibilityElement(
                     node = AccessibilityNode.Semantics(
                         semanticsNode = node,
                         mediator = this,
-                        ignoreSemanticChildren = true
+                        frame = nodes[node.id]?.adjustedBounds
                     ),
                     children = emptyList()
                 )
+            }
+
+            val flattenSemanticsChildren = ArrayList<SemanticsNode>()
+            flattenChildrenInsideTraversalGroup(node, flattenSemanticsChildren)
+
+            val children = flattenSemanticsChildren
+                .filter { it.boundsInWindow != Rect.Zero }
+                .let { node.sortByGeometryGroupings(it) }
+                .map(::traverseGroup)
+
+            val containerElement = node.takeIf { it.isImportantForAccessibility() }?.let {
                 createOrUpdateAccessibilityElement(
-                    node = AccessibilityNode.Container(containerNode = node, mediator = this),
-                    children = listOf(containerElement) + children
-                )
-            } else {
-                createOrUpdateAccessibilityElement(
-                    node = AccessibilityNode.Semantics(semanticsNode = node, mediator = this),
-                    children = children
+                    node = AccessibilityNode.Semantics(
+                        semanticsNode = node,
+                        mediator = this,
+                        ignoreSemanticChildren = true,
+                        frame = nodes[node.id]?.adjustedBounds
+                    ),
+                    children = emptyList()
                 )
             }
+
+            presentIds.add(node.containerKey)
+            return createOrUpdateAccessibilityElement(
+                node = AccessibilityNode.Container(
+                    containerNode = node,
+                    mediator = this,
+                    frame = nodes[node.id]?.adjustedBounds
+                ),
+                children = listOfNotNull(containerElement) + children
+            )
         }
 
-        val rootAccessibilityElement = traverseSemanticsNode(rootNode)
+        val rootAccessibilityElement = traverseGroup(rootNode)
 
         // Filter out [AccessibilityElement] in [accessibilityElementsMap] that are not present in the tree anymore
         accessibilityElementsMap.keys.retainAll {
             val isPresent = it in presentIds
 
             if (!isPresent) {
-                debugLogger?.log("$it removed")
+                accessibilityDebugLogger?.log("$it removed")
                 checkNotNull(accessibilityElementsMap[it]).dispose()
             }
 
@@ -927,7 +969,7 @@ internal class AccessibilityMediator(
 
         root.element = traverseSemanticsTree(rootSemanticsNode)
 
-        debugLogger?.let {
+        accessibilityDebugLogger?.let {
             debugTraverse(it, view)
         }
 
@@ -1128,32 +1170,12 @@ private fun debugContainmentChain(accessibilityObject: Any): String {
 private val SemanticsNode.semanticsKey get() = AccessibilityElementKey.Semantics(id)
 private val SemanticsNode.containerKey get() = AccessibilityElementKey.Container(id)
 
-/**
- * Sort the elements in their visual order using their bounds:
- * - from top to bottom,
- * - from left to right or from right to left, depending on language direction
- *
- * The sort is needed because [SemanticsNode.replacedChildren] order doesn't match the
- * expected order of the children in the accessibility tree.
- *
- * TODO: investigate if it's a bug, or some assumptions about the order are wrong.
- */
-private fun List<SemanticsNode>.sortedByAccessibilityOrder(isRTL: Boolean): List<SemanticsNode> {
-    return sortedWith { lhs, rhs ->
-        val result = lhs.boundsInWindow.topLeft.y.compareTo(rhs.boundsInWindow.topLeft.y)
-
-        if (result == 0) {
-            lhs.boundsInWindow.topLeft.x.compareTo(rhs.boundsInWindow.topLeft.x).let {
-                if (isRTL) -it else it
-            }
-        } else {
-            result
-        }
-    }
-}
-
 /**
  * Returns true if corresponding [LayoutNode] is placed and attached, false otherwise.
  */
 private val SemanticsNode.isValid: Boolean
     get() = layoutNode.isPlaced && layoutNode.isAttached
+
+private val SemanticsNode.isTraversalGroup: Boolean
+    get() = unmergedConfig.getOrElse(SemanticsProperties.IsTraversalGroup) { false }
+