bug 1061170 - constrain the reflow-state's available inline size for an orthogonal flow. r=smontagu

layout/base/nsPresShell.cpp

@@ -8776,6 +8776,7 @@ PresShell::DoReflow(nsIFrame* target, bool aInterruptible)
   LogicalSize reflowSize(wm, size.ISize(wm), NS_UNCONSTRAINEDSIZE);
   nsHTMLReflowState reflowState(mPresContext, target, rcx, reflowSize,
                                 nsHTMLReflowState::CALLER_WILL_INIT);
+  reflowState.mOrthogonalLimit = size.BSize(wm);
 
   if (rootFrame == target) {
     reflowState.Init(mPresContext);

layout/generic/WritingModes.h

@@ -283,6 +283,14 @@ public:
     return mWritingMode != aOther.mWritingMode;
   }
 
+  /**
+   * Check whether two modes are orthogonal to each other.
+   */
+  bool IsOrthogonalTo(const WritingMode& aOther) const
+  {
+    return IsVertical() != aOther.IsVertical();
+  }
+
 private:
   friend class LogicalPoint;
   friend class LogicalSize;

layout/generic/nsHTMLReflowState.cpp

@@ -60,6 +60,7 @@ nsHTMLReflowState::nsHTMLReflowState(nsPresContext*       aPresContext,
                                      uint32_t             aFlags)
   : nsCSSOffsetState(aFrame, aRenderingContext)
   , mBlockDelta(0)
+  , mOrthogonalLimit(NS_UNCONSTRAINEDSIZE)
   , mReflowDepth(0)
 {
   NS_PRECONDITION(aRenderingContext, "no rendering context");
@@ -165,6 +166,7 @@ nsHTMLReflowState::nsHTMLReflowState(nsPresContext*           aPresContext,
                                      uint32_t                 aFlags)
   : nsCSSOffsetState(aFrame, aParentReflowState.rendContext)
   , mBlockDelta(0)
+  , mOrthogonalLimit(NS_UNCONSTRAINEDSIZE)
   , mReflowDepth(aParentReflowState.mReflowDepth + 1)
   , mFlags(aParentReflowState.mFlags)
 {
@@ -329,6 +331,19 @@ nsHTMLReflowState::Init(nsPresContext* aPresContext,
                         const nsMargin* aBorder,
                         const nsMargin* aPadding)
 {
+  if (AvailableISize() == NS_UNCONSTRAINEDSIZE) {
+    // Look up the parent chain for an orthogonal inline limit,
+    // and reset AvailableISize() if found.
+    for (const nsHTMLReflowState *parent = parentReflowState;
+         parent != nullptr; parent = parent->parentReflowState) {
+      if (parent->GetWritingMode().IsOrthogonalTo(mWritingMode) &&
+          parent->mOrthogonalLimit != NS_UNCONSTRAINEDSIZE) {
+        AvailableISize() = parent->mOrthogonalLimit;
+        break;
+      }
+    }
+  }
+
   NS_WARN_IF_FALSE(AvailableISize() != NS_UNCONSTRAINEDSIZE,
                    "have unconstrained inline-size; this should only result from "
                    "very large sizes, not attempts at intrinsic inline-size "

layout/generic/nsHTMLReflowState.h

@@ -266,6 +266,15 @@ struct nsHTMLReflowState : public nsCSSOffsetState {
   // This takes on an arbitrary value the first time a block is reflowed
   nscoord mBlockDelta;
 
+  // If an nsHTMLReflowState finds itself initialized with an unconstrained
+  // inline-size, it will look up its parentReflowState chain for a state
+  // with an orthogonal writing mode and a non-NS_UNCONSTRAINEDSIZE value for
+  // orthogonal limit; when it finds such a reflow-state, it will use its
+  // orthogonal-limit value to constrain inline-size.
+  // This is initialized to NS_UNCONSTRAINEDSIZE (so it will be ignored),
+  // but reset to a suitable value for the reflow root by nsPresShell.
+  nscoord mOrthogonalLimit;
+
   // Accessors for the private fields below. Forcing all callers to use these
   // will allow us to introduce logical-coordinate versions and gradually
   // change clients from physical to logical as needed; and potentially switch