Improve nasm support in CMake build.

Fail at configure time when required features are not
present. Currently requires only necessary x86 object
formats and the presence of the -Ox opt mode arg.

BUG=aomedia:76

Change-Id: Idc372e8ed121a600e87c46c0d29d5322cfceaec8

build/cmake/aom_configure.cmake

@@ -90,10 +90,9 @@ if (NOT "${AOM_SUPPORTED_CPU_TARGETS}" MATCHES "${AOM_TARGET_CPU}")
 endif ()
 
 if ("${AOM_TARGET_CPU}" STREQUAL "x86" OR "${AOM_TARGET_CPU}" STREQUAL "x86_64")
-  # TODO(tomfinegan): Support nasm at least as well as the existing build
-  # system.
   if (ENABLE_NASM)
     find_program(AS_EXECUTABLE nasm $ENV{NASM_PATH})
+    test_nasm()
     set(AOM_AS_FLAGS ${AOM_AS_FLAGS} -Ox)
   else ()
     find_program(AS_EXECUTABLE yasm $ENV{YASM_PATH})

build/cmake/aom_optimization.cmake

@@ -203,4 +203,43 @@ function (add_gas_asm_library lib_name asm_sources dependent_target)
   set(${asm_sources} ${${asm_sources}} PARENT_SCOPE)
 endfunction ()
 
+# Terminates generation if nasm found in PATH does not meet requirements.
+# Currently checks only for presence of required object formats and support for
+# the -Ox argument (multipass optimization).
+function (test_nasm)
+  execute_process(COMMAND ${AS_EXECUTABLE} -hf
+                  OUTPUT_VARIABLE nasm_helptext)
+
+  if (NOT "${nasm_helptext}" MATCHES "-Ox")
+    message(FATAL_ERROR
+            "Unsupported nasm: multipass optimization not supported.")
+  endif ()
+
+  if ("${AOM_TARGET_CPU}" STREQUAL "x86")
+    if ("${AOM_TARGET_SYSTEM}" STREQUAL "Darwin")
+      if (NOT "${nasm_helptext}" MATCHES "macho32")
+        message(FATAL_ERROR
+                "Unsupported nasm: macho32 object format not supported.")
+      endif ()
+    elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Linux")
+      if (NOT "${nasm_helptext}" MATCHES "elf32")
+        message(FATAL_ERROR
+                "Unsupported nasm: elf32 object format not supported.")
+      endif ()
+    endif ()
+  else ()
+    if ("${AOM_TARGET_SYSTEM}" STREQUAL "Darwin")
+      if (NOT "${nasm_helptext}" MATCHES "macho64")
+        message(FATAL_ERROR
+                "Unsupported nasm: macho64 object format not supported.")
+      endif ()
+    elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Linux")
+      if (NOT "${nasm_helptext}" MATCHES "elf64")
+        message(FATAL_ERROR
+                "Unsupported nasm: elf64 object format not supported.")
+      endif ()
+    endif ()
+  endif ()
+endfunction ()
+
 endif ()  # AOM_BUILD_CMAKE_AOM_OPTIMIZATION_CMAKE_