diff --git a/spec/api-browser-window-spec.js b/spec/api-browser-window-spec.js
index a07f301c8b..4e13a14fca 100644
--- a/spec/api-browser-window-spec.js
+++ b/spec/api-browser-window-spec.js
@@ -513,6 +513,18 @@ describe('BrowserWindow module', function () {
       w.setAlwaysOnTop(true)
       assert.equal(w.isAlwaysOnTop(), true)
     })
+
+    it('raises an error when relativeLevel is out of bounds', function() {
+      if (process.platform !== 'darwin') return;
+
+      assert.throws(function() {
+        w.setAlwaysOnTop(true, '', -1)
+      })
+
+      assert.throws(function() {
+        w.setAlwaysOnTop(true, '', 2147483632)
+      })
+    })
   })
 
   describe('BrowserWindow.setAutoHideCursor(autoHide)', () => {