Update 'file' version to fix json<->html detection for reviewe… (#11690)

* Update 'file' version to fix json<->html detection for reviewer api.

Fixes #11534

* Add fixture file

* Rollback to xenial, install libmagic-dev manually

* Add libmagic1 dependency too

* test amd64

* Add required libmagic-mgc dependency

* Add explaining comment

* Try out using ubuntu disco sources for libmagic installation

* Force installation from the disco repo.

.travis.yml

@@ -52,6 +52,11 @@ before_install:
   - mysql -e 'create database olympia;'
   - export GOPATH=$HOME/go
   - export PATH=$HOME/usr/local/go/bin:$GOPATH/bin:$PATH
+  # Unfortunately we require the most recent libmagic-dev version to make use of more recent
+  # JSON detection features which we require for the code-manager related APIs
+  - echo "deb http://archive.ubuntu.com/ubuntu disco main" | sudo tee -a /etc/apt/sources.list
+  - sudo apt-get update -qq
+  - sudo apt-get install -t disco libmagic-dev libmagic1 libmagic-mgc
 
 install:
   - nvm current

Dockerfile

@@ -13,9 +13,9 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 RUN cat /etc/pki/gpg/GPG-KEY-nodesource | apt-key add -
 ADD docker/debian-stretch-nodesource-repo /etc/apt/sources.list.d/nodesource.list
-ADD docker/debian-stretch-backports-repo /etc/apt/sources.list.d/backports.list
+ADD docker/debian-buster-testing-repo /etc/apt/sources.list.d/testing.list
 
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get -t stretch install -y \
         # General (dev-) dependencies
         bash-completion \
         build-essential \
@@ -54,6 +54,13 @@ RUN mkdir -p /usr/local/share/GeoIP \
  && gunzip -c /tmp/GeoLite2-Country.mmdb.gz > /usr/local/share/GeoIP/GeoLite2-Country.mmdb \
  && rm -f /tmp/GeoLite2-Country.mmdb.gz
 
+RUN apt-get update && apt-get -t buster install -y \
+        # For an up-to-date `file` and `libmagic-dev` library for better file
+        # detection.
+       file \
+       libmagic-dev \
+    && rm -rf /var/lib/apt/lists/*
+
 # Compile required locale
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 

docker/debian-buster-testing-repo

@@ -0,0 +1 @@
+deb http://deb.debian.org/debian buster main

docker/debian-stretch-backports-repo

@@ -1 +0,0 @@
-deb http://deb.debian.org/debian stretch-backports main

src/olympia/files/fixtures/files/file_viewer_filetypes/html-containing.json

@@ -0,0 +1,3 @@
+{
+   "bar": "<table>"
+}

src/olympia/reviewers/tests/test_serializers.py

@@ -471,6 +471,12 @@ class TestFileEntriesDiffSerializer(TestCase):
         (MagicMock(type='blob'), 'bmp-v4.bmp', 'image', 'image/bmp'),
         (MagicMock(type='blob'), 'bmp-v5.bmp', 'image', 'image/bmp'),
         (MagicMock(type='blob'), 'bmp-os2-v1.bmp', 'image', 'image/bmp'),
+        # This is testing that a tag listed at
+        # https://github.com/file/file/blob/master/magic/Magdir/sgml#L57
+        # doesn't lead to the file being detected as HTML, which was fixed
+        # in most recent libmagic versions.
+        (MagicMock(type='blob'), 'html-containing.json', 'text',
+                                 'application/json'),
     ]
 )
 def test_file_entries_serializer_category_type(