Add license header and update NOTICE

Code/BertSequence.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import argparse
 import logging
 import os

Code/BertToken.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import argparse
 import logging
 import os

Code/run_squad.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
 # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
 # Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
 #

Code/run_xnli.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
 # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
 # Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
 #

Code/train_nli.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 TASK=${1:-NLI_EN_HI}
 MODEL=${2:-bert-base-multilingual-cased}

Code/train_qa.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 TASK=${1:-QA_EN_HI}
 MODEL=${2:-bert-base-multilingual-cased}

Code/train_sentence.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 TASK=${1:-Sentiment_EN_ES}
 MODEL=${2:-bert-base-multilingual-cased}

Code/train_token.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 TASK=${1:-POS_EN_ES}
 MODEL=${2:-bert-base-multilingual-cased}

Data/Preprocess_Scripts/preprocess_drqa.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import json
 import argparse

Data/Preprocess_Scripts/preprocess_lid_en_es.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import subprocess
 import shlex

Data/Preprocess_Scripts/preprocess_lid_en_hi.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import shutil
 import argparse

Data/Preprocess_Scripts/preprocess_ner_en_es.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import subprocess
 import shlex

Data/Preprocess_Scripts/preprocess_ner_en_hi.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import csv
 import os
 import argparse

Data/Preprocess_Scripts/preprocess_nli_en_hi.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import shutil
 import argparse

Data/Preprocess_Scripts/preprocess_pos_en_es.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import argparse
 

Data/Preprocess_Scripts/preprocess_pos_en_hi_fg.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import shutil
 import argparse

Data/Preprocess_Scripts/preprocess_pos_en_hi_ud.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import sys
 import os
 from subprocess import call

Data/Preprocess_Scripts/preprocess_qa.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 ORIGINAL_DATA_DIR=$PWD/Data/Original_Data
 PROCESSED_DIR=$PWD/Data/Processed_Data
 PREPROCESS_DIR=$PWD/Data/Preprocess_Scripts

Data/Preprocess_Scripts/preprocess_qa_en_hi.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import json
 import argparse

Data/Preprocess_Scripts/preprocess_sent_en_es.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import tweepy, sys, os, re, time, math
 import operator
 import unidecode

Data/Preprocess_Scripts/preprocess_sent_en_hi.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import shutil
 import argparse

NOTICE

@@ -1 +1,2 @@
-This repo uses open source Apache licensed code from https://github.com/huggingface/transformers
+This repo makes use of https://github.com/huggingface/transformers, which was originally distributed with an Apache license.
+Training scripts are borrowed from this repo and modified accordingly for the tasks in this benchmark.

azure_ml/submit.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import os
 import textwrap
 import ruamel.yaml as yaml

download_data.sh

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 SUBSCRIPTION_KEY=${1:-""}
 ORIGINAL_DIR="$REPO/Data/Original_Data"

train.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/bash
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 REPO=$PWD
 MODEL=${1:-bert-base-multilingual-cased}
 MODEL_TYPE=${2:-bert}

transliterator.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. Licensed under the MIT license.
+
 import requests
 import sys
 import os