Merge PR #2

* Update README.md

* update README.md

* add figure

* update README.md

* add model directory

model/BertModel.py

@@ -0,0 +1,93 @@
+from transformers import BertPreTrainedModel, BertModel
+import torch
+import torch.nn as nn
+
+class BertForSequenceClassification(BertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+    """
+
+    def __init__(self, config):
+        super(BertForSequenceClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.bert = BertModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
+
+        self.init_weights()
+
+    def expand_class_head(self, c_count):
+        self.c_count = c_count
+        if c_count > 1:
+            for i in range(1, c_count + 1):
+                setattr(self, "classifier_{}".format(i), nn.Linear(self.config.hidden_size, self.num_labels))
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        reduction="mean", is_gold=True,
+
+    ):
+
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+        )
+
+        pooled_output = outputs[1]
+        outputs = (pooled_output.detach(), )
+        pooled_output = self.dropout(pooled_output)
+
+        if self.c_count == 1 or is_gold:
+            logits = self.classifier(pooled_output)  # (N, C)
+        else:
+            logits = []
+            for i in range(1, self.c_count+1):
+                logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
+            logits = torch.cat(logits, dim=1)
+
+        outputs = (logits, ) + outputs
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss(reduction=reduction)
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)

model/CNNModel.py

@@ -0,0 +1,98 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+import warnings
+from transformers import DistilBertModel
+class CNN_Text(nn.Module):
+
+    def __init__(self, args):
+        super(CNN_Text, self).__init__()
+        self.args = args
+
+        # V = args.embed_num
+        # D = args.embed_dim
+        C = args.num_labels
+        Ci = 1
+        Co = args.kernel_num
+        Ks = args.kernel_sizes
+
+        # self.embed = nn.Embedding(V, D)
+        # self.convs1 = [nn.Conv2d(Ci, Co, (K, D)) for K in Ks]
+
+        '''
+        self.conv13 = nn.Conv2d(Ci, Co, (3, D))
+        self.conv14 = nn.Conv2d(Ci, Co, (4, D))
+        self.conv15 = nn.Conv2d(Ci, Co, (5, D))
+        '''
+        self.dropout = nn.Dropout(args.dropout)
+        self.classifier = nn.Linear(len(Ks) * Co, C)
+        self.Ks = Ks
+        self.Co = Co
+        self.C = C
+        self.c_count = 1
+        self.Ci= Ci
+        self.embed_shape = [30522, 768]
+        self.embed = nn.Embedding(self.embed_shape[0], self.embed_shape[1])
+        self.convs1 = nn.ModuleList(
+            [nn.Conv2d(self.Ci, self.Co, (K, self.embed_shape[1]))
+             for K in self.Ks])
+        # self.from_pretrained("distilbert-base-uncased")
+
+    def from_pretrained(self, model_name_or_path):
+        distilbert = DistilBertModel.from_pretrained(model_name_or_path)
+        state_dict = distilbert.state_dict()
+        embed_weight = state_dict['embeddings.word_embeddings.weight']
+        self.embed.from_pretrained(embed_weight)
+
+    def conv_and_pool(self, x, conv):
+        x = F.relu(conv(x)).squeeze(3)  # (N, Co, W)
+        x = F.max_pool1d(x, x.size(2)).squeeze(2)
+        return x
+
+    def expand_class_head(self, c_count):
+        self.c_count = c_count
+        if c_count > 1:
+            for i in range(1, c_count + 1):
+                setattr(self, "classifier_{}".format(i), nn.Linear(len(self.Ks) * self.Co, self.C))
+
+    def forward(self, input_ids, attention_mask=None, labels=None, reduction="mean", is_gold=True):
+        input_ids = self.embed(input_ids)  # (N, W, D)
+
+        # if self.args.static:
+        #     input_ids = Variable(input_ids)
+
+        input_ids = input_ids.unsqueeze(1)  # (N, Ci, W, D)
+
+        input_ids = [F.relu(conv(input_ids)).squeeze(3) for conv in self.convs1]  # [(N, Co, W), ...]*len(Ks)
+
+        input_ids = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in input_ids]  # [(N, Co), ...]*len(Ks)
+
+        hidden_state = torch.cat(input_ids, 1)
+
+        '''
+        x1 = self.conv_and_pool(x,self.conv13) #(N,Co)
+        x2 = self.conv_and_pool(x,self.conv14) #(N,Co)
+        x3 = self.conv_and_pool(x,self.conv15) #(N,Co)
+        x = torch.cat((x1, x2, x3), 1) # (N,len(Ks)*Co)
+        '''
+
+        outputs = (hidden_state.detach(), )
+        hidden_state = self.dropout(hidden_state)  # (N, len(Ks)*Co)
+
+        if self.c_count == 1 or is_gold:
+            logits = self.classifier(hidden_state)  # (N, C)
+        else:
+            logits = []
+            for i in range(1, self.c_count+1):
+                logits.append(self.__dict__['_modules']["classifier_{}".format(i)](hidden_state))
+            logits = torch.cat(logits, dim=1)
+
+        outputs = (logits, ) + outputs
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss(reduction=reduction)
+            loss = loss_fct(logits.view(-1, self.C), labels.view(-1))
+            outputs = (loss,) + outputs
+
+
+        return outputs

model/DistilBertModel.py

@@ -0,0 +1,51 @@
+from transformers import DistilBertPreTrainedModel, DistilBertModel
+import torch.nn as nn
+import torch
+class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
+    def __init__(self, config):
+        super(DistilBertForSequenceClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.distilbert = DistilBertModel(config)
+        self.pre_classifier = nn.Linear(config.dim, config.dim)
+        self.classifier = nn.Linear(config.dim, config.num_labels)
+        self.dropout = nn.Dropout(config.seq_classif_dropout)
+        self.config = config
+        self.c_count = 1
+        self.init_weights()
+
+    def expand_class_head(self, c_count):
+        self.c_count = c_count
+        if c_count > 1:
+            for i in range(1, c_count+1):
+                setattr(self, "classifier_{}".format(i), nn.Linear(self.config.dim, self.config.num_labels))
+
+    def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, labels=None,
+                reduction="mean", is_gold=True):
+        distilbert_output = self.distilbert(
+            input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, inputs_embeds=inputs_embeds
+        )
+        hidden_state = distilbert_output[0]  # (bs, seq_len, dim)
+        pooled_output = hidden_state[:, 0]  # (bs, dim)
+        pooled_output = self.pre_classifier(pooled_output)  # (bs, dim)
+        pooled_output = nn.ReLU()(pooled_output)  # (bs, dim)
+        pooled_output = self.dropout(pooled_output)  # (bs, dim)
+        if self.c_count == 1 or is_gold:
+            logits = self.classifier(pooled_output)  # (bs, dim)
+        else:
+            logits = []
+            for i in range(1, self.c_count+1):
+                logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
+            logits = torch.cat(logits, dim=1)
+
+        outputs = (logits,) + distilbert_output[1:]
+        if labels is not None:
+            if self.num_labels == 1:
+                loss_fct = nn.MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = nn.CrossEntropyLoss(reduction=reduction)
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)

model/FullWeightModel.py

@@ -0,0 +1,68 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+from transformers import DistilBertModel
+
+class FullWeightModel(nn.Module):
+    def __init__(self, n_groups, hidden_size):
+        super(FullWeightModel, self).__init__()
+        self.n_groups = 1 if n_groups == 0 else n_groups
+        #self.pesu_group_weight = nn.Parameter(torch.randn(1, self.n_groups), requires_grad=True)
+        self.embed_shape = [30522, 768]
+        # self.ins_embed = nn.Embedding(self.embed_shape[0], self.embed_shape[1])
+        self.cls_emb = 256 #self.embed_shape[1]
+        h_dim = 768
+        self.y_embed = nn.Embedding(2, self.cls_emb) 
+
+        #self.ins_weight = nn.Linear(hidden_size+self.cls_emb, 1)
+
+        self.ins_weight = nn.Sequential(
+            nn.Linear(hidden_size+self.cls_emb, h_dim),
+            nn.ReLU(), #Tanh(),
+            nn.Linear(h_dim, h_dim),
+            nn.ReLU(), #Tanh(),
+            nn.Linear(h_dim, 1)
+        )
+        
+    def reset_groups(self, new_groups):
+        self.n_groups = new_groups
+
+    def forward(self, x_feature, y_weak, item_loss=None):
+        '''
+        
+        Args:
+            item_loss: shape is [batchsize * 3, ].
+            e.g [item1_weak1_loss,
+                item1_weak2_loss,
+                item1_weak3_loss,
+                ....
+            ]
+            iw:
+
+        Returns:
+
+        '''
+        # detach the feature
+        x_feature = x_feature.detach()
+        feature_dim = x_feature.shape[-1]
+        x_feature = x_feature.repeat(1, self.n_groups).view(-1, feature_dim)
+        y_emb = self.y_embed(y_weak).view(-1, self.cls_emb)
+        #ATTENTION: weight depends on the pair of feature and weak label instead of the source.
+        #final_weight = F.softmax(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1)), dim=0).squeeze()
+        #return (final_weight * item_loss).sum() ,final_weight
+
+        # sigmoid with mean
+        final_weight = torch.sigmoid(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1))).squeeze()
+        if item_loss is None:
+            return final_weight
+        else:
+            return (final_weight * item_loss).mean(), final_weight
+    
+        
+        
+
+        #final_weight = F.relu(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1))).squeeze()
+        #return (final_weight * item_loss).mean()
+
+

model/GroupWeightModel.py

@@ -0,0 +1,35 @@
+import  torch.nn as nn
+import torch
+import math
+
+class GroupWeightModel(nn.Module):
+    def __init__(self, n_groups):
+        super(GroupWeightModel, self).__init__()
+        self.n_groups = n_groups
+        self.pesu_group_weight = nn.Parameter(torch.randn(1, self.n_groups), requires_grad=True)
+        self.init_weight()
+
+    def init_weight(self):
+        stdv = 1. / math.sqrt(self.pesu_group_weight.size(1))
+        self.pesu_group_weight.data.uniform_(-stdv, stdv)
+
+    def forward(self, item_loss, iw):
+        '''
+        
+        Args:
+            item_loss: shape is [batchsize * 3, ].
+            e.g [item1_weak1_loss,
+                item1_weak2_loss,
+                item1_weak3_loss,
+                ....
+            ]
+            iw:
+
+        Returns:
+
+        '''
+        group_weight = torch.sigmoid(self.pesu_group_weight)
+        final_weight = torch.matmul(iw.view(-1, 1), group_weight)
+        
+        return (final_weight * (item_loss.view(final_weight.shape))).sum()
+

model/RoBertModel.py

@@ -0,0 +1,89 @@
+from transformers import BertPreTrainedModel, RobertaConfig, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaModel
+import torch
+import torch.nn as nn
+
+class RobertaClassificationHead(nn.Module):
+    """Head for sentence-level classification tasks."""
+
+    def __init__(self, config):
+        super(RobertaClassificationHead, self).__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features[:, 1, :]  # take <s> token (equiv. to [CLS])
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+    
+class RobertaForSequenceClassification(BertPreTrainedModel):
+    config_class = RobertaConfig
+    pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "roberta"
+
+    def __init__(self, config):
+        super(RobertaForSequenceClassification, self).__init__(config)
+        self.num_labels = 2
+
+        self.roberta = RobertaModel(config)
+        # self.classifier = RobertaClassificationHead(config)
+        self.pre_classifier = nn.Linear(config.hidden_size, config.hidden_size)
+        self.classifier = nn.Linear(config.hidden_size, self.num_labels)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.config = config
+        self.c_count = 1
+        
+    def expand_class_head(self, c_count):
+        self.c_count = c_count
+        if c_count > 1:
+            for i in range(1, c_count+1):
+                setattr(self, "classifier_{}".format(i), nn.Linear(self.config.hidden_size, self.num_labels))
+                
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        reduction="mean", is_gold=True
+    ):
+        outputs = self.roberta(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+        )
+        # sequence_output = outputs[0]
+        # logits = self.classifier(sequence_output)
+        hidden_state = outputs[0]
+        pooled_output = hidden_state[:, 1]  # (bs, dim)
+        outputs = (pooled_output.detach(),)
+        pooled_output = self.pre_classifier(pooled_output)  # (bs, dim)
+        pooled_output = nn.ReLU()(pooled_output)  # (bs, dim)
+        pooled_output = self.dropout(pooled_output)  # (bs, dim)
+
+        if self.c_count == 1 or is_gold:
+            logits = self.classifier(pooled_output)  # (N, C)
+            # logits = self.classifier(sequence_output)  # (N, C)
+        else:
+            logits = []
+            for i in range(1, self.c_count+1):
+                logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
+            logits = torch.cat(logits, dim=1)
+
+        outputs = (logits, ) + outputs
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss(reduction=reduction)
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits

model/__init__.py

@@ -0,0 +1,6 @@
+from .RoBertModel import RobertaForSequenceClassification
+from .CNNModel import CNN_Text
+from .GroupWeightModel import  GroupWeightModel
+from .DistilBertModel import DistilBertForSequenceClassification
+from .FullWeightModel import  FullWeightModel
+from .BertModel import  BertForSequenceClassification