зеркало из https://github.com/microsoft/MWSS.git
Merge PR #2
* Update README.md * update README.md * add figure * update README.md * add model directory
This commit is contained in:
Yichuan LI
GitHub
Родитель
74cd790d04
Коммит
744cbb9dc0
|
|
@ -0,0 +1,93 @@
|
|||
from transformers import BertPreTrainedModel, BertModel
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
class BertForSequenceClassification(BertPreTrainedModel):
|
||||
r"""
|
||||
**labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels for computing the sequence classification/regression loss.
|
||||
Indices should be in ``[0, ..., config.num_labels - 1]``.
|
||||
If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
|
||||
If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
|
||||
|
||||
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
|
||||
**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
|
||||
Classification (or regression if config.num_labels==1) loss.
|
||||
**logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
|
||||
Classification (or regression if config.num_labels==1) scores (before SoftMax).
|
||||
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
|
||||
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
|
||||
of shape ``(batch_size, sequence_length, hidden_size)``:
|
||||
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
|
||||
**attentions**: (`optional`, returned when ``config.output_attentions=True``)
|
||||
list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
|
||||
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
|
||||
|
||||
Examples::
|
||||
|
||||
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
|
||||
model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
|
||||
input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1
|
||||
labels = torch.tensor([1]).unsqueeze(0) # Batch size 1
|
||||
outputs = model(input_ids, labels=labels)
|
||||
loss, logits = outputs[:2]
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super(BertForSequenceClassification, self).__init__(config)
|
||||
self.num_labels = config.num_labels
|
||||
|
||||
self.bert = BertModel(config)
|
||||
self.dropout = nn.Dropout(config.hidden_dropout_prob)
|
||||
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
|
||||
|
||||
self.init_weights()
|
||||
|
||||
def expand_class_head(self, c_count):
|
||||
self.c_count = c_count
|
||||
if c_count > 1:
|
||||
for i in range(1, c_count + 1):
|
||||
setattr(self, "classifier_{}".format(i), nn.Linear(self.config.hidden_size, self.num_labels))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
labels=None,
|
||||
reduction="mean", is_gold=True,
|
||||
|
||||
):
|
||||
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
pooled_output = outputs[1]
|
||||
outputs = (pooled_output.detach(), )
|
||||
pooled_output = self.dropout(pooled_output)
|
||||
|
||||
if self.c_count == 1 or is_gold:
|
||||
logits = self.classifier(pooled_output) # (N, C)
|
||||
else:
|
||||
logits = []
|
||||
for i in range(1, self.c_count+1):
|
||||
logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
|
||||
logits = torch.cat(logits, dim=1)
|
||||
|
||||
outputs = (logits, ) + outputs
|
||||
if labels is not None:
|
||||
loss_fct = nn.CrossEntropyLoss(reduction=reduction)
|
||||
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
||||
outputs = (loss,) + outputs
|
||||
|
||||
return outputs # (loss), logits, (hidden_states), (attentions)
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
import warnings
|
||||
from transformers import DistilBertModel
|
||||
class CNN_Text(nn.Module):
|
||||
|
||||
def __init__(self, args):
|
||||
super(CNN_Text, self).__init__()
|
||||
self.args = args
|
||||
|
||||
# V = args.embed_num
|
||||
# D = args.embed_dim
|
||||
C = args.num_labels
|
||||
Ci = 1
|
||||
Co = args.kernel_num
|
||||
Ks = args.kernel_sizes
|
||||
|
||||
# self.embed = nn.Embedding(V, D)
|
||||
# self.convs1 = [nn.Conv2d(Ci, Co, (K, D)) for K in Ks]
|
||||
|
||||
'''
|
||||
self.conv13 = nn.Conv2d(Ci, Co, (3, D))
|
||||
self.conv14 = nn.Conv2d(Ci, Co, (4, D))
|
||||
self.conv15 = nn.Conv2d(Ci, Co, (5, D))
|
||||
'''
|
||||
self.dropout = nn.Dropout(args.dropout)
|
||||
self.classifier = nn.Linear(len(Ks) * Co, C)
|
||||
self.Ks = Ks
|
||||
self.Co = Co
|
||||
self.C = C
|
||||
self.c_count = 1
|
||||
self.Ci= Ci
|
||||
self.embed_shape = [30522, 768]
|
||||
self.embed = nn.Embedding(self.embed_shape[0], self.embed_shape[1])
|
||||
self.convs1 = nn.ModuleList(
|
||||
[nn.Conv2d(self.Ci, self.Co, (K, self.embed_shape[1]))
|
||||
for K in self.Ks])
|
||||
# self.from_pretrained("distilbert-base-uncased")
|
||||
|
||||
def from_pretrained(self, model_name_or_path):
|
||||
distilbert = DistilBertModel.from_pretrained(model_name_or_path)
|
||||
state_dict = distilbert.state_dict()
|
||||
embed_weight = state_dict['embeddings.word_embeddings.weight']
|
||||
self.embed.from_pretrained(embed_weight)
|
||||
|
||||
def conv_and_pool(self, x, conv):
|
||||
x = F.relu(conv(x)).squeeze(3) # (N, Co, W)
|
||||
x = F.max_pool1d(x, x.size(2)).squeeze(2)
|
||||
return x
|
||||
|
||||
def expand_class_head(self, c_count):
|
||||
self.c_count = c_count
|
||||
if c_count > 1:
|
||||
for i in range(1, c_count + 1):
|
||||
setattr(self, "classifier_{}".format(i), nn.Linear(len(self.Ks) * self.Co, self.C))
|
||||
|
||||
def forward(self, input_ids, attention_mask=None, labels=None, reduction="mean", is_gold=True):
|
||||
input_ids = self.embed(input_ids) # (N, W, D)
|
||||
|
||||
# if self.args.static:
|
||||
# input_ids = Variable(input_ids)
|
||||
|
||||
input_ids = input_ids.unsqueeze(1) # (N, Ci, W, D)
|
||||
|
||||
input_ids = [F.relu(conv(input_ids)).squeeze(3) for conv in self.convs1] # [(N, Co, W), ...]*len(Ks)
|
||||
|
||||
input_ids = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in input_ids] # [(N, Co), ...]*len(Ks)
|
||||
|
||||
hidden_state = torch.cat(input_ids, 1)
|
||||
|
||||
'''
|
||||
x1 = self.conv_and_pool(x,self.conv13) #(N,Co)
|
||||
x2 = self.conv_and_pool(x,self.conv14) #(N,Co)
|
||||
x3 = self.conv_and_pool(x,self.conv15) #(N,Co)
|
||||
x = torch.cat((x1, x2, x3), 1) # (N,len(Ks)*Co)
|
||||
'''
|
||||
|
||||
outputs = (hidden_state.detach(), )
|
||||
hidden_state = self.dropout(hidden_state) # (N, len(Ks)*Co)
|
||||
|
||||
if self.c_count == 1 or is_gold:
|
||||
logits = self.classifier(hidden_state) # (N, C)
|
||||
else:
|
||||
logits = []
|
||||
for i in range(1, self.c_count+1):
|
||||
logits.append(self.__dict__['_modules']["classifier_{}".format(i)](hidden_state))
|
||||
logits = torch.cat(logits, dim=1)
|
||||
|
||||
outputs = (logits, ) + outputs
|
||||
if labels is not None:
|
||||
loss_fct = nn.CrossEntropyLoss(reduction=reduction)
|
||||
loss = loss_fct(logits.view(-1, self.C), labels.view(-1))
|
||||
outputs = (loss,) + outputs
|
||||
|
||||
|
||||
return outputs
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
from transformers import DistilBertPreTrainedModel, DistilBertModel
|
||||
import torch.nn as nn
|
||||
import torch
|
||||
class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
|
||||
def __init__(self, config):
|
||||
super(DistilBertForSequenceClassification, self).__init__(config)
|
||||
self.num_labels = config.num_labels
|
||||
|
||||
self.distilbert = DistilBertModel(config)
|
||||
self.pre_classifier = nn.Linear(config.dim, config.dim)
|
||||
self.classifier = nn.Linear(config.dim, config.num_labels)
|
||||
self.dropout = nn.Dropout(config.seq_classif_dropout)
|
||||
self.config = config
|
||||
self.c_count = 1
|
||||
self.init_weights()
|
||||
|
||||
def expand_class_head(self, c_count):
|
||||
self.c_count = c_count
|
||||
if c_count > 1:
|
||||
for i in range(1, c_count+1):
|
||||
setattr(self, "classifier_{}".format(i), nn.Linear(self.config.dim, self.config.num_labels))
|
||||
|
||||
def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, labels=None,
|
||||
reduction="mean", is_gold=True):
|
||||
distilbert_output = self.distilbert(
|
||||
input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, inputs_embeds=inputs_embeds
|
||||
)
|
||||
hidden_state = distilbert_output[0] # (bs, seq_len, dim)
|
||||
pooled_output = hidden_state[:, 0] # (bs, dim)
|
||||
pooled_output = self.pre_classifier(pooled_output) # (bs, dim)
|
||||
pooled_output = nn.ReLU()(pooled_output) # (bs, dim)
|
||||
pooled_output = self.dropout(pooled_output) # (bs, dim)
|
||||
if self.c_count == 1 or is_gold:
|
||||
logits = self.classifier(pooled_output) # (bs, dim)
|
||||
else:
|
||||
logits = []
|
||||
for i in range(1, self.c_count+1):
|
||||
logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
|
||||
logits = torch.cat(logits, dim=1)
|
||||
|
||||
outputs = (logits,) + distilbert_output[1:]
|
||||
if labels is not None:
|
||||
if self.num_labels == 1:
|
||||
loss_fct = nn.MSELoss()
|
||||
loss = loss_fct(logits.view(-1), labels.view(-1))
|
||||
else:
|
||||
loss_fct = nn.CrossEntropyLoss(reduction=reduction)
|
||||
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
||||
outputs = (loss,) + outputs
|
||||
|
||||
return outputs # (loss), logits, (hidden_states), (attentions)
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import math
|
||||
from transformers import DistilBertModel
|
||||
|
||||
class FullWeightModel(nn.Module):
|
||||
def __init__(self, n_groups, hidden_size):
|
||||
super(FullWeightModel, self).__init__()
|
||||
self.n_groups = 1 if n_groups == 0 else n_groups
|
||||
#self.pesu_group_weight = nn.Parameter(torch.randn(1, self.n_groups), requires_grad=True)
|
||||
self.embed_shape = [30522, 768]
|
||||
# self.ins_embed = nn.Embedding(self.embed_shape[0], self.embed_shape[1])
|
||||
self.cls_emb = 256 #self.embed_shape[1]
|
||||
h_dim = 768
|
||||
self.y_embed = nn.Embedding(2, self.cls_emb)
|
||||
|
||||
#self.ins_weight = nn.Linear(hidden_size+self.cls_emb, 1)
|
||||
|
||||
self.ins_weight = nn.Sequential(
|
||||
nn.Linear(hidden_size+self.cls_emb, h_dim),
|
||||
nn.ReLU(), #Tanh(),
|
||||
nn.Linear(h_dim, h_dim),
|
||||
nn.ReLU(), #Tanh(),
|
||||
nn.Linear(h_dim, 1)
|
||||
)
|
||||
|
||||
def reset_groups(self, new_groups):
|
||||
self.n_groups = new_groups
|
||||
|
||||
def forward(self, x_feature, y_weak, item_loss=None):
|
||||
'''
|
||||
|
||||
Args:
|
||||
item_loss: shape is [batchsize * 3, ].
|
||||
e.g [item1_weak1_loss,
|
||||
item1_weak2_loss,
|
||||
item1_weak3_loss,
|
||||
....
|
||||
]
|
||||
iw:
|
||||
|
||||
Returns:
|
||||
|
||||
'''
|
||||
# detach the feature
|
||||
x_feature = x_feature.detach()
|
||||
feature_dim = x_feature.shape[-1]
|
||||
x_feature = x_feature.repeat(1, self.n_groups).view(-1, feature_dim)
|
||||
y_emb = self.y_embed(y_weak).view(-1, self.cls_emb)
|
||||
#ATTENTION: weight depends on the pair of feature and weak label instead of the source.
|
||||
#final_weight = F.softmax(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1)), dim=0).squeeze()
|
||||
#return (final_weight * item_loss).sum() ,final_weight
|
||||
|
||||
# sigmoid with mean
|
||||
final_weight = torch.sigmoid(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1))).squeeze()
|
||||
if item_loss is None:
|
||||
return final_weight
|
||||
else:
|
||||
return (final_weight * item_loss).mean(), final_weight
|
||||
|
||||
|
||||
|
||||
|
||||
#final_weight = F.relu(self.ins_weight(torch.cat([x_feature, y_emb], dim=-1))).squeeze()
|
||||
#return (final_weight * item_loss).mean()
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import torch.nn as nn
|
||||
import torch
|
||||
import math
|
||||
|
||||
class GroupWeightModel(nn.Module):
|
||||
def __init__(self, n_groups):
|
||||
super(GroupWeightModel, self).__init__()
|
||||
self.n_groups = n_groups
|
||||
self.pesu_group_weight = nn.Parameter(torch.randn(1, self.n_groups), requires_grad=True)
|
||||
self.init_weight()
|
||||
|
||||
def init_weight(self):
|
||||
stdv = 1. / math.sqrt(self.pesu_group_weight.size(1))
|
||||
self.pesu_group_weight.data.uniform_(-stdv, stdv)
|
||||
|
||||
def forward(self, item_loss, iw):
|
||||
'''
|
||||
|
||||
Args:
|
||||
item_loss: shape is [batchsize * 3, ].
|
||||
e.g [item1_weak1_loss,
|
||||
item1_weak2_loss,
|
||||
item1_weak3_loss,
|
||||
....
|
||||
]
|
||||
iw:
|
||||
|
||||
Returns:
|
||||
|
||||
'''
|
||||
group_weight = torch.sigmoid(self.pesu_group_weight)
|
||||
final_weight = torch.matmul(iw.view(-1, 1), group_weight)
|
||||
|
||||
return (final_weight * (item_loss.view(final_weight.shape))).sum()
|
||||
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
from transformers import BertPreTrainedModel, RobertaConfig, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaModel
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
class RobertaClassificationHead(nn.Module):
|
||||
"""Head for sentence-level classification tasks."""
|
||||
|
||||
def __init__(self, config):
|
||||
super(RobertaClassificationHead, self).__init__()
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.dropout = nn.Dropout(config.hidden_dropout_prob)
|
||||
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
||||
|
||||
def forward(self, features, **kwargs):
|
||||
x = features[:, 1, :] # take <s> token (equiv. to [CLS])
|
||||
x = self.dropout(x)
|
||||
x = self.dense(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.dropout(x)
|
||||
x = self.out_proj(x)
|
||||
return x
|
||||
|
||||
class RobertaForSequenceClassification(BertPreTrainedModel):
|
||||
config_class = RobertaConfig
|
||||
pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
|
||||
base_model_prefix = "roberta"
|
||||
|
||||
def __init__(self, config):
|
||||
super(RobertaForSequenceClassification, self).__init__(config)
|
||||
self.num_labels = 2
|
||||
|
||||
self.roberta = RobertaModel(config)
|
||||
# self.classifier = RobertaClassificationHead(config)
|
||||
self.pre_classifier = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.classifier = nn.Linear(config.hidden_size, self.num_labels)
|
||||
self.dropout = nn.Dropout(config.hidden_dropout_prob)
|
||||
self.config = config
|
||||
self.c_count = 1
|
||||
|
||||
def expand_class_head(self, c_count):
|
||||
self.c_count = c_count
|
||||
if c_count > 1:
|
||||
for i in range(1, c_count+1):
|
||||
setattr(self, "classifier_{}".format(i), nn.Linear(self.config.hidden_size, self.num_labels))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
labels=None,
|
||||
reduction="mean", is_gold=True
|
||||
):
|
||||
outputs = self.roberta(
|
||||
input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
# sequence_output = outputs[0]
|
||||
# logits = self.classifier(sequence_output)
|
||||
hidden_state = outputs[0]
|
||||
pooled_output = hidden_state[:, 1] # (bs, dim)
|
||||
outputs = (pooled_output.detach(),)
|
||||
pooled_output = self.pre_classifier(pooled_output) # (bs, dim)
|
||||
pooled_output = nn.ReLU()(pooled_output) # (bs, dim)
|
||||
pooled_output = self.dropout(pooled_output) # (bs, dim)
|
||||
|
||||
if self.c_count == 1 or is_gold:
|
||||
logits = self.classifier(pooled_output) # (N, C)
|
||||
# logits = self.classifier(sequence_output) # (N, C)
|
||||
else:
|
||||
logits = []
|
||||
for i in range(1, self.c_count+1):
|
||||
logits.append(self.__dict__['_modules']["classifier_{}".format(i)](pooled_output))
|
||||
logits = torch.cat(logits, dim=1)
|
||||
|
||||
outputs = (logits, ) + outputs
|
||||
if labels is not None:
|
||||
loss_fct = nn.CrossEntropyLoss(reduction=reduction)
|
||||
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
||||
outputs = (loss,) + outputs
|
||||
|
||||
return outputs # (loss), logits
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
from .RoBertModel import RobertaForSequenceClassification
|
||||
from .CNNModel import CNN_Text
|
||||
from .GroupWeightModel import GroupWeightModel
|
||||
from .DistilBertModel import DistilBertForSequenceClassification
|
||||
from .FullWeightModel import FullWeightModel
|
||||
from .BertModel import BertForSequenceClassification
|
||||
Загрузка…
Ссылка в новой задаче