"""A single training/test example for the CommonsenseQA dataset."""

def __init__(self,

hit_id,

context_sentence,

ending_0,

ending_1,

ending_2,

ending_3,

label = None):

self.hit_id = hit_id

self.context_sentence = context_sentence

self.endings = [

ending_0,

ending_1,

ending_2,

ending_3,

]

self.label = label

def __str__(self):

return self.__repr__()

def __repr__(self):

l = [

f"question_id: {self.hit_id}",

f"context_sentence: {self.context_sentence}",

f"ending_0: {self.endings[0]}",

f"ending_1: {self.endings[1]}",

f"ending_2: {self.endings[2]}",

f"ending_3: {self.endings[3]}",

]

if self.label is not None:

l.append(f"label: {self.label}")

def __init__(self,

example_id,

choices_features,

label

):

self.example_id = example_id

self.choices_features = [

{

'input_ids': input_ids,

'input_mask': input_mask,

'segment_ids': segment_ids

}

for _, input_ids, input_mask, segment_ids in choices_features

]

examples = []

with open(input_file, 'r') as csv_file:

reader = csv.DictReader(csv_file)

for row in reader:

hit_id = row["id"]

context = row["context"]

question = row["question"]

answer1 = row["answer1"]

answer2 = row["answer2"]

answer3 = row["answer3"]

answer4 = row["answer4"]

label = row["label"]

if len(question.strip()) > 0 and len(answer1.strip()) > 0 and len(answer2.strip()) > 0 and \

len(answer3.strip()) > 0 and len(answer4.strip()) > 0:

examples.append(CommonsenseQAExample(hit_id = hit_id, context_sentence = question + " " + context,

ending_0 = answer1,

ending_1 = answer2,

ending_2 = answer3,

ending_3 = answer4,

label = int(label) if is_training else None))

is_training):

"""Loads a data file into a list of `InputBatch`s."""

# Swag is a multiple choice task. To perform this task using Bert,

# we will use the formatting proposed in "Improving Language

# Understanding by Generative Pre-Training" and suggested by

# @jacobdevlin-google in this issue

# https://github.com/google-research/bert/issues/38.

#

# Each choice will correspond to a sample on which we run the

# inference. For a given Swag example, we will create the 4

# following inputs:

# - [CLS] context [SEP] choice_1 [SEP]

# - [CLS] context [SEP] choice_2 [SEP]

# - [CLS] context [SEP] choice_3 [SEP]

# - [CLS] context [SEP] choice_4 [SEP]

# The model will output a single value for each input. To get the

# final decision of the model, we will run a softmax over these 4

# outputs.

features = []

for example_index, example in enumerate(examples):

context_tokens = tokenizer.tokenize(example.context_sentence)

# start_ending_tokens = tokenizer.tokenize(example.start_ending)

choices_features = []

for ending_index, ending in enumerate(example.endings):

# We create a copy of the context tokens in order to be

# able to shrink it according to ending_tokens

context_tokens_choice = context_tokens[:]

ending_tokens = tokenizer.tokenize(ending)

# Modifies `context_tokens_choice` and `ending_tokens` in

# place so that the total length is less than the

# specified length. Account for [CLS], [SEP], [SEP] with

# "- 3"

_truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]

segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

input_ids = tokenizer.convert_tokens_to_ids(tokens)

input_mask = [1] * len(input_ids)

# Zero-pad up to the sequence length.

padding = [0] * (max_seq_length - len(input_ids))

input_ids += padding

input_mask += padding

segment_ids += padding

assert len(input_ids) == max_seq_length

assert len(input_mask) == max_seq_length

assert len(segment_ids) == max_seq_length

choices_features.append((tokens, input_ids, input_mask, segment_ids))

label = example.label

# if example_index < 5:

# logger.info("*** Example ***")

# logger.info(f"hit_id: {example.hit_id}")

# for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):

# logger.info(f"choice: {choice_idx}")

# logger.info(f"tokens: {' '.join(tokens)}")

# logger.info(f"input_ids: {' '.join(map(str, input_ids))}")

# logger.info(f"input_mask: {' '.join(map(str, input_mask))}")

# logger.info(f"segment_ids: {' '.join(map(str, segment_ids))}")

# if is_training:

# logger.info(f"label: {label}")

features.append(

InputFeatures(

example_id = example.hit_id,

choices_features = choices_features,

label = label

)

)

"""Truncates a sequence pair in place to the maximum length."""

# This is a simple heuristic which will always truncate the longer sequence

# one token at a time. This makes more sense than truncating an equal percent

# of tokens from each, since if one sequence is very short then each token

# that's truncated likely contains more information than a longer sequence.

while True:

total_length = len(tokens_a) + len(tokens_b)

if total_length <= max_length:

break

if len(tokens_a) > len(tokens_b):

tokens_a.pop()

else:

return [

[

choice[field]

for choice in feature.choices_features

]

for feature in features

parser = argparse.ArgumentParser()

#root = "/Users/lifuh/Documents/Codes/"

root = "/data/m1/huangl7/"

# Required parameters

parser.add_argument("--data_dir",

default=root + "openai-gpt-pytorch/data/commonsenseqa/",

type=str,

help="The input data dir. Should contain the .csv files (or other data files) for the task.")

parser.add_argument("--bert_model", default="bert-large-cased", type=str,

help="Bert pre-trained model selected in the list: bert-base-uncased, "

"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "

"bert-base-multilingual-cased, bert-base-chinese.")

parser.add_argument("--output_dir",

default=root + "openai-gpt-pytorch/data/commonsenseqa/output/",

type=str,

help="The output directory where the model checkpoints will be written.")

# Other parameters

parser.add_argument("--max_seq_length",

default=128,

type=int,

help="The maximum total input sequence length after WordPiece tokenization. \n"

"Sequences longer than this will be truncated, and sequences shorter \n"

"than this will be padded.")

parser.add_argument("--do_train",

default=True,

action='store_true',

help="Whether to run training.")

parser.add_argument("--do_eval",

default=True,

action='store_true',

help="Whether to run eval on the dev set.")

parser.add_argument("--do_lower_case",

action='store_true',

help="Set this flag if you are using an uncased model.")

parser.add_argument("--train_batch_size",

default=4,

type=int,

help="Total batch size for training.")

parser.add_argument("--eval_batch_size",

default=1,

type=int,

help="Total batch size for eval.")

parser.add_argument("--learning_rate",

default=5e-5,

type=float,

help="The initial learning rate for Adam.")

parser.add_argument("--num_train_epochs",

default=4.0,

type=float,

help="Total number of training epochs to perform.")

parser.add_argument("--warmup_proportion",

default=0.1,

type=float,

help="Proportion of training to perform linear learning rate warmup for. "

"E.g., 0.1 = 10%% of training.")

parser.add_argument("--no_cuda",

action='store_true',

help="Whether not to use CUDA when available")

parser.add_argument("--local_rank",

type=int,

default=-1,

help="local_rank for distributed training on gpus")

parser.add_argument('--seed',

type=int,

default=42,

help="random seed for initialization")

parser.add_argument('--gradient_accumulation_steps',

type=int,

default=1,

help="Number of updates steps to accumulate before performing a backward/update pass.")

parser.add_argument('--fp16',

action='store_true',

help="Whether to use 16-bit float precision instead of 32-bit")

parser.add_argument('--loss_scale',

type=float, default=0,

help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"

"0 (default value): dynamic loss scaling.\n"

"Positive power of 2: static loss scaling value.\n")

args = parser.parse_args()

# if args.local_rank == -1 or args.no_cuda:

# device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")

# n_gpu = torch.cuda.device_count()

# else:

# torch.cuda.set_device(args.local_rank)

# device = torch.device("cuda", args.local_rank)

# n_gpu = 1

# # Initializes the distributed backend which will take care of sychronizing nodes/GPUs

# torch.distributed.init_process_group(backend='nccl')

# logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(

# device, n_gpu, bool(args.local_rank != -1), args.fp16))

device = "cuda:4"

n_gpu = 1

if args.gradient_accumulation_steps < 1:

raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(

args.gradient_accumulation_steps))

args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)

random.seed(args.seed)

np.random.seed(args.seed)

torch.manual_seed(args.seed)

if n_gpu > 0:

torch.cuda.manual_seed_all(args.seed)

if not args.do_train and not args.do_eval:

raise ValueError("At least one of `do_train` or `do_eval` must be True.")

if os.path.exists(args.output_dir) and os.listdir(args.output_dir):

raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))

os.makedirs(args.output_dir, exist_ok=True)

tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)

train_examples = None

num_train_steps = None

if args.do_train:

# Batch_3542660_batch_results_commonsenseqa_v3_bert.csv

train_examples = read_qa_examples(os.path.join(args.data_dir, 'train.csv'), is_training = True)

num_train_steps = int(

len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)

# Prepare model

model = BertForMultipleChoice.from_pretrained(args.bert_model,

cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format(args.local_rank),

num_choices=4)

if args.fp16:

model.half()

model.to(device)

if args.local_rank != -1:

try:

from apex.parallel import DistributedDataParallel as DDP

except ImportError:

raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

model = DDP(model)

elif n_gpu > 1:

model = torch.nn.DataParallel(model)

# Prepare optimizer

param_optimizer = list(model.named_parameters())

# hack to remove pooler, which is not used

# thus it produce None grad that break apex

param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]

no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']

optimizer_grouped_parameters = [

{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},

{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

]

t_total = num_train_steps

if args.local_rank != -1:

t_total = t_total // torch.distributed.get_world_size()

if args.fp16:

try:

from apex.optimizers import FP16_Optimizer

from apex.optimizers import FusedAdam

except ImportError:

raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

optimizer = FusedAdam(optimizer_grouped_parameters,

lr=args.learning_rate,

bias_correction=False,

max_grad_norm=1.0)

if args.loss_scale == 0:

optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)

else:

optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)

else:

optimizer = BertAdam(optimizer_grouped_parameters,

lr=args.learning_rate,

warmup=args.warmup_proportion,

t_total=t_total)

global_step = 0

if args.do_train:

train_features = convert_examples_to_features(

train_examples, tokenizer, args.max_seq_length, True)

logger.info("***** Running training *****")

logger.info(" Num examples = %d", len(train_examples))

logger.info(" Batch size = %d", args.train_batch_size)

logger.info(" Num steps = %d", num_train_steps)

all_input_ids = torch.tensor(select_field(train_features, 'input_ids'), dtype=torch.long)

all_input_mask = torch.tensor(select_field(train_features, 'input_mask'), dtype=torch.long)

all_segment_ids = torch.tensor(select_field(train_features, 'segment_ids'), dtype=torch.long)

all_label = torch.tensor([f.label for f in train_features], dtype=torch.long)

train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)

if args.local_rank == -1:

train_sampler = RandomSampler(train_data)

else:

train_sampler = DistributedSampler(train_data)

train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)

model.train()

for _ in trange(int(args.num_train_epochs), desc="Epoch"):

tr_loss = 0

nb_tr_examples, nb_tr_steps = 0, 0

all_train_accurate = 0.0

all_train = 0.0

for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):

batch = tuple(t.to(device) for t in batch)

input_ids, input_mask, segment_ids, label_ids = batch

loss, logits = model(input_ids, segment_ids, input_mask, label_ids)

if n_gpu > 1:

loss = loss.mean() # mean() to average on multi-gpu.

if args.fp16 and args.loss_scale != 1.0:

# rescale loss for fp16 training

# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html

loss = loss * args.loss_scale

if args.gradient_accumulation_steps > 1:

loss = loss / args.gradient_accumulation_steps

logits = logits.detach().cpu().numpy()

label_ids = label_ids.to('cpu').numpy()

tmp_train_accurate = accuracy(logits, label_ids)

all_train_accurate += tmp_train_accurate

all_train += input_ids.size(0)

tr_loss += loss.item()

nb_tr_examples += input_ids.size(0)

nb_tr_steps += 1

if args.fp16:

optimizer.backward(loss)

else:

loss.backward()

if (step + 1) % args.gradient_accumulation_steps == 0:

# modify learning rate with special warm up BERT uses

lr_this_step = args.learning_rate * warmup_linear(global_step/t_total, args.warmup_proportion)

for param_group in optimizer.param_groups:

param_group['lr'] = lr_this_step

optimizer.step()

optimizer.zero_grad()

global_step += 1

train_acc = all_train_accurate / all_train

print("current train accuracy is " + str(train_acc))

# Save a trained model

model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self

output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")

torch.save(model_to_save.state_dict(), output_model_file)

# Load a trained model that you have fine-tuned

model_state_dict = torch.load(output_model_file)

model = BertForMultipleChoice.from_pretrained(args.bert_model,

state_dict=model_state_dict,

num_choices=4)

model.to(device)

if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):

# Batch_3542660_batch_results_commonsenseqa_v3_bert.csv

eval_examples = read_qa_examples(os.path.join(args.data_dir, 'valid.csv'), is_training = True)

eval_features = convert_examples_to_features(

eval_examples, tokenizer, args.max_seq_length, True)

logger.info("***** Running evaluation *****")

logger.info(" Num examples = %d", len(eval_examples))

logger.info(" Batch size = %d", args.eval_batch_size)

all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'), dtype=torch.long)

all_input_mask = torch.tensor(select_field(eval_features, 'input_mask'), dtype=torch.long)

all_segment_ids = torch.tensor(select_field(eval_features, 'segment_ids'), dtype=torch.long)

all_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)

eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)

# Run prediction for full data

eval_sampler = SequentialSampler(eval_data)

eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

model.eval()

eval_loss, eval_accuracy = 0, 0

all_pred_labels = []

all_anno_labels = []

all_logits = []

nb_eval_steps, nb_eval_examples = 0, 0

for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:

input_ids = input_ids.to(device)

input_mask = input_mask.to(device)

segment_ids = segment_ids.to(device)

label_ids = label_ids.to(device)

with torch.no_grad():

tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)

logits = logits.detach().cpu().numpy()

output_labels = np.argmax(logits, axis=1)

all_pred_labels.extend(output_labels.tolist())

all_logits.extend(list(logits))

all_anno_labels.extend(list(label_ids.detach().cpu().numpy()))

label_ids = label_ids.to('cpu').numpy()

tmp_eval_accuracy = accuracy(logits, label_ids)

eval_loss += tmp_eval_loss.mean().item()

eval_accuracy += tmp_eval_accuracy

nb_eval_examples += input_ids.size(0)

nb_eval_steps += 1

eval_loss = eval_loss / nb_eval_steps

eval_accuracy = eval_accuracy / nb_eval_examples

print("eval accuracy is " + str(eval_accuracy))

result = {'eval_loss': eval_loss,

'eval_accuracy': eval_accuracy,

'global_step': global_step,

'loss': tr_loss/nb_tr_steps}

output_eval_file = os.path.join(args.output_dir, "bert_eval_results.txt")

with open(output_eval_file, "w") as writer:

logger.info("***** Eval results *****")

for key in sorted(result.keys()):

logger.info(" %s = %s", key, str(result[key]))

writer.write("%s = %s\n" % (key, str(result[key])))

for i in range(len(all_pred_labels)):

writer.write(str(i) + "\t" + str(all_anno_labels[i]) + "\t" +