From 2e8f50e3e330e0b2c37d95158c1322844515c35f Mon Sep 17 00:00:00 2001 From: Uladzislau Sazanovich Date: Sun, 8 Apr 2018 00:44:47 +0300 Subject: [PATCH] Some venv fixes --- requirements.txt | 3 +- scripts/{download.sh => data_download.sh} | 0 scripts/{data.py => data_process.py} | 0 scripts/{data.sh => data_process.sh} | 2 +- scripts/run.sh | 3 + scripts/tensorboard.sh | 5 + .../js/ast_level/model/n2n_sum_attention.py | 150 ++++++++++++++++++ zerogercrnn/lib/visualization/plotter.py | 7 +- 8 files changed, 163 insertions(+), 7 deletions(-) rename scripts/{download.sh => data_download.sh} (100%) rename scripts/{data.py => data_process.py} (100%) rename scripts/{data.sh => data_process.sh} (65%) mode change 100644 => 100755 create mode 100755 scripts/run.sh create mode 100755 scripts/tensorboard.sh create mode 100644 zerogercrnn/experiments/js/ast_level/model/n2n_sum_attention.py diff --git a/requirements.txt b/requirements.txt index fe59114..2a8757d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,4 @@ tensorboard tensorboardX matplotlib visdom -pylint -urllib3 \ No newline at end of file +pylint \ No newline at end of file diff --git a/scripts/download.sh b/scripts/data_download.sh similarity index 100% rename from scripts/download.sh rename to scripts/data_download.sh diff --git a/scripts/data.py b/scripts/data_process.py similarity index 100% rename from scripts/data.py rename to scripts/data_process.py diff --git a/scripts/data.sh b/scripts/data_process.sh old mode 100644 new mode 100755 similarity index 65% rename from scripts/data.sh rename to scripts/data_process.sh index 1072254..521d3b4 --- a/scripts/data.sh +++ b/scripts/data_process.sh @@ -1,3 +1,3 @@ -#!/usr/bin/env bash +#!/bin/bash PYTHONPATH=. python3 scripts/data.py diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..77dd3a9 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +PYTHONPATH=. python3 zerogercrnn/experiments/js/ast_level/main/py_main.py --config_file "zerogercrnn/experiments/js/ast_level/main/config.json" --title $1 diff --git a/scripts/tensorboard.sh b/scripts/tensorboard.sh new file mode 100755 index 0000000..ad04fd9 --- /dev/null +++ b/scripts/tensorboard.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mkdir tensorboard +mkdir tensorboard/runs +tensorboard --logdir tensorboard/runs \ No newline at end of file diff --git a/zerogercrnn/experiments/js/ast_level/model/n2n_sum_attention.py b/zerogercrnn/experiments/js/ast_level/model/n2n_sum_attention.py new file mode 100644 index 0000000..a8dae78 --- /dev/null +++ b/zerogercrnn/experiments/js/ast_level/model/n2n_sum_attention.py @@ -0,0 +1,150 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable + +from zerogercrnn.experiments.js.ast_level.model.core import RecurrentCore +from zerogercrnn.experiments.js.ast_level.model.utils import init_layers_uniform +from zerogercrnn.lib.utils.time import logger + +"""Base rnn model for JS AST prediction (in use since 01Apr till 01Apr)""" + + +class N2NSumAttentionModel(nn.Module): + """Model that predicts next non-terminals by the sequence of non-terminals.""" + + def __init__( + self, + non_terminal_vocab_size, + embedding_size, + recurrent_layer: RecurrentCore, + ): + super(N2NSumAttentionModel, self).__init__() + assert recurrent_layer.input_size == embedding_size + + self.recurrent_out_size = recurrent_layer.hidden_size + self.embedding_size = embedding_size + self.non_terminal_output_size = non_terminal_vocab_size + + # PyTorch is not able to have one optimizer for sparse and non-sparse layers, so we should split parameters + self.dense_params = [] + self.sparse_params = [] + + # Layer that encodes one-hot vector of non-terminals (A) + self.non_terminal_embedding = self.sparse_model(nn.Embedding( + num_embeddings=non_terminal_vocab_size, + embedding_dim=embedding_size, + sparse=True + )) + + # Recurrent layer that will have A as an input + self.recurrent = self.dense_model( + recurrent_layer + ) + + self.attn = self.dense_model(nn.Linear( + in_features=self.recurrent_out_size, + out_features=self.recurrent_out_size, + bias=False + )) + + self.attn_softmax = nn.Softmax(dim=0) + + # Layer that transforms hidden state of recurrent layer into next non-terminal + self.h2o = self.dense_model( + nn.Linear(2 * self.recurrent_out_size, self.non_terminal_output_size) + ) + self.softmax_o = nn.LogSoftmax(dim=1) + + self._init_params_() + + def forward(self, non_terminal_input, hidden): + """ + :param non_terminal_input: tensor of size [seq_len, batch_size, 1] + :param hidden: hidden state of recurrent layer + """ + seq_len = non_terminal_input.size()[0] + batch_size = non_terminal_input.size()[1] + + logger.reset_time() + non_terminal_input = torch.squeeze(non_terminal_input, dim=2) + + # this tensors will be the size of [batch_size, seq_len, embedding_dim] + non_terminal_emb = self.non_terminal_embedding(non_terminal_input.permute(1, 0)) + + non_terminal_emb = non_terminal_emb.permute(1, 0, 2) + + recurrent_input = non_terminal_emb + logger.log_time_ms('PRE_IN') + + # output_tensor will be the size of (seq_len, batch_size, hidden_size * num_directions) + recurrent_output, hidden = self.recurrent(recurrent_input, hidden) + logger.log_time_ms('RECURRENT') + + recurrent_besides_last = recurrent_output.narrow(0, 0, recurrent_output.size()[0] - 1) + recurrent_last = recurrent_output[-1] + attn_weights = self.attn(recurrent_besides_last).matmul(recurrent_last[-1]) + attn_weights = self.attn_softmax(attn_weights) + + # calc cntx vector as sum of h-s multiplied by alpha, then sigmoid + cntx = attn_weights.permute(1, 0).unsqueeze(1).matmul(recurrent_besides_last.permute(1, 0, 2)).squeeze(1) + + # concatenate cntx and hidden from last timestamp + recurrent_attention_output = torch.cat([cntx, recurrent_last], dim=1) + + o = self.h2o(recurrent_attention_output) + o = self.softmax_o(o) + + logger.log_time_ms('PRE_OUT') + + return o, hidden + + def sparse_model(self, model): + self.sparse_params += model.parameters() + return model + + def dense_model(self, model): + self.dense_params += model.parameters() + return model + + def _init_params_(self): + init_layers_uniform( + min_value=-0.05, + max_value=0.05, + layers=[ + self.non_terminal_embedding, + self.attn, + self.h2o + ] + ) + + def init_hidden(self, batch_size, cuda): + return self.recurrent.init_hidden(batch_size, cuda) + + +if __name__ == '__main__': + core = RecurrentCore( + input_size=100, + hidden_size=1500, + num_layers=1 + ) + + model = N2NSumAttentionModel( + non_terminal_vocab_size=1000, + embedding_size=100, + recurrent_layer=core + ) + + in_tensor = Variable(torch.LongTensor(50, 80, 1).zero_()) + hidden_tensor = Variable(torch.randn((1, 80, 1500))) + + model.forward(non_terminal_input=in_tensor, hidden=hidden_tensor) + + # o_t = torch.randn((50, 80, 1500)) + # w_attn = torch.randn((1500, 1500)) + # h_k = torch.randn(80, 1500) + # + # # o_t = o_t.view(-1, 1500) + # attn_weights = o_t.matmul(w_attn) + # attn_weights = attn_weights.matmul(h_k[-1]) + # + # print(torch.bmm(o_t, h_k.expand).size()) diff --git a/zerogercrnn/lib/visualization/plotter.py b/zerogercrnn/lib/visualization/plotter.py index 0be8727..28ce9fa 100644 --- a/zerogercrnn/lib/visualization/plotter.py +++ b/zerogercrnn/lib/visualization/plotter.py @@ -1,12 +1,11 @@ import os import matplotlib.pyplot as plt -import visdom import numpy as np - +import visdom from tensorboardX import SummaryWriter -TENSORBOARD_DIR = '/tensorboard/runs/' +TENSORBOARD_DIR = 'tensorboard/runs/' class Plotter: @@ -68,7 +67,7 @@ def on_new_point(self, label, x, y): class TensorboardPlotter(Plotter): def __init__(self, title): - path = TENSORBOARD_DIR + title + path = os.path.join(os.getcwd(), TENSORBOARD_DIR + title) self.writer = SummaryWriter(path) def on_new_point(self, label, x, y):