accelerate

PranavUikey · Mar 23, 2021 · 43caf07 · 43caf07
1 parent 21d7276
commit 43caf07
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 22 deletions.
diff --git a/CAT/model/IRT.py b/CAT/model/IRT.py
@@ -163,6 +163,32 @@ def evaluate(self, adaptest_data: AdapTestDataset):
             'cov': cov,
         }
 
+    def get_pred(self, adaptest_data: AdapTestDataset):
+        """
+        Returns:
+            predictions, dict[sid][qid]
+        """
+        data = adaptest_data.data
+        concept_map = adaptest_data.concept_map
+        device = self.config['device']
+
+        pred_all = {}
+
+        with torch.no_grad():
+            self.model.eval()
+            for sid in data:
+                pred_all[sid] = {}
+                student_ids = [sid] * len(data[sid])
+                question_ids = list(data[sid].keys())
+                student_ids = torch.LongTensor(student_ids).to(device)
+                question_ids = torch.LongTensor(question_ids).to(device)
+                output = self.model(student_ids, question_ids).view(-1).tolist()
+                for i, qid in enumerate(list(data[sid].keys())):
+                    pred_all[sid][qid] = output[i]
+            self.model.train()
+
+        return pred_all
+
     def _loss_function(self, pred, real):
         return -(real * torch.log(0.0001 + pred) + (1 - real) * torch.log(1.0001 - pred)).mean()
 
@@ -171,29 +197,29 @@ def get_alpha(self, question_id):
         Args:
             question_id: int, question id
         Returns:
-            alpha of the given question
+            alpha of the given question, shape (num_dim, )
         """
-        return self.model.alpha.weight.data.numpy()[question_id]
+        return self.model.alpha.weight.data.cpu().numpy()[question_id]
 
     def get_beta(self, question_id):
         """ get beta of one question
         Args:
             question_id: int, question id
         Returns:
-            beta of the given question
+            beta of the given question, shape (1, )
         """
-        return self.model.beta.weight.data.numpy()[question_id]
+        return self.model.beta.weight.data.cpu().numpy()[question_id]
 
     def get_theta(self, student_id):
         """ get theta of one student
         Args:
             student_id: int, student id
         Returns:
-            theta of the given student
+            theta of the given student, shape (num_dim, )
         """
-        return self.model.theta.weight.data.numpy()[student_id]
+        return self.model.theta.weight.data.cpu().numpy()[student_id]
 
-    def get_kli(self, student_id, question_id, n):
+    def get_kli(self, student_id, question_id, n, pred_all):
         """ get KL information
         Args:
             student_id: int, student id
@@ -208,10 +234,10 @@ def get_kli(self, student_id, question_id, n):
         dim = self.model.num_dim
         sid = torch.LongTensor([student_id]).to(device)
         qid = torch.LongTensor([question_id]).to(device)
-        theta = self.model.theta(sid).clone().detach().numpy()[0] # (num_dim, )
-        alpha = self.model.alpha(qid).clone().detach().numpy()[0] # (num_dim, )
-        beta = self.model.beta(qid).clone().detach().numpy()[0][0] # float value
-        pred_estimate = self.model(sid, qid).data.numpy()[0][0] # float value
+        theta = self.get_theta(sid) # (num_dim, )
+        alpha = self.get_alpha(qid) # (num_dim, )
+        beta = self.get_beta(qid)[0] # float value
+        pred_estimate = pred_all[student_id][question_id]
         def kli(x):
             """ The formula of KL information. Used for integral.
             Args:
@@ -228,13 +254,15 @@ def kli(x):
         c = 3
         boundaries = [[theta[i] - c / np.sqrt(n), theta[i] + c / np.sqrt(n)] for i in range(dim)]
         if len(boundaries) == 1:
+            # KLI
             v, err = integrate.quad(kli, boundaries[0][0], boundaries[0][1])
             return v
+        # MKLI
         integ = vegas.Integrator(boundaries)
         result = integ(kli, nitn=10, neval=1000)
         return result.mean
 
-    def get_fisher(self, student_id, question_id):
+    def get_fisher(self, student_id, question_id, pred_all):
         """ get Fisher information
         Args:
             student_id: int, student id
@@ -243,15 +271,14 @@ def get_fisher(self, student_id, question_id):
             fisher_info: matrix(num_dim * num_dim), Fisher information
         """
         device = self.config['device']
-        sid = torch.LongTensor([student_id]).to(device)
         qid = torch.LongTensor([question_id]).to(device)
-        alpha = self.model.alpha(qid).clone().detach()
-        pred = self.model(sid, qid).data
+        alpha = self.model.alpha(qid).clone().detach().cpu()
+        pred = pred_all[student_id][question_id]
         q = 1 - pred
         fisher_info = (q*pred*(alpha * alpha.T)).numpy()
         return fisher_info
 
-    def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset):
+    def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset, pred_all: dict):
         """ get expected model change
         Args:
             student_id: int, student id
@@ -298,7 +325,7 @@ def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDatas
         for param in self.model.parameters():
             param.requires_grad = True
 
-        pred = self.model(student_id, question_id).item()
+        pred = pred_all[sid][qid]
         return pred * torch.norm(pos_weights - original_weights).item() + \
                (1 - pred) * torch.norm(neg_weights - original_weights).item()
 

diff --git a/CAT/model/NCD.py b/CAT/model/NCD.py
@@ -165,6 +165,7 @@ def adaptest_update(self, adaptest_data: AdapTestDataset):
                 student_ids = student_ids.to(device)
                 question_ids = question_ids.to(device)
                 labels = labels.to(device)
+                concepts_emb = concepts_emb.to(device)
                 pred = self.model(student_ids, question_ids, concepts_emb)
                 bz_loss = self._loss_function(pred, labels)
                 optimizer.zero_grad()
@@ -223,7 +224,35 @@ def evaluate(self, adaptest_data: AdapTestDataset):
             'cov': cov,
         }
 
-    def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset):
+    def get_pred(self, adaptest_data: AdapTestDataset):
+        data = adaptest_data.data
+        concept_map = adaptest_data.concept_map
+        device = self.config['device']
+
+        pred_all = {}
+        with torch.no_grad():
+            self.model.eval()
+            for sid in data:
+                pred_all[sid] = {}
+                student_ids = [sid] * len(data[sid])
+                question_ids = list(data[sid].keys())
+                concepts_embs = []
+                for qid in question_ids:
+                    concepts = concept_map[qid]
+                    concepts_emb = [0.] * adaptest_data.num_concepts
+                    for concept in concepts:
+                        concepts_emb[concept] = 1.0
+                    concepts_embs.append(concepts_emb)
+                student_ids = torch.LongTensor(student_ids).to(device)
+                question_ids = torch.LongTensor(question_ids).to(device)
+                concepts_embs = torch.Tensor(concepts_embs).to(device)
+                output = self.model(student_ids, question_ids, concepts_embs).view(-1).tolist()
+                for i, qid in enumerate(list(data[sid].keys())):
+                    pred_all[sid][qid] = output[i]
+            self.model.train()
+        return pred_all
+
+    def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset, pred_all: dict):
         """ get expected model change
         Args:
             student_id: int, student id
@@ -275,6 +304,7 @@ def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDatas
         for param in self.model.parameters():
             param.requires_grad = True
 
-        pred = self.model(student_id, question_id, concepts_emb).item()
+        # pred = self.model(student_id, question_id, concepts_emb).item()
+        pred = pred_all[sid][qid]
         return pred * torch.norm(pos_weights - original_weights).item() + \
                (1 - pred) * torch.norm(neg_weights - original_weights).item()
diff --git a/CAT/strategy/KLI_strategy.py b/CAT/strategy/KLI_strategy.py
@@ -17,12 +17,15 @@ def name(self):
     def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset):
         assert hasattr(model, 'get_kli'), \
             'the models must implement get_kli method'
+        assert hasattr(model, 'get_pred'), \
+            'the models must implement get_pred method for accelerating'
+        pred_all = model.get_pred(adaptest_data)
         selection = {}
         n = len(adaptest_data.tested[0])
         for sid in range(adaptest_data.num_students):
             theta = model.get_theta(sid)
             untested_questions = np.array(list(adaptest_data.untested[sid]))
-            untested_kli = [model.get_kli(sid, qid, n) for qid in untested_questions]
+            untested_kli = [model.get_kli(sid, qid, n, pred_all) for qid in untested_questions]
             j = np.argmax(untested_kli)
             selection[sid] = untested_questions[j]
         return selection

diff --git a/CAT/strategy/MAAT_strategy.py b/CAT/strategy/MAAT_strategy.py
@@ -29,10 +29,11 @@ def _compute_coverage_gain(self, sid, qid, adaptest_data: AdapTestDataset):
     def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset):
         assert hasattr(model, 'expected_model_change'), \
             'the models must implement expected_model_change method'
+        pred_all = model.get_pred(adaptest_data)
         selection = {}
         for sid in range(adaptest_data.num_students):
             untested_questions = np.array(list(adaptest_data.untested[sid]))
-            emc_arr = [model.expected_model_change(sid, qid, adaptest_data) for qid in untested_questions]
+            emc_arr = [model.expected_model_change(sid, qid, adaptest_data, pred_all) for qid in untested_questions]
             candidates = untested_questions[np.argsort(emc_arr)[::-1][:self.n_candidates]]
             selection[sid] = max(candidates, key=lambda qid: self._compute_coverage_gain(sid, qid, adaptest_data))
         return selection
diff --git a/CAT/strategy/MFI_strategy.py b/CAT/strategy/MFI_strategy.py
@@ -22,6 +22,9 @@ def name(self):
     def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset):
         assert hasattr(model, 'get_fisher'), \
             'the models must implement get_fisher method'
+        assert hasattr(model, 'get_pred'), \
+            'the models must implement get_pred method for accelerating'
+        pred_all = model.get_pred(adaptest_data)
         if self.I is None:
             self.I = [np.zeros((model.model.num_dim, model.model.num_dim))] * adaptest_data.num_students    
         selection = {}
@@ -31,7 +34,7 @@ def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset):
             untested_dets = []
             untested_fisher = []
             for qid in untested_questions:
-                fisher_info = model.get_fisher(sid, qid)
+                fisher_info = model.get_fisher(sid, qid, pred_all)
                 untested_fisher.append(fisher_info)
                 untested_dets.append(np.linalg.det(self.I[sid] + fisher_info))
             j = np.argmax(untested_dets)