diff --git a/CAT/model/IRT.py b/CAT/model/IRT.py index 6864566..23d9e3f 100644 --- a/CAT/model/IRT.py +++ b/CAT/model/IRT.py @@ -163,6 +163,32 @@ def evaluate(self, adaptest_data: AdapTestDataset): 'cov': cov, } + def get_pred(self, adaptest_data: AdapTestDataset): + """ + Returns: + predictions, dict[sid][qid] + """ + data = adaptest_data.data + concept_map = adaptest_data.concept_map + device = self.config['device'] + + pred_all = {} + + with torch.no_grad(): + self.model.eval() + for sid in data: + pred_all[sid] = {} + student_ids = [sid] * len(data[sid]) + question_ids = list(data[sid].keys()) + student_ids = torch.LongTensor(student_ids).to(device) + question_ids = torch.LongTensor(question_ids).to(device) + output = self.model(student_ids, question_ids).view(-1).tolist() + for i, qid in enumerate(list(data[sid].keys())): + pred_all[sid][qid] = output[i] + self.model.train() + + return pred_all + def _loss_function(self, pred, real): return -(real * torch.log(0.0001 + pred) + (1 - real) * torch.log(1.0001 - pred)).mean() @@ -171,29 +197,29 @@ def get_alpha(self, question_id): Args: question_id: int, question id Returns: - alpha of the given question + alpha of the given question, shape (num_dim, ) """ - return self.model.alpha.weight.data.numpy()[question_id] + return self.model.alpha.weight.data.cpu().numpy()[question_id] def get_beta(self, question_id): """ get beta of one question Args: question_id: int, question id Returns: - beta of the given question + beta of the given question, shape (1, ) """ - return self.model.beta.weight.data.numpy()[question_id] + return self.model.beta.weight.data.cpu().numpy()[question_id] def get_theta(self, student_id): """ get theta of one student Args: student_id: int, student id Returns: - theta of the given student + theta of the given student, shape (num_dim, ) """ - return self.model.theta.weight.data.numpy()[student_id] + return self.model.theta.weight.data.cpu().numpy()[student_id] - def get_kli(self, student_id, question_id, n): + def get_kli(self, student_id, question_id, n, pred_all): """ get KL information Args: student_id: int, student id @@ -208,10 +234,10 @@ def get_kli(self, student_id, question_id, n): dim = self.model.num_dim sid = torch.LongTensor([student_id]).to(device) qid = torch.LongTensor([question_id]).to(device) - theta = self.model.theta(sid).clone().detach().numpy()[0] # (num_dim, ) - alpha = self.model.alpha(qid).clone().detach().numpy()[0] # (num_dim, ) - beta = self.model.beta(qid).clone().detach().numpy()[0][0] # float value - pred_estimate = self.model(sid, qid).data.numpy()[0][0] # float value + theta = self.get_theta(sid) # (num_dim, ) + alpha = self.get_alpha(qid) # (num_dim, ) + beta = self.get_beta(qid)[0] # float value + pred_estimate = pred_all[student_id][question_id] def kli(x): """ The formula of KL information. Used for integral. Args: @@ -228,13 +254,15 @@ def kli(x): c = 3 boundaries = [[theta[i] - c / np.sqrt(n), theta[i] + c / np.sqrt(n)] for i in range(dim)] if len(boundaries) == 1: + # KLI v, err = integrate.quad(kli, boundaries[0][0], boundaries[0][1]) return v + # MKLI integ = vegas.Integrator(boundaries) result = integ(kli, nitn=10, neval=1000) return result.mean - def get_fisher(self, student_id, question_id): + def get_fisher(self, student_id, question_id, pred_all): """ get Fisher information Args: student_id: int, student id @@ -243,15 +271,14 @@ def get_fisher(self, student_id, question_id): fisher_info: matrix(num_dim * num_dim), Fisher information """ device = self.config['device'] - sid = torch.LongTensor([student_id]).to(device) qid = torch.LongTensor([question_id]).to(device) - alpha = self.model.alpha(qid).clone().detach() - pred = self.model(sid, qid).data + alpha = self.model.alpha(qid).clone().detach().cpu() + pred = pred_all[student_id][question_id] q = 1 - pred fisher_info = (q*pred*(alpha * alpha.T)).numpy() return fisher_info - def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset): + def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset, pred_all: dict): """ get expected model change Args: student_id: int, student id @@ -298,7 +325,7 @@ def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDatas for param in self.model.parameters(): param.requires_grad = True - pred = self.model(student_id, question_id).item() + pred = pred_all[sid][qid] return pred * torch.norm(pos_weights - original_weights).item() + \ (1 - pred) * torch.norm(neg_weights - original_weights).item() diff --git a/CAT/model/NCD.py b/CAT/model/NCD.py index 1b946dd..f8c2d36 100644 --- a/CAT/model/NCD.py +++ b/CAT/model/NCD.py @@ -165,6 +165,7 @@ def adaptest_update(self, adaptest_data: AdapTestDataset): student_ids = student_ids.to(device) question_ids = question_ids.to(device) labels = labels.to(device) + concepts_emb = concepts_emb.to(device) pred = self.model(student_ids, question_ids, concepts_emb) bz_loss = self._loss_function(pred, labels) optimizer.zero_grad() @@ -223,7 +224,35 @@ def evaluate(self, adaptest_data: AdapTestDataset): 'cov': cov, } - def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset): + def get_pred(self, adaptest_data: AdapTestDataset): + data = adaptest_data.data + concept_map = adaptest_data.concept_map + device = self.config['device'] + + pred_all = {} + with torch.no_grad(): + self.model.eval() + for sid in data: + pred_all[sid] = {} + student_ids = [sid] * len(data[sid]) + question_ids = list(data[sid].keys()) + concepts_embs = [] + for qid in question_ids: + concepts = concept_map[qid] + concepts_emb = [0.] * adaptest_data.num_concepts + for concept in concepts: + concepts_emb[concept] = 1.0 + concepts_embs.append(concepts_emb) + student_ids = torch.LongTensor(student_ids).to(device) + question_ids = torch.LongTensor(question_ids).to(device) + concepts_embs = torch.Tensor(concepts_embs).to(device) + output = self.model(student_ids, question_ids, concepts_embs).view(-1).tolist() + for i, qid in enumerate(list(data[sid].keys())): + pred_all[sid][qid] = output[i] + self.model.train() + return pred_all + + def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDataset, pred_all: dict): """ get expected model change Args: student_id: int, student id @@ -275,6 +304,7 @@ def expected_model_change(self, sid: int, qid: int, adaptest_data: AdapTestDatas for param in self.model.parameters(): param.requires_grad = True - pred = self.model(student_id, question_id, concepts_emb).item() + # pred = self.model(student_id, question_id, concepts_emb).item() + pred = pred_all[sid][qid] return pred * torch.norm(pos_weights - original_weights).item() + \ (1 - pred) * torch.norm(neg_weights - original_weights).item() \ No newline at end of file diff --git a/CAT/strategy/KLI_strategy.py b/CAT/strategy/KLI_strategy.py index b379b7b..f1bffb1 100644 --- a/CAT/strategy/KLI_strategy.py +++ b/CAT/strategy/KLI_strategy.py @@ -17,12 +17,15 @@ def name(self): def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset): assert hasattr(model, 'get_kli'), \ 'the models must implement get_kli method' + assert hasattr(model, 'get_pred'), \ + 'the models must implement get_pred method for accelerating' + pred_all = model.get_pred(adaptest_data) selection = {} n = len(adaptest_data.tested[0]) for sid in range(adaptest_data.num_students): theta = model.get_theta(sid) untested_questions = np.array(list(adaptest_data.untested[sid])) - untested_kli = [model.get_kli(sid, qid, n) for qid in untested_questions] + untested_kli = [model.get_kli(sid, qid, n, pred_all) for qid in untested_questions] j = np.argmax(untested_kli) selection[sid] = untested_questions[j] return selection diff --git a/CAT/strategy/MAAT_strategy.py b/CAT/strategy/MAAT_strategy.py index 8f635b9..3df9f28 100644 --- a/CAT/strategy/MAAT_strategy.py +++ b/CAT/strategy/MAAT_strategy.py @@ -29,10 +29,11 @@ def _compute_coverage_gain(self, sid, qid, adaptest_data: AdapTestDataset): def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset): assert hasattr(model, 'expected_model_change'), \ 'the models must implement expected_model_change method' + pred_all = model.get_pred(adaptest_data) selection = {} for sid in range(adaptest_data.num_students): untested_questions = np.array(list(adaptest_data.untested[sid])) - emc_arr = [model.expected_model_change(sid, qid, adaptest_data) for qid in untested_questions] + emc_arr = [model.expected_model_change(sid, qid, adaptest_data, pred_all) for qid in untested_questions] candidates = untested_questions[np.argsort(emc_arr)[::-1][:self.n_candidates]] selection[sid] = max(candidates, key=lambda qid: self._compute_coverage_gain(sid, qid, adaptest_data)) return selection \ No newline at end of file diff --git a/CAT/strategy/MFI_strategy.py b/CAT/strategy/MFI_strategy.py index 88f34ca..fffd77c 100644 --- a/CAT/strategy/MFI_strategy.py +++ b/CAT/strategy/MFI_strategy.py @@ -22,6 +22,9 @@ def name(self): def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset): assert hasattr(model, 'get_fisher'), \ 'the models must implement get_fisher method' + assert hasattr(model, 'get_pred'), \ + 'the models must implement get_pred method for accelerating' + pred_all = model.get_pred(adaptest_data) if self.I is None: self.I = [np.zeros((model.model.num_dim, model.model.num_dim))] * adaptest_data.num_students selection = {} @@ -31,7 +34,7 @@ def adaptest_select(self, model: AbstractModel, adaptest_data: AdapTestDataset): untested_dets = [] untested_fisher = [] for qid in untested_questions: - fisher_info = model.get_fisher(sid, qid) + fisher_info = model.get_fisher(sid, qid, pred_all) untested_fisher.append(fisher_info) untested_dets.append(np.linalg.det(self.I[sid] + fisher_info)) j = np.argmax(untested_dets)