- fixed plys_to_end list to only include values for moves that really…

… have been used - added counter for games without positions for current phase
QueensGambit · QueensGambit · May 3, 2024 · Jun 22, 2023 · Jun 22, 2023 · Jun 24, 2023
commit 826f3b650f84a973f5e97c42abe2dc369a1439b5
diff --git a/DeepCrazyhouse/src/preprocessing/pgn_converter_util.py b/DeepCrazyhouse/src/preprocessing/pgn_converter_util.py
@@ -91,6 +91,7 @@ def get_planes_from_game(game, mate_in_one=False):
     x = []
     y_value = []
     y_policy = []
+    plys_to_end = []  # save the number of plys until the end of the game for each position that was considered
     board = game.board()  # get the initial board state
     # update the y value accordingly
     if board.turn == chess.WHITE:
@@ -109,8 +110,7 @@ def get_planes_from_game(game, mate_in_one=False):
     # you don't want to push the last move on the board because you had no movement policy to learn from in this case
     # The moves get pushed at the end of the for-loop and is only used in the next loop.
     # Therefore we can iterate over 'all' moves
-    plys = 0
-    for move in all_moves:
+    for plys, move in enumerate(all_moves):
         board_occ = 0  # by default the positions hasn't occurred before
         fen = board.fen()
         # remove the halfmove counter & move counter from this fen to make repetitions possible
@@ -147,24 +147,15 @@ def get_planes_from_game(game, mate_in_one=False):
                 # add the next move defined in policy vector notation to the policy list
                 # the network always sees the board as if he's the white player, that's the move is mirrored fro black
                 y_policy.append(move_to_policy(next_move, mirror_policy=mirror_policy(board)))
+                plys_to_end.append(len(all_moves) - 1 - plys)
 
         y_init *= -1  # flip the y_init value after each move
         board.push(move)  # push the next move on the board
-        plys += 1
 
-    plys_to_end = np.arange(plys)[::-1]
-
-    # check if there has been any moves
+    # check if there has been any moves and stack the lists
     if x and y_value and y_policy:
         x = np.stack(x, axis=0)
         y_value = np.stack(y_value, axis=0)
         y_policy = np.stack(y_policy, axis=0)
-    else:
-        #print("game.headers:")
-        #print(game.headers)
-        #print("len(all_moves)", len(all_moves))
-        #print("game", game)
-        return x, y_value, y_policy, plys_to_end
-        #raise Exception("The given pgn file's mainline is empty!")
 
     return x, y_value, y_policy, plys_to_end
diff --git a/DeepCrazyhouse/src/preprocessing/pgn_to_planes_converter.py b/DeepCrazyhouse/src/preprocessing/pgn_to_planes_converter.py
@@ -493,12 +493,13 @@ def export_pgn_batch(self, cur_part, game_idx_start, game_idx_end, pgn_sel, nb_w
 
         logging.info("starting conversion to planes...")
         t_s = time()
-        pool = Pool()
+        pool = Pool(processes=None)  # If processes is None then the number returned by os.cpu_count() is used
         x_dic = {}
         y_value_dic = {}
         y_policy_dic = {}
         plys_to_end_dic = {}
         metadata_dic = {}
+        num_games_without_fitting_moves = 0
 
         if not os.path.exists(self._export_dir):
             os.makedirs(self._export_dir)
@@ -520,6 +521,8 @@ def export_pgn_batch(self, cur_part, game_idx_start, game_idx_end, pgn_sel, nb_w
                 y_value_dic[game_idx] = y_value
                 y_policy_dic[game_idx] = y_policy
                 plys_to_end_dic[game_idx] = plys_to_end
+            else:
+                num_games_without_fitting_moves += 1
         pool.close()
         pool.join()
         t_e = time() - t_s
@@ -547,6 +550,9 @@ def export_pgn_batch(self, cur_part, game_idx_start, game_idx_end, pgn_sel, nb_w
         logging.debug("x.shape %s", x.shape)
         logging.debug("y_value.shape %s", y_value.shape)
         logging.debug("y_policy.shape %s", y_policy.shape)
+        logging.debug("plys_to_end.shape %s", plys_to_end.shape)
+        logging.debug("num_games_without_fitting_moves %s", num_games_without_fitting_moves)
+        assert x.shape[0] == y_value.shape[0] == y_policy.shape[0] == plys_to_end.shape[0]
         # Save the dataset to a file
         logging.info("saving the dataset to a file...")
         # define the compressor object