benchmark docs (#109)

* benchmark docs and cleanup * file formatting
dsg-uwaterloo · adriandavila · Dec 29, 2023 · Sep 22, 2023 · Sep 25, 2023 · Sep 28, 2023
commit f33a72b2840901a70403ede1a04f1bedbbe2652f
diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md
@@ -0,0 +1,26 @@
+# ORTOA-TEE Benchmarking
+
+```txt
+usage: main.py [-h] [-e EXPERIMENTS [EXPERIMENTS ...]] [-d EXPERIMENT_DIRS [EXPERIMENT_DIRS ...]] [-w WORKING_DIR]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -w WORKING_DIR, --working-dir WORKING_DIR
+                        Directory to use as base for experiment directory tree (default: out/benchmark-2023-12-29-11-38-05)
+
+Experiments:
+  Options to control experiments selected for compilation
+
+  -e EXPERIMENTS [EXPERIMENTS ...], --experiments EXPERIMENTS [EXPERIMENTS ...]
+                        List of experiments to compile (experiment name should match zoo object)
+  -d EXPERIMENT_DIRS [EXPERIMENT_DIRS ...], --experiment-dirs EXPERIMENT_DIRS [EXPERIMENT_DIRS ...]
+                        List of local directories to use for experiment files
+```
+
+## Entrypoint
+
+The main entry point is `extras/ortoa/benchmark/infrastructure/main.py`
+
+## Jobs
+
+The `ClientJob` class defined in `extras/ortoa/benchmark/infrastructure/jobs.py` implements the `JobProtocol` class. Most importantly, is specifies how a job is run.
diff --git a/extras/ortoa/benchmark/infrastucture/experiment_collection.py b/extras/ortoa/benchmark/infrastucture/experiment_collection.py
@@ -17,7 +17,6 @@ def model_post_init(self, __context: Any) -> None:
         assert self.experiment_path.is_file()
         return super().model_post_init(__context)
 
-
     @classmethod
     def construct_experiments(cls, experiment: Path) -> List[Self]:
         """
@@ -38,9 +37,7 @@ def from_path(cls, experiment: Path) -> List[Self]:
 
     @classmethod
     def from_dir(cls, experiment_dir: Path) -> List[Self]:
-        return [
-            cls(experiment_path=e) for e in experiment_dir.glob("**/*.yaml")
-        ]
+        return [cls(experiment_path=e) for e in experiment_dir.glob("**/*.yaml")]
 
 
 def collect_experiments(experiments: Iterable[Path]) -> List[ExperimentPath]:
@@ -49,6 +46,9 @@ def collect_experiments(experiments: Iterable[Path]) -> List[ExperimentPath]:
     """
     return list(
         itertools.chain.from_iterable(
-            [ExperimentPath.construct_experiments(experiment) for experiment in experiments]
+            [
+                ExperimentPath.construct_experiments(experiment)
+                for experiment in experiments
+            ]
         )
     )
diff --git a/extras/ortoa/benchmark/infrastucture/main.py b/extras/ortoa/benchmark/infrastucture/main.py
@@ -51,14 +51,6 @@ def parse_args() -> argparse.Namespace:
         required=False,
         help="Directory to use as base for experiment directory tree (default: %(default)s)",
     )
-    parser.add_argument(
-        "-m",
-        "--max-processes",
-        type=int,
-        default=None,
-        required=False,
-        help="Maximum number of processes to use when running experiments (default: %(default)s)",
-    )
 
     args = parser.parse_args()
 
@@ -73,8 +65,6 @@ def parse_args() -> argparse.Namespace:
 def benchmark(
     experiment_base: Path,
     experiment_names: List[Path],
-    max_processes: Optional[int] = None,
-    log_errors_in_main_thread: bool = False,
 ) -> Stats:
     """Main entrypoint to the benchmarking flow
 
@@ -110,8 +100,6 @@ def benchmark(
     # Orchestrate and the jobs
     orchestration = JobOrchestration(
         jobs=jobs,
-        max_processes=max_processes,
-        log_errors_in_main_thread=log_errors_in_main_thread,
     )
     results = orchestration.run_sequential()
 
@@ -123,9 +111,7 @@ def main():
     args = parse_args()
     args.working_dir.mkdir(parents=True, exist_ok=True)
 
-    stats: Stats = benchmark(
-        args.working_dir, args.experiments + args.experiment_dirs, args.max_processes
-    )
+    stats: Stats = benchmark(args.working_dir, args.experiments + args.experiment_dirs)
 
     stats.save_to(args.working_dir)
 

diff --git a/extras/ortoa/benchmark/infrastucture/runner.py b/extras/ortoa/benchmark/infrastucture/runner.py
@@ -63,8 +63,6 @@ class Config:
         arbitrary_types_allowed = True
 
     jobs: Sequence[JobT]
-    max_processes: Optional[int]
-    log_errors_in_main_thread: bool = False
 
     def model_post_init(self, __context: Any) -> None:
         if len(self.jobs) == 0:

diff --git a/extras/ortoa/benchmark/interface/experiment.py b/extras/ortoa/benchmark/interface/experiment.py
@@ -74,11 +74,21 @@ def generate_data(self) -> None:
                 output_file,
             )
 
-            if self.client_config.data.seed and not self.client_config.data.seed.exists():
-                raise FileNotFoundError(f"Seed file {self.client_config.data.seed} was not found.")
+            if (
+                self.client_config.data.seed
+                and not self.client_config.data.seed.exists()
+            ):
+                raise FileNotFoundError(
+                    f"Seed file {self.client_config.data.seed} was not found."
+                )
 
-            if self.client_config.data.operations and not self.client_config.data.operations.exists():
-                raise FileNotFoundError(f"Operations file {self.client_config.data.operations} was not found.")
+            if (
+                self.client_config.data.operations
+                and not self.client_config.data.operations.exists()
+            ):
+                raise FileNotFoundError(
+                    f"Operations file {self.client_config.data.operations} was not found."
+                )
 
 
 def load_experiments(
@@ -133,7 +143,7 @@ def atomicize_experiments(experiments: List[Experiment]) -> List[AtomicExperimen
         assert isinstance(experiment.client_config.data, SeedData)
         assert experiment.client_config.data.seed is not None
         assert experiment.client_config.data.operations is not None
-        
+
         all_client_flags = [
             flag.get_atomic_flags() for flag in experiment.client_config.flags
         ]

diff --git a/extras/ortoa/benchmark/interface/parameter.py b/extras/ortoa/benchmark/interface/parameter.py
@@ -10,6 +10,7 @@
 # Abstractions
 ##########################
 
+
 class IntType(BaseModel):
     type: Literal["int"] = Field(default="int", frozen=True)
 
@@ -32,6 +33,7 @@ class RangeParameter(Parameter, Generic[NumberT]):
 # Parameter Types
 ##########################
 
+
 class IntegerIncrementRange(RangeParameter[int], IntType):
     step: int