@article{howard_integration_2023,
	title = {Integration of clinical features and deep learning on pathology for the prediction of breast cancer recurrence assays and risk of recurrence},
	volume = {9},
	copyright = {2023 The Author(s)},
	issn = {2374-4677},
	url = {https://www.nature.com/articles/s41523-023-00530-5},
	doi = {10.1038/s41523-023-00530-5},
	abstract = {Gene expression-based recurrence assays are strongly recommended to guide the use of chemotherapy in hormone receptor-positive, HER2-negative breast cancer, but such testing is expensive, can contribute to delays in care, and may not be available in low-resource settings. Here, we describe the training and independent validation of a deep learning model that predicts recurrence assay result and risk of recurrence using both digital histology and clinical risk factors. We demonstrate that this approach outperforms an established clinical nomogram (area under the receiver operating characteristic curve of 0.83 versus 0.76 in an external validation cohort, p = 0.0005) and can identify a subset of patients with excellent prognoses who may not need further genomic testing.},
	language = {en},
	number = {1},
	urldate = {2023-04-17},
	journal = {npj Breast Cancer},
	author = {Howard, Frederick M. and Dolezal, James and Kochanny, Sara and Khramtsova, Galina and Vickery, Jasmine and Srisuwananukorn, Andrew and Woodard, Anna and Chen, Nan and Nanda, Rita and Perou, Charles M. and Olopade, Olufunmilayo I. and Huo, Dezheng and Pearson, Alexander T.},
	month = apr,
	year = {2023},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Breast cancer, Prognostic markers, Translational research},
	pages = {1--6},
}

"slides": location of the whole slide images
"roi": location of region of interest annotations. We have provided our region of interest annotations from TCGA in the /roi/ directory
"tiles": location to extract free image tiles. We disable this in our extract image function
"tfrecords": location of tfrecords containing the extracted image tiles for slides

python model_training.py -e

SFP = sf.Project(join(PROJECT_ROOT, "UCH_RS")) # specifies the location of the project folder
SFP.annotations = join(PROJECT_ROOT, "UCH_RS", "uch_brca_complete.csv") # specifies the location of the annotations file to use as a guide for slides to extract
SFP.sources = ["UCH_BRCA_RS"] # specifies the dataset location
SFP.extract_tiles(tile_px=tile_px, tile_um=tile_um, skip_missing_roi=False, save_tiles=False, skip_extracted=overwrite, roi_method = 'ignore', source="UCH_BRCA_RS", buffer=join(PROJECT_ROOT, "buffer")) # extracts tiles for the 'UCH_BRCA_RS' dataset using the specified tile pixel size (302), tile size in microns (299), without using regions of interest

UCH_BRCA_RS - extracted tiles from University of Chicago slides. These should be placed in the appropriate tfrecords directory from the datasets.json using the anonymized files uploaded to Zenodo
UCH_BRCA_RS_FULL_ROI - extracted tiles from University of Chicago slides using pathologist specified regions of interest - an optional dataset used to assess training models from University of Chicago and validating on TCGA, not used for primary analysis of this project. 
TCGA_BRCA_FULL_ROI - extracted tiles from TCGA using specified regions of interest
TCGA_BRCA_NO_ROI - extracted tiles from TCGA without regions of interest
TCGA_BRCA_NORMAL - extracted tiles from TCGA using inverse region of interest (essentially the normal tissue surrounding tumors in the dataset, used for training the tumor likelihood model)
TCGA_BRCA_FILTERED - a dataset where tiles are filtered from the TCGA using the tumor detection module rather than pathologist annotations - an optional dataset not used for the primary analysis of this project

python model_training.py --hpsearch run --hpprefix DESIRED_PREFIX --hpstart 1 --hpcount 50

Dropout: 0 - 0.5
Hidden Layer Width (dimension of fully connected layers after Xception backbone): 128 - 1024
Hidden Layers (number of fully connected hidden layers): 1 - 5
Learning Rate: 0.00001 - 0.001
Learning Rate Decay Steps (# of batches until applying learning rate decay): 128 - 1024
Learning Rate Decay Ratio (ratio with which to reduce the learning rate): 0 - 1
Learning Rate Decay (whether to decay learning rate): True / False
Loss (for linear loss, will train models on numerical recurrence score; for categorical loss will train on high / low cutoff of recurrence score): mean squared error, mean absolute error, sparse categorical cross entropy
Batch Size: 8 - 128
Augment (‘x’ performs horizontal flipping, ‘y’ performs vertical flipping, ‘r’ performs rotation, ‘j’ performs random JPEG compression): xyr, xyrj, xyrjb
Normalizer (whether to apply stain normalization): reinhard, none
L1 weight: 0 - 0.1
L1 (whether to perform L1 regularization): True, False
L2 weight: 0 - 0.1
L2 (whether to perform L2 regularization): True, False
L1 dense weight: 0 - 0.1
L1 dense (whether to perform l1 regularization to dense layers): True, False
L2 dense weight: 0 - 0.1
L2 dense (whether to perform l1 regularization to dense layers): True, False

python model_training.py -t --hpsearch read --hpprefix DESIRED_PREFIX --hpstart 0 --hpcount 50

python model_training.py -t --hpsearch old

python model_training.py -t --hpsearch old -uf #The -uf or --use_filtered command will train models using tiles selected by the tumor-likelihood model instead of from pathologist annotations
python model_training.py -t --hpsearch old -tr #The -tr or --train_receptors command will train models using only HR+/HER2- patients from TCGA
python model_training.py -t --hpsearch old -rev #The -rev or --train_reverse command will train models on the UCMC dataset for validation in TCGA

SFP = sf.Project(join(PROJECT_ROOT, "UCH_RS"))
SFP.annotations = join(PROJECT_ROOT, "UCH_RS", "tcga_brca_complete.csv")
SFP.sources = ["TCGA_BRCA_FULL_ROI"]
hp = hp_opt #uses saved optimal hyperparameters from optimization
odx_train_name = "GHI_RS_Model_NJEM.2004_PMID.15591335" #numerical OncotypeDx score in annotation file
odx_val_name = "odx85" #categorical representation of OncotypeDx score in annotation file
exp_label = "ODX_Final_BRCAROI"

SFP_TUMOR_ROI = sf.Project(join(PROJECT_ROOT, "TCGA_BRCA_ROI")) # need to setup the slideflow project for where tumor region of interest predictions are stored
SFP_TUMOR_ROI.annotations = join(PROJECT_ROOT, "TCGA_BRCA_ROI", "tumor_roi.csv")
hp.weight_model = assign_tumor_roi_model(SFP_TUMOR_ROI, hp.tile_px, hp.normalizer) #assigns the tumor likelihood model (only used for validation)
SFP.train(exp_label=exp_label, outcome_label_headers=odx_train_name,  val_outcome_label_headers=odx_val_name, params = hp, filters = mergeDict(filters, {odx_val_name: ["H","L"]}), val_strategy = 'k-fold-manual', val_k_fold=3, val_k_fold_header = "CV3_odx85_mip", multi_gpu=True, save_predictions=True)

#Key arguments used in our slideflow training setup -
#exp_label - defaults to ODX_Final_BRCAROI - saved label for the experiment
#outcome_label_headers - defaults to the recurrence score header in the TCGA dataset - GHI_RS_Model_NJEM.2004_PMID.15591335 for Oncotype and Pcorr_NKI70_Good_Correlation_Nature.2002_PMID.11823860 for MammaPrint
#params - assigns model hyperparameters as described above
#filters - specify ranges allowed for specific columns in the annotations file
#val_strategy - set to k-fold-manual to allow k-fold cross validation with a consistent set of preserved site folds as generated above and included in our annotations file
#val_k_fold_header - specifies the header that assigns specific k-fold to each patient
#multi_gpu - allows training to utilize all available GPUs
#save_predictions - to save the outcome of model training

model_training.py -v

#Set up a slideflow project referencing the validation dataset annotations and dataset source
SFP = sf.Project(join(PROJECT_ROOT, "UCH_RS"))
SFP.annotations = join(PROJECT_ROOT, "UCH_RS", "uch_brca_complete.csv")
SFP.sources = ["UCH_BRCA_RS"]

#Set up the slideflow project for where tumor region of interest predictions are performed
SFP_TUMOR_ROI = sf.Project(join(PROJECT_ROOT, "TCGA_BRCA_ROI"))
SFP_TUMOR_ROI.annotations = join(PROJECT_ROOT, "TCGA_BRCA_ROI", "tumor_roi.csv")

#Load saved hyperparameters and assign the tumor likelihood model (weight_model) hyperparameter
hp = hp_opt 
hp.weight_model = assign_tumor_roi_model(SFP_TUMOR_ROI, hp.tile_px, hp.normalizer)

#Specifiy experiment names which were used for model training, and specify the column names used for outcome from training and validation
exp_label = "ODX_Final_BRCAROI"
odx_train_name = "GHI_RS_Model_NJEM.2004_PMID.15591335"
odx_val_name = "RSHigh"

#Finds the trained model within the SFP project folder (in this case /PROJECTS/UCH_RS/models/ folder)
m = find_model(SFP, exp_label, outcome = odx_train_name, epoch=hp.epochs[0])

#re-assign hyperparameters for categorical validation of a linear model
params = sf.util.get_model_config(m)
params["hp"]["loss"] = "sparse_categorical_crossentropy"
params["model_type"] = "categorical"
params["outcome_labels"] = {"0":"H","1":"L"}
params["onehot_pool"] = 'false'
sf.util.write_json(params, join(m, "params_eval.json"))

#Run OncotypeDx model validation
SFP.evaluate(model=m, outcome_label_headers=odx_val_name, save_predictions=True, model_config=join(m, "params_eval.json"))

python model_analysis.py -s #The -s or --saved command will use saved pathologic model predictions from /PROJECT/saved_results/ and embedded in the uch_brca_complete.csv and tcga_brca_complete.csv files (rather then reloading the predictions from the /PROJECTS/UCH_RS/eval/ directory).
python model_analysis.py -uf #The -uf or --use_filtered command will train models using tiles selected by the tumor-likelihood model instead of from pathologist 
annotations
python model_analysis.py -tr #The -tr or --train_receptors command will train models using only HR+/HER2- patients from TCGA
python model_analysis.py -rev #The -rev or --train_reverse command will train models on the UCMC dataset for validation in TCGA

patient - patient identifier
slide - slide file name
Age - numeric age in years - for calculation of clinical nomogram
hist_type - histologic subtype - Ductal, Lobular, D&L, or Other - for calculation of clinical nomogram
grade - 1, 2, or 3 - for calculation of clinical nomogram
tumor_size - tumor size measured in mm - for calculation of clinical nomogram
PR  - 'Pos', or 'Neg', for calculation of clinical nomogram

python model_training.py --extract --annotation <annotation.csv file name, assumed to be in /PROJECTS/UCH/> --source <dataset name specified in dataset.json> --experiment_label <unique label for experiment; duplicate models with same prefix leads to errors>
python model_training.py --validate --annotation <annotation.csv file name, assumed to be in /PROJECTS/UCH/> --source <dataset name specified in dataset.json> --experiment_label <label>

python model_analysis.py -pred --outcome <RS for Oncotype or MP for MammaPrint> --dataset <name of CSV file with annotations in the UCH_RS folder>  --experiment_label <label>

python model_training.py --heatmaps_tumor_roi TCGA
python model_training.py --heatmaps_odx TCGA