review_config:
  # User configuration for the review
  # - benchmark - use the user config from the benchmark reviews
  # - <value> - use the value directly
  user_config:
    enable_ai_review: true
    enable_rule_comments: false

    enable_complexity_comments: benchmark
    enable_docstring_comments: benchmark
    enable_security_comments: benchmark
    enable_tests_comments: benchmark
    enable_comment_suggestions: benchmark

    enable_approvals: true

  ai_review_config:
    # The model responses to use for the experiment
    # - benchmark - use the model responses from the benchmark reviews
    # - llm - call the language model to generate responses
    model_responses:
      comments_model: benchmark
      comment_validation_model: benchmark
      comment_suggestion_model: benchmark
      complexity_model: benchmark
      docstrings_model: benchmark
      security_model: benchmark
      tests_model: benchmark

# The pull request dataset to run the experiment on
pull_request_dataset:
- https://github.com/2lambda123/KasperskyLab-TinyCheck/pull/3
- https://github.com/2lambda123/KasperskyLab-TinyCheck/pull/6
- https://github.com/2lambda123/KasperskyLab-TinyCheck/pull/9
- https://github.com/2lambda123/KasperskyLab-klara/pull/2
- https://github.com/2lambda123/KasperskyLab-klara/pull/7
- https://github.com/2lambda123/beego-beego/pull/11
- https://github.com/2lambda123/dexie-Dexie.js/pull/1
- https://github.com/2lambda123/sandstorm-io-sandstorm/pull/3
- https://github.com/2lambda123/sandstorm-io-sandstorm/pull/4
- https://github.com/2lambda123/typestack-class-transformer/pull/10
- https://github.com/2lambda123/typestack-class-transformer/pull/7
- https://github.com/2lambda123/typestack-class-transformer/pull/8
- https://github.com/2lambda123/typestack-typedi/pull/8
- https://github.com/2lambda123/typestack-typeorm-typedi-extensions/pull/2
- https://github.com/2lambda123/typestack-typeorm-typedi-extensions/pull/3
- https://github.com/2lambda123/unitaryai-detoxify/pull/2
- https://github.com/2lambda123/unitaryai-detoxify/pull/4
- https://github.com/Bilbottom/sql-models/pull/5
- https://github.com/CPUT-DEVS/devpost-hackathon/pull/24
- https://github.com/ErosVece/Discounted-Udemy-Course-Enroller/pull/2
- https://github.com/LemurPwned/video-sampler/pull/21
- https://github.com/TeKrop/overfast-api/pull/109
- https://github.com/TeamPGM/PagerMaid_Plugins_Pyro/pull/181
- https://github.com/Xmaster6y/mulsi/pull/11
- https://github.com/adryells/github-followers-manager/pull/3
- https://github.com/albumentations-team/albumentations/pull/1663
- https://github.com/aleph23/Composer/pull/18
- https://github.com/bengosney/cerberus/pull/800
- https://github.com/bikerski-ctrl/trackingtasks/pull/5
- https://github.com/bruAristimunha/moabb/pull/5
- https://github.com/code-Harsh247/FRSS-project/pull/58
- https://github.com/code-Harsh247/FRSS-project/pull/59
- https://github.com/code-Harsh247/FRSS-project/pull/60
- https://github.com/code-Harsh247/FRSS-project/pull/61
- https://github.com/code-Harsh247/FRSS-project/pull/62
- https://github.com/code-Harsh247/FRSS-project/pull/63
- https://github.com/code-Harsh247/FRSS-project/pull/64
- https://github.com/code-Harsh247/FRSS-project/pull/65
- https://github.com/code-Harsh247/FRSS-project/pull/66
- https://github.com/code-Harsh247/FRSS-project/pull/67
- https://github.com/code-Harsh247/FRSS-project/pull/68
- https://github.com/emoss08/Trenova/pull/247
- https://github.com/emoss08/Trenova/pull/248
- https://github.com/emoss08/Trenova/pull/249
- https://github.com/erxes/erxes/pull/5136
- https://github.com/erxes/erxes/pull/5137
- https://github.com/erxes/erxes/pull/5139
- https://github.com/gdsfactory/gdsfactory/pull/2678
- https://github.com/gdsfactory/kfactory/pull/295
- https://github.com/imflop/brew-scout/pull/17
- https://github.com/jackdewinter/pymarkdown/pull/1058
- https://github.com/jquagga/ttt/pull/53
- https://github.com/jquagga/ttt/pull/56
- https://github.com/jquagga/ttt/pull/57
- https://github.com/jquagga/ttt/pull/58
- https://github.com/jquagga/ttt/pull/59
- https://github.com/kloudlite/iot-devices/pull/2
- https://github.com/kloudlite/iot-devices/pull/3
- https://github.com/kloudlite/operator/pull/177
- https://github.com/kurianbenoy/Indic-Subtitler/pull/14
- https://github.com/manoelhc/test-actions/pull/21
- https://github.com/martimlobao/dotfiles/pull/4
- https://github.com/martimlobao/dotfiles/pull/5
- https://github.com/mraniki/dxsp/pull/628
- https://github.com/naya1503/Mix-Userbot/pull/25
- https://github.com/nbhirud/system_update/pull/10
- https://github.com/nbhirud/system_update/pull/9
- https://github.com/nikhilbadyal/docker-py-revanced/pull/496
- https://github.com/nikhilbadyal/docker-py-revanced/pull/498
- https://github.com/nikhilbadyal/docker-py-revanced/pull/499
- https://github.com/noverd/FL/pull/2
- https://github.com/nuxeo/nuxeo-drive/pull/4793
- https://github.com/nuxeo/nuxeo-python-client/pull/305
- https://github.com/otpless-tech/otpless-android-sdk/pull/69
- https://github.com/pay-theory/pay-theory-documentation/pull/81
- https://github.com/pay-theory/pay-theory-documentation/pull/82
- https://github.com/rbanffy/3270font/pull/125
- https://github.com/robbiebyrd/platinum/pull/54
- https://github.com/simnova/ownercommunity/pull/27
- https://github.com/simnova/ownercommunity/pull/32
- https://github.com/simnova/ownercommunity/pull/33
- https://github.com/simnova/ownercommunity/pull/34
- https://github.com/simnova/ownercommunity/pull/35
- https://github.com/skypointcloud/skypoint-colbert/pull/1
- https://github.com/sourcery-ai-experiments/atari-rl/pull/1
- https://github.com/strawberry-graphql/strawberry/pull/3452
- https://github.com/supabase-community/realtime-py/pull/120
- https://github.com/supabase-community/supabase-py/pull/770
- https://github.com/suttacentral/suttacentral/pull/3134
- https://github.com/suttacentral/suttacentral/pull/3135
- https://github.com/synodriver/cycurl/pull/4
- https://github.com/ultralytics/hub/pull/643
- https://github.com/yuceltoluyag/Rise-of-Kingdoms-Bot/pull/17

# Questions to ask to label the review comments
review_comment_labels:
- label: correct
  question: Is this comment correct?
- label: helpful
  question: Is this comment helpful?
- label: comment-type
  question: Is the comment type correct?
- label: comment-area
  question: Is the comment area correct?

# Benchmark reviews generated by running
#   python -m scripts.experiment benchmark <experiment_name>
benchmark_reviews:
- dataset_pull_request: https://github.com/2lambda123/sandstorm-io-sandstorm/pull/4
  review_pull_request: https://github.com/sourcery-ai-experiments/sandstorm-io-sandstorm/pull/1
- dataset_pull_request: https://github.com/2lambda123/typestack-class-transformer/pull/10
  review_pull_request: https://github.com/sourcery-ai-experiments/typestack-class-transformer/pull/1
- dataset_pull_request: https://github.com/2lambda123/typestack-class-transformer/pull/7
  review_pull_request: https://github.com/sourcery-ai-experiments/typestack-class-transformer/pull/3
- dataset_pull_request: https://github.com/2lambda123/typestack-class-transformer/pull/8
  review_pull_request: https://github.com/sourcery-ai-experiments/typestack-class-transformer/pull/2
- dataset_pull_request: https://github.com/Bilbottom/sql-models/pull/5
  review_pull_request: https://github.com/sourcery-ai-experiments/sql-models/pull/1