Implementation:Huggingface Trl TrlParser GRPOConfig

Property	Value
Implementation Name	TrlParser and GRPOConfig
Library	Huggingface TRL
Type	API Doc
Source Files	trl/scripts/utils.py (L241-389), trl/trainer/grpo_config.py (L20-892), trl/scripts/grpo.py (L63-95)
Import	from trl import GRPOConfig; from trl import TrlParser

Overview

Description

GRPOConfig is a dataclass that extends transformers.TrainingArguments with all parameters specific to Group Relative Policy Optimization training. TrlParser is a subclass of HfArgumentParser that adds YAML config file support and environment variable injection. GRPOScriptArguments extends ScriptArguments with reward function selection for the GRPO CLI script.

Usage

from trl import GRPOConfig, TrlParser, GRPOTrainer

# Programmatic usage
config = GRPOConfig(
    output_dir="./grpo_output",
    num_generations=8,
    max_completion_length=256,
    temperature=1.0,
    beta=0.0,
    epsilon=0.2,
    loss_type="dapo",
    use_vllm=False,
    learning_rate=1e-6,
    gradient_checkpointing=True,
)

# CLI usage with TrlParser
parser = TrlParser(dataclass_types=[GRPOScriptArguments, GRPOConfig, ModelConfig, DatasetMixtureConfig])
script_args, training_args, model_args, dataset_args, _ = parser.parse_args_and_config(
    return_remaining_strings=True
)

Code Reference

Source Location

Class	File	Lines
GRPOConfig	trl/trainer/grpo_config.py	L20-892
TrlParser	trl/scripts/utils.py	L241-389
GRPOScriptArguments	trl/scripts/grpo.py	L63-95

Signature

@dataclass
class GRPOConfig(TrainingArguments):
    # Overridden defaults from TrainingArguments
    learning_rate: float = 1e-6
    logging_steps: float = 10
    gradient_checkpointing: bool = True
    bf16: bool | None = None  # defaults to True if fp16 is not set

    # Model and reference model
    model_init_kwargs: dict | str | None = None
    disable_dropout: bool = False
    cast_lm_head_to_fp32: bool = False

    # Data preprocessing
    remove_unused_columns: bool | None = False
    num_generations: int | None = 8
    num_generations_eval: int | None = None
    max_completion_length: int | None = 256
    ds3_gather_for_generation: bool = True
    shuffle_dataset: bool | None = True

    # Generation control
    generation_batch_size: int | None = None
    steps_per_generation: int | None = None
    temperature: float = 1.0
    top_p: float = 1.0
    top_k: int = 0
    min_p: float | None = None
    generation_kwargs: dict | None = None
    chat_template_kwargs: dict | None = None
    repetition_penalty: float = 1.0
    use_transformers_paged: bool = False
    cache_implementation: str | None = None

    # vLLM acceleration
    use_vllm: bool = False
    vllm_mode: str = "server"
    vllm_model_impl: str = "vllm"
    vllm_enable_sleep_mode: bool = False
    vllm_structured_outputs_regex: str | None = None
    vllm_server_base_url: str | None = None
    vllm_server_host: str = "0.0.0.0"
    vllm_server_port: int = 8000
    vllm_server_timeout: float = 240.0
    vllm_group_port: int = 51216
    vllm_gpu_memory_utilization: float = 0.3
    vllm_max_model_length: int | None = None
    vllm_tensor_parallel_size: int = 1

    # Training
    beta: float = 0.0
    num_iterations: int = 1
    epsilon: float = 0.2
    delta: float | None = None
    epsilon_high: float | None = None
    sapo_temperature_neg: float = 1.05
    sapo_temperature_pos: float = 1.0
    importance_sampling_level: str = "token"
    reward_weights: list[float] | None = None
    multi_objective_aggregation: str = "sum_then_normalize"
    scale_rewards: str = "group"
    loss_type: str = "dapo"
    mask_truncated_completions: bool = False
    sync_ref_model: bool = False
    ref_model_mixup_alpha: float = 0.6
    ref_model_sync_steps: int = 512
    top_entropy_quantile: float = 1.0
    max_tool_calling_iterations: int | None = None
    vllm_importance_sampling_correction: bool = True
    vllm_importance_sampling_mode: str = "sequence_mask"
    vllm_importance_sampling_cap: float = 3.0
    off_policy_mask_threshold: float | None = None
    use_bias_correction_kl: bool = False

    # Logging
    log_completions: bool = False
    num_completions_to_print: int | None = None
    log_unique_prompts: bool = False
    log_completions_hub_repo: str | None = None

@dataclass
class GRPOScriptArguments(ScriptArguments):
    reward_model_name_or_path: str | None = None
    reward_funcs: list[str] | None = None

class TrlParser(HfArgumentParser):
    def __init__(self, dataclass_types: DataClassType | Iterable[DataClassType] | None = None, **kwargs):
        ...

    def parse_args_and_config(
        self,
        args: Iterable[str] | None = None,
        return_remaining_strings: bool = False,
        fail_with_unknown_args: bool = True,
    ) -> tuple[DataClass, ...]:
        ...

    def set_defaults_with_config(self, **kwargs) -> list[str]:
        ...

Import

from trl import GRPOConfig
from trl import TrlParser
from trl import ScriptArguments

I/O Contract

Inputs

Parameter	Type	Default	Description
num_generations	int	8	Number of completions per prompt (G in the paper). Minimum 2.
max_completion_length	int	256	Maximum token length for generated completions.
temperature	float	1.0	Sampling temperature for generation diversity.
beta	float	0.0	KL coefficient. 0.0 disables reference model loading.
epsilon	float	0.2	Lower clipping bound for the importance-sampling ratio.
loss_type	str	"dapo"	Loss normalization strategy: grpo, dr_grpo, dapo, bnpo, cispo, sapo.
use_vllm	bool	False	Whether to use vLLM for accelerated generation.
scale_rewards	str	"group"	Reward scaling: "group", "batch", or "none".

Outputs

Output	Type	Description
Parsed config	GRPOConfig	Validated configuration object passed to GRPOTrainer.
Parsed script args	GRPOScriptArguments	Reward function names and model paths for the training script.

Usage Examples

YAML config file:

# grpo_config.yaml
output_dir: ./grpo_output
num_generations: 16
max_completion_length: 1024
temperature: 0.9
beta: 0.001
epsilon: 0.2
epsilon_high: 0.28
loss_type: dapo
mask_truncated_completions: true
use_vllm: true
vllm_mode: server
vllm_server_host: "0.0.0.0"
vllm_server_port: 8000
learning_rate: 1.0e-6
per_device_train_batch_size: 4
gradient_accumulation_steps: 4

CLI invocation:

python -m trl grpo \
    --config grpo_config.yaml \
    --model_name_or_path Qwen/Qwen2.5-7B-Instruct \
    --reward_funcs accuracy_reward think_format_reward \
    --dataset_name trl-lib/DeepMath-103K

Related Pages

• Principle:Huggingface_Trl_GRPO_Argument_Configuration