Unverified Commit b7423038 by Yuchen Zhang Committed by GitHub

Resolve the issue of PRIME getting stuck during math verification. (#469)

Since searching for an appropriate `simplify` algorithm may cause
`sympy.simplify` to timeout, and `ProcessPool` may get stuck due to
excessive concurrency, the timeout mechanism in
`verl/verl/workers/reward_manager/prime.py` cannot capture the timeout.
To address this issue, a timeout detection mechanism is added to
`verl/verl/utils/reward_score/prime_math/__init__.py` for
`sympy.simplify` to solve it easily.
parent 386cfabe
......@@ -22,7 +22,7 @@ import re
import sympy
from pylatexenc import latex2text
from sympy.parsing import sympy_parser
import os
from . import math_normalize
from .grader import math_equal
......@@ -35,6 +35,33 @@ BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"]
TUPLE_CHARS = "()[]"
def timeout(timeout_seconds: int = 8):
if os.name == "posix":
import signal
def decorator(func):
def handler(signum, frame):
raise TimeoutError("Operation timed out!")
def wrapper(*args, **kwargs):
old_handler = signal.getsignal(signal.SIGALRM)
signal.signal(signal.SIGALRM, handler)
signal.alarm(timeout_seconds)
try:
return func(*args, **kwargs)
finally:
signal.alarm(0)
signal.signal(signal.SIGALRM, old_handler)
return wrapper
return decorator
else:
raise NotImplementedError(f"Unsupported OS: {os.name}")
def _sympy_parse(expr: str):
"""Parses an expression with sympy."""
py_expr = expr.replace("^", "**")
......@@ -209,6 +236,7 @@ def should_allow_eval(expr: str):
return True
@timeout(timeout_seconds=10)
def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str):
are_equal = False
try:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment