It’s all about how you’re handling the decrypted data and verifying its type, try this code, I commented every part:

PHP_FUNCTION(my_openssl_public_decrypt) {
    zend_string *data, *public_key;
    zval func_name, result;
    zval params[3];
    zval decrypted_zval;

    if (zend_parse_parameters(ZEND_NUM_ARGS(), "SS", &data, &public_key) == FAILURE) {
        RETURN_FALSE;
    }

    // Decode Base64
    zend_string *raw_data = php_base64_decode_ex((const unsigned char *)(ZSTR_VAL(data)), ZSTR_LEN(data), 0);
    if (!raw_data) {
        php_error_docref(NULL, E_WARNING, "Failed to decode Base64 data");
        RETURN_FALSE;
    }

    // Prepare parameters
    ZVAL_STR(&params[0], raw_data);
    ZVAL_NULL(&decrypted_zval);
    ZVAL_NEW_REF(&params[1], &decrypted_zval);
    ZVAL_STR(&params[2], public_key);

    // Call openssl_public_decrypt
    ZVAL_STR(&func_name, zend_string_init("openssl_public_decrypt", strlen("openssl_public_decrypt"), 0));
    if (call_user_function(EG(function_table), NULL, &func_name, &result, 3, params) == FAILURE) {
        php_error_docref(NULL, E_WARNING, "Failed to call openssl_public_decrypt");
        RETURN_FALSE;
    }

    // Check if decryption was successful
    if (Z_TYPE(result) != IS_TRUE) {
        php_error_docref(NULL, E_WARNING, "openssl_public_decrypt failed");
        RETURN_FALSE;
    }

    // Verify decrypted data
    if (Z_TYPE_P(Z_REFVAL_P(&params[1])) != IS_STRING) {
        php_error_docref(NULL, E_WARNING, "Decrypted data is not a string");
        RETURN_FALSE;
    }

    // Return decrypted string
    RETVAL_STR(Z_STR_P(Z_REFVAL_P(&params[1])));
}