From 262101197062cf1051486edcb6c815a83e1995b3 Mon Sep 17 00:00:00 2001
From: Maxime MORGE <maxime.morge@univ-lille.fr>
Date: Wed, 5 Mar 2025 16:07:05 +0100
Subject: [PATCH] Add LLM based strategies for the ring-network game
---
.idea/csv-editor.xml | 42 +++++++++++++++++++++++
README.md | 81 +++++++++++++++++++++++++++-----------------
src/ring/ring.py | 43 +++++++++++++++++++++--
3 files changed, 132 insertions(+), 34 deletions(-)
diff --git a/.idea/csv-editor.xml b/.idea/csv-editor.xml
index 4fb626a..eb269ce 100644
--- a/.idea/csv-editor.xml
+++ b/.idea/csv-editor.xml
@@ -87,6 +87,48 @@
</Attribute>
</value>
</entry>
+ <entry key="$PROJECT_DIR$/data/dictator/dictator.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
+ <entry key="$PROJECT_DIR$/data/ring/ring.1.a.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
+ <entry key="$PROJECT_DIR$/data/ring/ring.1.b.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
+ <entry key="$PROJECT_DIR$/data/ring/ring.1.c.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
+ <entry key="$PROJECT_DIR$/data/ring/ring.1.d.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
+ <entry key="$PROJECT_DIR$/data/ring/ring.2.a.csv">
+ <value>
+ <Attribute>
+ <option name="separator" value="," />
+ </Attribute>
+ </value>
+ </entry>
</map>
</option>
</component>
diff --git a/README.md b/README.md
index 037f6ed..6b6d763 100644
--- a/README.md
+++ b/README.md
@@ -80,8 +80,8 @@ they exhibit perfect alignment with the predefined preferences except for DeepSe
which does not generate valid code.
When models are prompted to generate actions, GPT-4.5 consistently aligns well across all preferences
but struggles with utilitarianism when generating actions.
-Llama3 performs well for selfish and altruistic preferences but shows weaker alignment f
-or utilitarian and egalitarian choices.
+Llama3 performs well for selfish and altruistic preferences but shows weaker alignment for
+utilitarian and egalitarian choices.
Mistral-small aligns best with altruistic preferences and maintains moderate performance on utilitarianism,
but struggles with selfish and egalitarian preferences.
Deepseek-r1 performs best for utilitarianism but has poor accuracy in other categories.
@@ -90,25 +90,23 @@ Bad action selections can be explained either by arithmetic errors (e.g., it is
or by misinterpretations of preferences (e.g., ‘I’m choosing to prioritize the common interest by keeping a
relatively equal split with the other player’).
+## Rationality
-
-## Ring-network game
-
-A player is rational if she plays a best response to her beliefs.
+An autonomous agent is rational if she plays a best response to her beliefs.
She satisfies second-order rationality if she is rational and also believes that others are rational.
In other words, a second-order rational agent not only considers the best course of action for herself
but also anticipates how others make their decisions.
-The experiments conduct by Kneeland (2015) demonstrate that 93% of the subjects are rational,
-while 71% exhibit second-order rationality.
+To assess players’ first- and second-order rationality, we consider a simplified version of the
+ring-network game introduced by Kneeland (2015). His experiments conduct by Kneeland (2015)
+demonstrate that 93% of the subjects are rational, while 71% exhibit second-order rationality.
**[Identifying Higher-Order Rationality](https://doi.org/10.3982/ECTA11983)**
Terri Kneeland (2015) Published in *Econometrica*, Volume 83, Issue 5, Pages 2065-2079
DOI: [10.3982/ECTA11983](https://doi.org/10.3982/ECTA11983)
-Ring games are designed to isolate the behavioral implications of different levels of rationality.
-To assess players’ first- and second-order rationality, we consider a simplified version of the ring-network game.
-This game features two players, each with two available strategies, where both players aim to maximize their own payoff.
+This game features two players, each with two available strategies, where
+both players aim to maximize their own payoff.
The corresponding payoff matrix is shown below:
| Player 1 \ Player 2 | Strategy A | Strategy B |
@@ -135,18 +133,26 @@ We set up three forms of belief:
- *explicit* belief which analyze actions of Player 2 (B is strictly dominated by A).
- *given* belief* where optimal action of Player 1is explicitly provided in the prompt;
-### Player 2
+### First order rationality
The models evaluated include Gpt-4.5-preview-2025-02-27, Mistral-Small, Llama3, and DeepSeek-R1.
The results indicate how well each model performs under each belief type.
-| Model | Given | Explicit | Implicit |
-|----------------|---------|-----------|----------|
-| gpt-4.5 | 1.00 | 1.00 | 1.00 |
-| mistral-small | 1.00 | 1.00 | 0.87 |
-| llama3 | 1.00 | 0.90 | 0.17 |
-| deepseek-r1 | 0.83 | 0.57 | 0.60 |
-
+| *Model* | *Generation* | *Given* | *Explicit* | *Implicit* |
+|-----------------|--------------|---------|------------|------------|
+| *gpt-4.5* | *strategy* | 1.00 | 1.00 | 1.00 |
+| *mistral-small* | *strategy* | 1.00 | 1.00 | 1.00 |
+| *llama3* | *strategy* | 0.5 | 0.5 | 0.5 |
+| *deepseek-r1* | *strategy* | - | - | - |
+| *gpt-4.5* | *actions* | 1.00 | 1.00 | 1.00 |
+| *mistral-small* | *actions* | 1.00 | 1.00 | 0.87 |
+| *llama3* | *actions* | 1.00 | 0.90 | 0.17 |
+| *deepseek-r1* | *actions* | 0.83 | 0.57 | 0.60 |
+
+When the models generate strategies instead of selecting individual actions, GPT-4.5 and
+Mistral-Small exhibit a rational behaviour while Llama3 use a random strategy.
+DeepSeek-R1 does not generate valid code.
+When the models generates individual actions instead of a strategy,
GPT-4.5 achieves a perfect score across all belief types,
demonstrating an exceptional ability to take rational decisions, even in the implicit belief condition.
Mistral-Small consistently outperforms the other open-weight models across all belief types.
@@ -157,7 +163,7 @@ suggesting it may struggle to infer optimal actions solely from natural language
DeepSeek-R1 shows the weakest performance, particularly with explicit beliefs,
indicating it may not be a good candidate to simulate rationality as the other models.
-### Player 1
+### Second-order rationality
In order to adjust the difficulty of taking the optimal
action, we consider 4 versions of the player’s payoff matrix:
@@ -173,14 +179,27 @@ action, we consider 4 versions of the player’s payoff matrix:
-| Model | | Given (a) | Explicit (a) | Implicit (a) | | Given (b) | Explicit (b) | Implicit (b) | | Given (c) | Explicit (c) | Implicit (c) | | Given (d) | Explicit (d) | Implicit (d) |
-|---------------|-|-----------|--------------|--------------|-|-----------|--------------|--------------|--|-----------|--------------|--------------|--|-----------|--------------|--------------|
-| gpt4-.5 | | 1.00 | 1.00 | 1.00 | | 1.00 | 0.67 | 0.00 | | 0.86 | 0.83 | 0.00 | | 0.50 | 0.90 | 0.00 |
-| llama3 | | 0.97 | 1.00 | 1.00 | | 0.77 | 0.80 | 0.60 | | 0.97 | 0.90 | 0.93 | | 0.83 | 0.90 | 0.60 |
-| mistral-small | | 0.93 | 0.97 | 1.00 | | 0.87 | 0.77 | 0.60 | | 0.77 | 0.60 | 0.70 | | 0.73 | 0.57 | 0.37 |
-| deepseek-r1 | | 0.80 | 0.53 | 0.57 | | 0.67 | 0.60 | 0.53 | | 0.67 | 0.63 | 0.47 | | 0.70 | 0.50 | 0.57 |
+| Model | Generation | Given (a) | Explicit (a) | Implicit (a) | | Given (b) | Explicit (b) | Implicit (b) | | Given (c) | Explicit (c) | Implicit (c) | | Given (d) | Explicit (d) | Implicit (d) |
+|---------------|--------------|-------------|----------------|----------------|-|-------------|----------------|----------------|--|-------------|----------------|----------------|--|-------------|----------------|----------------|
+| gpt4-.5 | strategy | 1.00 | 1.00 | 1.00 | | 0.00 | 0.00 | 0.00 | | 1.00 | 1.OO | 1.00 | | 1.00 | 1.00 | 1.00 |
+| llama3 | strategy | 0.50 | 0.50 | 0.50 | | 0.50 | 0.50 | 0.50 | | 0.50 | 0.50 | 0.50 | | 0.50 | 0.50 | 0.50 |
+| mistral-small | strategy | 1.00 | 1.00 | 1.00 | | 1.00 | 1.00 | 1.00 | | 1.00 | 1.00 | 1.00 | | 1.00 | 1.00 | 1.00 |
+| deepseek-r1 | strategy | - | - | - | | - | - | - | | - | - | - | | - | - | - |
+|---------------| ------------ | ----------- | -------------- | -------------- |-| ----------- | -------------- | -------------- |--| ----------- | -------------- | -------------- |--| ----------- | -------------- | -------------- |
+| gpt4-.5 | actions | 1.00 | 1.00 | 1.00 | | 1.00 | 0.67 | 0.00 | | 0.86 | 0.83 | 0.00 | | 0.50 | 0.90 | 0.00 |
+| llama3 | actions | 0.97 | 1.00 | 1.00 | | 0.77 | 0.80 | 0.60 | | 0.97 | 0.90 | 0.93 | | 0.83 | 0.90 | 0.60 |
+| mistral-small | actions | 0.93 | 0.97 | 1.00 | | 0.87 | 0.77 | 0.60 | | 0.77 | 0.60 | 0.70 | | 0.73 | 0.57 | 0.37 |
+| deepseek-r1 | actions | 0.80 | 0.53 | 0.57 | | 0.67 | 0.60 | 0.53 | | 0.67 | 0.63 | 0.47 | | 0.70 | 0.50 | 0.57 |
+
+
+When the model generate strategies, GPT-4.5 performs perfectly in the setups (a), (c) and (b) but
+fails in setup (b) in differentiating the optimal strategy from a near-optimal one.
+Llama3 adopt a random approach to decision-making rather than a structured understanding of rationality.
+Mistral-Small consistently achieves a 100% success rate across all setups, demonstrating robust reasoning abilities.
+DeepSeek-R1 does not produce valid responses, further reinforcing that it may not be a viable candidate
+for generating rational strategies.
-GPT-4.5 achieves perfect performance in the standard (a) setup but struggles significantly with implicit belief
+When they generates individual actions, GPT-4.5 achieves perfect performance in the standard (a) setup but struggles significantly with implicit belief
when the payoff structure changes (b, c, d). This suggests that while it excels when conditions are straightforward,
it is confused by the altered payoffs.
LLama3 demonstrates the most consistent and robust performance, capable of adapting to various belief types
@@ -189,10 +208,10 @@ Mistral-Small, while performing well with given and explicit beliefs, faces chal
DeepSeek-R1 appears to be the least capable, suggesting it may not be an ideal candidate for modeling second-order rationality.
-## Guess the Next Move
+## Belief
-In order to evaluate the ability of LLMs to predict the opponent’s next move, we consider a
-simplified version of the Rock-Paper-Scissors game.
+In order to evaluate the ability of LLMs to refine belief by predicting the opponent’s next move,
+we consider a simplified version of the Rock-Paper-Scissors game.
Rules:
1. The opponent follows a hidden strategy (repeating pattern).
@@ -221,7 +240,7 @@ adopts a more complex pattern. Neither Llama3 nor DeepSeek-R1 were able to gener
-## Rock-Paper-Scissors
+## From belief to action
To evaluate the ability of LLMs to predict not only the opponent’s next move but also to act rationally
based on their prediction, we consider the Rock-Paper-Scissors (RPS) game.
diff --git a/src/ring/ring.py b/src/ring/ring.py
index 10fc059..6f1679d 100644
--- a/src/ring/ring.py
+++ b/src/ring/ring.py
@@ -2,13 +2,13 @@ import os
import asyncio
from typing import Dict, Literal
-from networkx.algorithms.threshold import swap_d
from pydantic import BaseModel
from autogen_agentchat.agents import AssistantAgent
from autogen_agentchat.messages import TextMessage
from autogen_core import CancellationToken
from autogen_ext.models.openai import OpenAIChatCompletionClient
import json
+import random
from torchgen.dest.ufunc import eligible_for_binary_scalar_specialization
@@ -30,7 +30,7 @@ class AgentResponse(BaseModel):
class Ring:
debug=False
- def __init__(self, player_id: int, belief: Belief, swap: bool, version: str, model: str, temperature: float, max_retries: int = 3):
+ def __init__(self, player_id: int, belief: Belief, swap: bool, version: str, model: str, temperature: float, strategy = False, max_retries: int = 3):
self.player_id = player_id
self.belief = belief
self.swap = swap
@@ -38,6 +38,7 @@ class Ring:
self.version = version
self.model = model
self.temperature = temperature
+ self.strategy = strategy
self.max_retries = max_retries # Maximum retry attempts in case of hallucinations
is_openai_model = model.startswith("gpt")
@@ -62,6 +63,9 @@ class Ring:
async def run(self) -> Dict:
"""Runs the model and ensures a valid response."""
+ if self.strategy:
+ return self.apply_strategy()
+ # (Rest of the method continues using the model-based approach)
action_description = (
' - `"action"`: Your move ("A" or "B")' if self.player_id == 2
else ' - `"action"`: Your move ("X" or "Y")'
@@ -149,9 +153,42 @@ class Ring:
else:
return agent_response.action == self.X
+ def apply_strategy(self) -> Dict[str, str]:
+ """Applies a heuristic-based strategy instead of relying on the model if strategy is enabled."""
+ if self.model == "gpt-4.5-preview-2025-02-27":
+ if self.strategy:
+ if self.player_id == 2:
+ action = self.A # Always choose A, as B is strictly dominated
+ reasoning = f"Choosing {self.A} because {self.B} is strictly dominated and rational players avoid dominated strategies."
+ else:
+ action = self.X if self.version in ["a", "c", "d"] else self.Y
+ reasoning = f"Choosing {action} based on the given game structure and expected rational behavior from Player 2."
+ if self.model == "llama3":
+ if self.player_id == 1:
+ action = self.X if random.random() < 0.5 else self.Y
+ reasoning = "The reasoning behind this choice is..."
+ elif self.player_id == 2:
+ action = self.B if random.random() < 0.5 else self.A
+ reasoning = "The reasoning behind this choice is..."
+ if self.model == "mistral-small":
+ #Always choose 'A' or 'X' based on player_id
+ if self.player_id == 1:
+ action = "X"
+ reasoning = f"Player {self.player_id} always chooses X as per the predefined strategy."
+ elif self.player_id == 2:
+ action = "B"
+ reasoning = f"Player {self.player_id} always chooses B as per the predefined strategy."
+ # Validate the rationality of the chosen action
+ rational = 1.0 if self.check_rationality(AgentResponse(action=action, reasoning=reasoning)) else 0.0
+ return {
+ "action": action,
+ "rationality": rational,
+ "reasoning": reasoning
+ }
+
# Run the async function and return the response
if __name__ == "__main__":
- game_agent = Ring(1, Belief.IMPLICIT, swap = True, version="a", model="llama3", temperature=0.7)
+ game_agent = Ring(1, Belief.IMPLICIT, swap = True, version="b", model="mistral-small", temperature=0.7, strategy = True)
response_json = asyncio.run(game_agent.run())
print(response_json)
\ No newline at end of file
--
GitLab