Skip to content
Snippets Groups Projects
Commit dbd50f20 authored by Maxime Morge's avatar Maxime Morge :construction_worker:
Browse files

Ivestment XP with Pagoda

parent 8cfacf81
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
.idea/PyGAAMAS.iml
+ 3
− 3
View file @ dbd50f20
<?xml version="1.0" encoding="UTF-8"?>
<module version="4">
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
<component name="NewModuleRootManager">
<orderEntry type="jdk" jdkName="Python 3.13" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/misc.xml
+ 1
− 1
View file @ dbd50f20
......@@ -3,5 +3,5 @@
<component name="Black">
<option name="sdkName" value="Python 3.12" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
data/investment/investment.csv
+ 116
− 0
View file @ dbd50f20
......@@ -119,3 +119,119 @@ iteration,model,temperature,ccei
28,deepseek-r1,0.0,0.1870967741935484
29,deepseek-r1,0.0,0.2575
30,deepseek-r1,0.0,0.1514285714285714
1,mixtral:8x7b,0.0,0.8152173913043478
2,mixtral:8x7b,0.0,0.8418604651162791
3,mixtral:8x7b,0.0,0.8540540540540541
4,mixtral:8x7b,0.0,0.875
5,mixtral:8x7b,0.0,0.8085106382978723
6,mixtral:8x7b,0.0,0.8444444444444444
7,mixtral:8x7b,0.0,0.9032258064516129
8,mixtral:8x7b,0.0,0.8298319327731092
9,mixtral:8x7b,0.0,0.8333333333333334
10,mixtral:8x7b,0.0,0.875
11,mixtral:8x7b,0.0,0.8260869565217391
12,mixtral:8x7b,0.0,0.82
13,mixtral:8x7b,0.0,0.8636363636363636
14,mixtral:8x7b,0.0,0.8333333333333334
15,mixtral:8x7b,0.0,0.8444444444444444
16,mixtral:8x7b,0.0,0.8636363636363636
17,mixtral:8x7b,0.0,0.82
18,mixtral:8x7b,0.0,0.82
19,mixtral:8x7b,0.0,0.8787878787878788
20,mixtral:8x7b,0.0,0.8285714285714286
21,mixtral:8x7b,0.0,0.8085106382978723
22,mixtral:8x7b,0.0,0.8787878787878788
23,mixtral:8x7b,0.0,0.8785714285714286
24,mixtral:8x7b,0.0,0.7894736842105263
25,mixtral:8x7b,0.0,0.8421052631578947
1,llama3.3:latest,0.0,1.0
2,llama3.3:latest,0.0,1.0
3,llama3.3:latest,0.0,1.0
4,llama3.3:latest,0.0,1.0
5,llama3.3:latest,0.0,1.0
6,llama3.3:latest,0.0,1.0
7,llama3.3:latest,0.0,1.0
8,llama3.3:latest,0.0,1.0
9,llama3.3:latest,0.0,1.0
10,llama3.3:latest,0.0,1.0
11,llama3.3:latest,0.0,1.0
12,llama3.3:latest,0.0,1.0
13,llama3.3:latest,0.0,1.0
14,llama3.3:latest,0.0,1.0
15,llama3.3:latest,0.0,1.0
16,llama3.3:latest,0.0,1.0
17,llama3.3:latest,0.0,1.0
18,llama3.3:latest,0.0,1.0
19,llama3.3:latest,0.0,1.0
20,llama3.3:latest,0.0,1.0
21,llama3.3:latest,0.0,1.0
22,llama3.3:latest,0.0,1.0
23,llama3.3:latest,0.0,1.0
24,llama3.3:latest,0.0,1.0
25,llama3.3:latest,0.0,1.0
26,llama3.3:latest,0.0,1.0
27,llama3.3:latest,0.0,1.0
28,llama3.3:latest,0.0,1.0
29,llama3.3:latest,0.0,1.0
30,llama3.3:latest,0.0,1.0
1,deepseek-r1:7b,0.0,1.0
2,deepseek-r1:7b,0.0,1.0
3,deepseek-r1:7b,0.0,1.0
4,deepseek-r1:7b,0.0,1.0
5,deepseek-r1:7b,0.0,1.0
6,deepseek-r1:7b,0.0,1.0
7,deepseek-r1:7b,0.0,1.0
8,deepseek-r1:7b,0.0,1.0
9,deepseek-r1:7b,0.0,1.0
10,deepseek-r1:7b,0.0,1.0
11,deepseek-r1:7b,0.0,1.0
12,deepseek-r1:7b,0.0,1.0
13,deepseek-r1:7b,0.0,1.0
14,deepseek-r1:7b,0.0,1.0
15,deepseek-r1:7b,0.0,1.0
16,deepseek-r1:7b,0.0,1.0
17,deepseek-r1:7b,0.0,1.0
18,deepseek-r1:7b,0.0,1.0
19,deepseek-r1:7b,0.0,1.0
20,deepseek-r1:7b,0.0,1.0
21,deepseek-r1:7b,0.0,1.0
22,deepseek-r1:7b,0.0,1.0
23,deepseek-r1:7b,0.0,1.0
24,deepseek-r1:7b,0.0,1.0
25,deepseek-r1:7b,0.0,1.0
26,deepseek-r1:7b,0.0,1.0
27,deepseek-r1:7b,0.0,1.0
28,deepseek-r1:7b,0.0,1.0
29,deepseek-r1:7b,0.0,1.0
30,deepseek-r1:7b,0.0,1.0
1,gpt-4.5-preview-2025-02-27,0.0,1.0
2,gpt-4.5-preview-2025-02-27,0.0,1.0
3,gpt-4.5-preview-2025-02-27,0.0,1.0
4,gpt-4.5-preview-2025-02-27,0.0,1.0
5,gpt-4.5-preview-2025-02-27,0.0,1.0
6,gpt-4.5-preview-2025-02-27,0.0,1.0
7,gpt-4.5-preview-2025-02-27,0.0,1.0
8,gpt-4.5-preview-2025-02-27,0.0,1.0
9,gpt-4.5-preview-2025-02-27,0.0,1.0
10,gpt-4.5-preview-2025-02-27,0.0,1.0
11,gpt-4.5-preview-2025-02-27,0.0,1.0
12,gpt-4.5-preview-2025-02-27,0.0,1.0
13,gpt-4.5-preview-2025-02-27,0.0,1.0
14,gpt-4.5-preview-2025-02-27,0.0,1.0
15,gpt-4.5-preview-2025-02-27,0.0,1.0
16,gpt-4.5-preview-2025-02-27,0.0,1.0
17,gpt-4.5-preview-2025-02-27,0.0,1.0
18,gpt-4.5-preview-2025-02-27,0.0,1.0
19,gpt-4.5-preview-2025-02-27,0.0,1.0
20,gpt-4.5-preview-2025-02-27,0.0,1.0
21,gpt-4.5-preview-2025-02-27,0.0,1.0
22,gpt-4.5-preview-2025-02-27,0.0,1.0
23,gpt-4.5-preview-2025-02-27,0.0,1.0
24,gpt-4.5-preview-2025-02-27,0.0,1.0
25,gpt-4.5-preview-2025-02-27,0.0,1.0
26,gpt-4.5-preview-2025-02-27,0.0,1.0
27,gpt-4.5-preview-2025-02-27,0.0,1.0
28,gpt-4.5-preview-2025-02-27,0.0,1.0
29,gpt-4.5-preview-2025-02-27,0.0,1.0
30,gpt-4.5-preview-2025-02-27,0.0,1.0
figures/investment/investment_violin.svg
+ 1606
− 1135
View file @ dbd50f20
This diff is collapsed.
src/investment/investment.py
+ 140
− 45
View file @ dbd50f20
......@@ -9,31 +9,47 @@ from autogen_core import CancellationToken
from autogen_ext.models.openai import OpenAIChatCompletionClient
from pydantic import BaseModel
from scipy.optimize import linprog
import requests
import json
import httpx
import re
# Load API key from environment variable
# Load API key from environment variables
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
if not OPENAI_API_KEY:
raise ValueError("Missing OPENAI_API_KEY. Set it as an environment variable.")
PAGODA_API_KEY = os.getenv("PAGODA_API_KEY")
if not PAGODA_API_KEY:
raise ValueError("Missing PAGODA_API_KEY. Set it as an environment variable.")
# Define the expected response format as a Pydantic model
class AgentResponse(BaseModel):
assetA: int
assetB: int
assetA: float
assetB: float
reasoning: str
# The investment game simulation class
class Investment:
def __init__(self, model: str, temperature: float, max_retries: int = 3):
self.debug = False
self.debug = True
self.model = model
self.temperature = temperature
self.strategy = random
self.max_retries = max_retries # Maximum retry attempts in case of hallucinations
if not model == "random" and not model == "optimal":
is_openai_model = model.startswith("gpt")
base_url = "https://api.openai.com/v1" if is_openai_model else "http://localhost:11434/v1"
is_openai_model = model.startswith("gpt")
is_pagoda_model = ":" in model
base_url = (
"https://api.openai.com/v1" if is_openai_model else
"https://ollama-ui.pagoda.liris.cnrs.fr/ollama/api/generate" if is_pagoda_model else
"http://localhost:11434/v1"
)
self.base_url = base_url
key = OPENAI_API_KEY if is_openai_model else PAGODA_API_KEY
if not model in ["random", "optimal"]:
model_info = {
"temperature": self.temperature,
"function_calling": True,
......@@ -46,13 +62,12 @@ class Investment:
self.model_client = OpenAIChatCompletionClient(
model=self.model,
base_url=base_url,
api_key=OPENAI_API_KEY,
api_key=key,
model_info=model_info,
response_format=AgentResponse
)
async def run(self, m: float, n: float) -> Dict:
"""Runs the model if strategy is False, otherwise uses a classical method."""
if self.model == "random":
return self.apply_random(m, n)
if self.model == "optimal":
......@@ -66,11 +81,15 @@ class Investment:
Your response should be in JSON format with `assetA`, `assetB`, and `reasoning`.
"""
is_pagoda_model = ":" in self.model
if is_pagoda_model:
return await self.run_pagoda(instruction)
for attempt in range(self.max_retries):
agent = AssistantAgent(
name="Investor",
model_client=self.model_client,
system_message="You are a helpful assistant. You will be given 25 rounds of decision-making tasks and will be responsible for making decisions. You should use your best judgment to come up with solutions that you like most."
system_message="You are a helpful assistant."
)
response = await agent.on_messages(
......@@ -80,50 +99,110 @@ class Investment:
try:
response_data = response.chat_message.content
agent_response = AgentResponse.model_validate_json(response_data) # Parse JSON
agent_response = AgentResponse.model_validate_json(response_data)
assetA, assetB = agent_response.assetA, agent_response.assetB
if self.debug:
print(f"Response (Attempt {attempt+1}): {response_data}")
# Validate values: ensure they sum to $100 considering the values M and N
if 0 <= assetA and assetA <= 100 and 0 <= assetB and assetB <= 100 and assetA + assetB == 100:
if 0 <= assetA <= 100 and 0 <= assetB <= 100 and assetA + assetB == 100:
return agent_response.model_dump()
else:
if self.debug:
print(f"Invalid response detected (Attempt {attempt+1}): {response_data}")
except Exception as e:
print(f"Error parsing response (Attempt {attempt+1}): {e}")
raise ValueError("Model failed to provide a valid response after multiple attempts.")
def apply_random(self, m: int, n: int) -> Dict:
"""Generates a response."""
assetA = random.randint(0, 100)
assetB = 100 - assetA
return {
"assetA": assetA,
"assetB": assetB,
"reasoning": "Random choice"
async def run_pagoda(self, instruction: str) -> Dict:
"""Runs the Pagoda model using a direct request with improved response parsing."""
url = self.base_url
headers = {
"Authorization": f"Bearer {PAGODA_API_KEY}",
"Content-Type": "application/json"
}
def apply_optimal(self, m: int, n: int) -> Dict:
"""Generates a response."""
if m > n:
assetA = 100
assetB = 0
else:
assetA = 0
assetB = 100
return {
"assetA": assetA,
"assetB": assetB,
"reasoning": "Optimal choice"
payload = {
"model": self.model,
"temperature": self.temperature,
"prompt": instruction,
"stream": False,
"response_format": "json"
}
def generate_M_N(self):
while True:
M = random.uniform(0.1, 1) # Random value in [0.1, 1]
N = random.uniform(0.1, 1) # Random value in [0.1, 1]
if max(M, N) >= 0.5: # Ensure max(M, N) is at least 0.5
return round(M, 1), round(N, 1)
# Print equivalent cURL command for debugging
curl_cmd = f"""
curl -X POST {url} \\
-H "Authorization: Bearer {PAGODA_API_KEY}" \\
-H "Content-Type: application/json" \\
-d '{json.dumps(payload, indent=2)}'
"""
print("Run this cURL command in your terminal to manually test the request:\n")
print(curl_cmd)
for attempt in range(self.max_retries):
try:
async with httpx.AsyncClient(verify=False, timeout=30) as client:
response = await client.post(url, headers=headers, json=payload)
response.raise_for_status() # Raise an error for HTTP status codes 4xx/5xx
response_data = response.json()
if self.debug:
print(f"Raw response (Attempt {attempt + 1}): {response_data}")
response_json = response_data.get("response")
if not response_json:
raise ValueError(f"Missing 'response' field (Attempt {attempt + 1})")
# Print full response for debugging
if self.debug:
print(f"Full response content (Attempt {attempt + 1}): {response_data}")
# Clean and parse JSON
if isinstance(response_json, str):
response_json = response_json.strip()
match = re.search(r'```json\n(.*?)\n```', response_json, re.DOTALL)
if match:
response_json = match.group(1)
response_json = response_json.replace("\n", "").replace("\\", "")
try:
response_dict = json.loads(response_json)
except json.JSONDecodeError:
print(f"Failed JSON: {response_json}")
raise ValueError(f"Failed to parse JSON (Attempt {attempt + 1})")
elif isinstance(response_json, dict):
response_dict = response_json
else:
raise TypeError(f"Unexpected response type: {type(response_json)}")
# Validate and adjust asset allocation
agent_response = AgentResponse(**response_dict)
assetA, assetB = round(agent_response.assetA), round(agent_response.assetB)
difference = 100 - (assetA + assetB)
if assetA >= assetB:
assetA += difference
else:
assetB += difference
if assetA + assetB != 100:
raise ValueError(f"Invalid allocation sum: {assetA}, {assetB}")
if not (0 <= assetA <= 100 and 0 <= assetB <= 100):
raise ValueError(f"Invalid asset allocation: {assetA}, {assetB}")
return agent_response.dict()
except httpx.HTTPStatusError as e:
print(
f"HTTP error from Pagoda API (Attempt {attempt + 1}): {e.response.status_code} - {e.response.text}")
except httpx.RequestError as e:
print(f"Request error in Pagoda API (Attempt {attempt + 1}): {e}")
except json.JSONDecodeError as e:
print(f"JSON parsing error (Attempt {attempt + 1}): {e}")
except ValueError as e:
print(f"Value error (Attempt {attempt + 1}): {e}")
except Exception as e:
print(f"Unexpected error (Attempt {attempt + 1}): {e}")
raise ValueError("Pagoda model failed to provide a valid response after multiple attempts.")
async def run_rounds(self, nb_rounds: int) -> float:
"""Runs the investment game for n rounds and computes the CCEI."""
......@@ -151,10 +230,26 @@ class Investment:
print(f"choices: {choices}")
print(f"budgets: {budgets}")
print(f"CCEI: {ccei_value}")
return ccei_value
def apply_random(self, m: int, n: int) -> Dict:
assetA = random.randint(0, 100)
assetB = 100 - assetA
return {"assetA": assetA, "assetB": assetB, "reasoning": "Random choice"}
def apply_optimal(self, m: int, n: int) -> Dict:
assetA = 100 if m > n else 0
assetB = 100 - assetA
return {"assetA": assetA, "assetB": assetB, "reasoning": "Optimal choice"}
def generate_M_N(self):
while True:
M = random.uniform(0.1, 1)
N = random.uniform(0.1, 1)
if max(M, N) >= 0.5:
return round(M, 1), round(N, 1)
def compute_ccei(self, prices, choices, budgets):
"""
Computes the Critical Cost Efficiency Index (CCEI).
......@@ -186,6 +281,6 @@ class Investment:
# Run the async function and return the response
if __name__ == "__main__":
game_agent = Investment(model="mistral-small", temperature=0.0) # Toggle strategy here
game_agent = Investment(model="mistral-small", temperature=0.0)
response = asyncio.run(game_agent.run_rounds(30))
print(response)
src/investment/investment_draw_violin.py
+ 19
− 6
View file @ dbd50f20
......@@ -4,13 +4,26 @@ import matplotlib.pyplot as plt
# Custom color palette
color_palette = {
'random' : '#333333', # Black
'gpt-4.5-preview-2025-02-27': '#7abaff', # Blue
'llama3': '#32a68c', # Green
'mistral-small': '#ff6941', # Orange
'deepseek-r1': '#5862ed' # Indigo
'random': '#333333', # Black
'gpt-4.5-preview-2025-02-27': '#7abaff', # BlueEscape
'llama3': '#32a68c', # GreenFuture
'llama3.3:latest': '#4b9f7d', # GreenLlama3.3
'mistral-small': '#ff6941', # WarmOrange
'mixtral:8x7b': '#f1a61a', # YellowMixtral
'deepseek-r1': '#5862ed', # InclusiveIndigo
'deepseek-r1:7b': '#9a7bff' # PurpleDeepseek-r1:7b
}
# Specify the order of models for the x-axis
model_order = [
'random',
'gpt-4.5-preview-2025-02-27',
'llama3', 'llama3.3:latest', # Place llama3 and llama3.3:latest together
'mistral-small', 'mixtral:8x7b', # Bring mistral-small and mixtral:8x7b closer
'deepseek-r1', 'deepseek-r1:7b'
]
# Load CSV file
file_path = "../../data/investment/investment.csv" # Update path
df = pd.read_csv(file_path)
......@@ -29,7 +42,7 @@ sns.set(style="whitegrid")
plt.figure(figsize=(10, 6))
# Draw violin plot (replacing boxplot with violinplot)
sns.violinplot(data=df, x="model", y="ccei", palette=color_palette)
sns.violinplot(data=df, x="model", y="ccei", palette=color_palette, order=model_order)
# Add plot labels
plt.title("CCEI Distribution by Model", fontsize=14)
......
src/investment/investment_experiments.py
+ 2
− 2
View file @ dbd50f20
import asyncio
import csv
from investment import Investment # Assuming this is in a separate file
from investment import Investment
# Define models, temperature, and iterations
models = ["optimal", "random", "llama3", "mistral-small", "deepseek-r1"] # "gpt-4.5-preview-2025-02-27", "optimal", "random", "llama3", "mistral-small", "deepseek-r1"
models = ["deepseek-r1:7b"] # "gpt-4.5-preview-2025-02-27", "optimal", "random", "llama3", "mistral-small", "deepseek-r1", "mixtral:8x7b", "llama3.3:latest",
temperature = 0.0
iterations = 30
output_file = "../../data/investment/investment.csv"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment