Skip to content
GitLab
Commit 9602c60f authored by Chanelle Lee's avatar Chanelle Lee
Browse files

Updated test params

parent 3d0ae38c
Expand all Hide whitespace changes
Inline Side-by-side
.ipynb_checkpoints/BotSimLogAnalysis-checkpoint.ipynb
View file @ 9602c60f
This diff is collapsed.
This diff is collapsed.
.ipynb_checkpoints/ProbConfusionAnalysis-checkpoint.ipynb
View file @ 9602c60f
This diff is collapsed.
This diff is collapsed.
BotSimLogAnalysis.ipynb
View file @ 9602c60f
This diff is collapsed.
This diff is collapsed.
ProbConfusionAnalysis.ipynb
View file @ 9602c60f
......@@ -97,7 +97,7 @@
{
"data": {
"text/plain": [
"[<matplotlib.lines.Line2D at 0x97bb1d0>]"
"[<matplotlib.lines.Line2D at 0x22e0f4ad240>]"
]
},
"execution_count": 7,
......@@ -129,7 +129,7 @@
{
"data": {
"text/plain": [
"[<matplotlib.lines.Line2D at 0x9851c88>]"
"[<matplotlib.lines.Line2D at 0x22e0f548f28>]"
]
},
"execution_count": 8,
......@@ -353,7 +353,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
......@@ -828,7 +828,7 @@
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x5394898>"
"<matplotlib.axes._subplots.AxesSubplot at 0x22e106ee1d0>"
]
},
"execution_count": 20,
......@@ -860,7 +860,7 @@
{
"data": {
"text/plain": [
"<matplotlib.lines.Line2D at 0x9984320>"
"<matplotlib.lines.Line2D at 0x22e1083ae80>"
]
},
"execution_count": 21,
......@@ -994,31 +994,16 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 52,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\cl15753\\AppData\\Local\\Continuum\\anaconda2\\envs\\Py3\\lib\\site-packages\\pandas\\core\\frame.py:6211: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
"of pandas will change to not sort by default.\n",
"\n",
"To accept the future behavior, pass 'sort=False'.\n",
"\n",
"To retain the current behavior and silence the warning, pass 'sort=True'.\n",
"\n",
" sort=sort)\n"
]
}
],
"outputs": [],
"source": [
"testParams_DF = testParams_DF.append([{'sigma': row[1], 'alpha': row[2], 'chance': '10%', 'note': 'Prob_q2'} for row in df_1Ranges.itertuples()])"
"testParams_DF = testParams_DF.append([{'sigma': row[1], 'alpha': row[2], 'chance': '10%', 'note': 'Prob_q2'} for row in df_1Ranges.itertuples()], sort=True)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 53,
"metadata": {},
"outputs": [
{
......@@ -1030,7 +1015,7 @@
" 0.028578442773621412]"
]
},
"execution_count": 25,
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
......@@ -1042,7 +1027,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 54,
"metadata": {},
"outputs": [],
"source": [
......@@ -1059,7 +1044,7 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 55,
"metadata": {},
"outputs": [],
"source": [
......@@ -1073,7 +1058,7 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 56,
"metadata": {},
"outputs": [
{
......@@ -1149,7 +1134,7 @@
"35 0.175175 0.250550 0.089228 0.021781 0.003561"
]
},
"execution_count": 28,
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
......@@ -1160,16 +1145,16 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 57,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.lines.Line2D at 0xb114470>"
"<matplotlib.lines.Line2D at 0x22e11174400>"
]
},
"execution_count": 29,
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
},
......@@ -1199,7 +1184,7 @@
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 58,
"metadata": {},
"outputs": [],
"source": [
......@@ -1208,7 +1193,7 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
......@@ -1217,7 +1202,7 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
......@@ -1227,7 +1212,26 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [],
"source": [
"sigmas = set(testParams_DF['sigma'])\n",
"chances = [testParams_DF.loc[testParams_DF['sigma'] == sigma, 'chance'].iloc[0] for sigma in sigmas]\n",
"testParams_DF = testParams_DF.append([{'sigma': sigma, 'alpha': 0.1, 'chance': chances[i], 'note': 'control'}\n",
" for i, sigma in enumerate(sigmas)])"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {
"scrolled": true
},
......@@ -1236,23 +1240,105 @@
"name": "stdout",
"output_type": "stream",
"text": [
" alpha chance note sigma\n",
"0 0.375167 10.0 Prob_q2 0.370370\n",
"1 0.337069 10.0 Prob_q2 0.280280\n",
"2 0.262259 10.0 Prob_q2 0.185185\n",
"3 0.107618 10.0 Prob_q2 0.095095\n",
"0 0.227216 10.0 mean 0.370370\n",
"1 0.171782 10.0 mean 0.280280\n",
"2 0.099345 10.0 mean 0.185185\n",
"3 0.028578 10.0 mean 0.095095\n",
"0 0.433217 25.0 Prob_q2 0.700701\n",
"1 0.411280 25.0 Prob_q2 0.525526\n",
"2 0.368292 25.0 Prob_q2 0.350350\n",
"3 0.250550 25.0 Prob_q2 0.175175\n",
"0 0.339747 25.0 mean 0.700701\n",
"1 0.293395 25.0 mean 0.525526\n",
"2 0.216131 25.0 mean 0.350350\n",
"3 0.091280 25.0 mean 0.175175\n"
" alpha chance note \\\n",
"0 0.375167 10% Prob_q2 \n",
"1 0.337069 10% Prob_q2 \n",
"2 0.262259 10% Prob_q2 \n",
"3 0.107618 10% Prob_q2 \n",
"0 0.227216 10% mean \n",
"1 0.171782 10% mean \n",
"2 0.099345 10% mean \n",
"3 0.028578 10% mean \n",
"0 0.433217 25% Prob_q2 \n",
"1 0.411280 25% Prob_q2 \n",
"2 0.368292 25% Prob_q2 \n",
"3 0.250550 25% Prob_q2 \n",
"0 0.339747 25% mean \n",
"1 0.293395 25% mean \n",
"2 0.216131 25% mean \n",
"3 0.091280 25% mean \n",
"0 0.100000 0 10%\n",
"0 10%\n",
"Name: chance, dtype: object control \n",
"1 0.100000 0 25%\n",
"0 25%\n",
"Name: chance, dtype: object control \n",
"2 0.100000 3 10%\n",
"3 10%\n",
"Name: chance, dtype: object control \n",
"3 0.100000 1 10%\n",
"1 10%\n",
"Name: chance, dtype: object control \n",
"4 0.100000 2 10%\n",
"2 10%\n",
"Name: chance, dtype: object control \n",
"5 0.100000 1 25%\n",
"1 25%\n",
"Name: chance, dtype: object control \n",
"6 0.100000 2 25%\n",
"2 25%\n",
"Name: chance, dtype: object control \n",
"7 0.100000 3 25%\n",
"3 25%\n",
"Name: chance, dtype: object control \n",
"0 0.433217 25% Prob_q2 \n",
"1 0.411280 25% Prob_q2 \n",
"2 0.368292 25% Prob_q2 \n",
"3 0.250550 25% Prob_q2 \n",
"0 0.339747 25% mean \n",
"1 0.293395 25% mean \n",
"2 0.216131 25% mean \n",
"3 0.091280 25% mean \n",
"0 0.100000 0 ... control \n",
"1 0.100000 0 ... control \n",
"2 0.100000 3 ... control \n",
"3 0.100000 1 ... control \n",
"4 0.100000 2 ... control \n",
"5 0.100000 1 ... control \n",
"6 0.100000 2 ... control \n",
"7 0.100000 3 ... control \n",
"\n",
" sigma \n",
"0 0.370370 \n",
"1 0.280280 \n",
"2 0.185185 \n",
"3 0.095095 \n",
"0 0.370370 \n",
"1 0.280280 \n",
"2 0.185185 \n",
"3 0.095095 \n",
"0 0.700701 \n",
"1 0.525526 \n",
"2 0.350350 \n",
"3 0.175175 \n",
"0 0.700701 \n",
"1 0.525526 \n",
"2 0.350350 \n",
"3 0.175175 \n",
"0 0.370370 \n",
"1 0.700701 \n",
"2 0.095095 \n",
"3 0.280280 \n",
"4 0.185185 \n",
"5 0.525526 \n",
"6 0.350350 \n",
"7 0.175175 \n",
"0 0.700701 \n",
"1 0.525526 \n",
"2 0.350350 \n",
"3 0.175175 \n",
"0 0.700701 \n",
"1 0.525526 \n",
"2 0.350350 \n",
"3 0.175175 \n",
"0 0.370370 \n",
"1 0.700701 \n",
"2 0.095095 \n",
"3 0.280280 \n",
"4 0.185185 \n",
"5 0.525526 \n",
"6 0.350350 \n",
"7 0.175175 \n"
]
}
],
......@@ -1262,34 +1348,9 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" alpha chance note sigma\n",
"paramSet \n",
"1 0.375167 10.0 Prob_q2 0.370370\n",
"2 0.337069 10.0 Prob_q2 0.280280\n",
"3 0.262259 10.0 Prob_q2 0.185185\n",
"4 0.107618 10.0 Prob_q2 0.095095\n",
"5 0.227216 10.0 mean 0.370370\n",
"6 0.171782 10.0 mean 0.280280\n",
"7 0.099345 10.0 mean 0.185185\n",
"8 0.028578 10.0 mean 0.095095\n",
"9 0.433217 25.0 Prob_q2 0.700701\n",
"10 0.411280 25.0 Prob_q2 0.525526\n",
"11 0.368292 25.0 Prob_q2 0.350350\n",
"12 0.250550 25.0 Prob_q2 0.175175\n",
"13 0.339747 25.0 mean 0.700701\n",
"14 0.293395 25.0 mean 0.525526\n",
"15 0.216131 25.0 mean 0.350350\n",
"16 0.091280 25.0 mean 0.175175\n"
]
}
],
"outputs": [],
"source": [
"testParams_DF['paramSet'] = range(1, len(testParams_DF)+1)\n",
"testParams_DF = testParams_DF.set_index('paramSet')\n",
......@@ -1298,7 +1359,7 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
......@@ -1307,44 +1368,65 @@
},
{
"cell_type": "code",
"execution_count": 36,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.37037046296296294"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"testParams_DF.iloc[0]['sigma']"
]
},
{
"cell_type": "code",
"execution_count": 37,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"set(testParams_DF.index.values) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"testParams_DF.index.values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"testParams_DF.iloc[15]['sigma']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def convert_paramSetNum(filePath):\n",
" df = pd.read_csv(filePath, names=['time', 'paramSet', 'simNum', 'timestep', 'agent', 'iteration', 'option', 'belief'])\n",
" df['paramSet'] +=1\n",
" df.to_csv(filePath)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os, glob\n",
"path = 'Results/'\n",
"all_files = glob.glob(os.path.join(path, \"botSim_beliefs_[0-9].csv\")) \n",
"for f in all_files:\n",
" convert_paramSetNum(f)"
]
},
{
"cell_type": "code",
"execution_count": null,
......@@ -1369,7 +1451,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
"version": "3.7.2"
}
},
"nbformat": 4,
......
%% Cell type:code id: tags:
``` python
import numpy as np
import scipy
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
```
%% Cell type:markdown id: tags:
# Analysis of the probability of confusing qualities for different values of sigma
%% Cell type:markdown id: tags:
First need the pdf and cdf of the normal distributions which will represent the noise around each of the quality values
%% Cell type:code id: tags:
``` python
def f(x, mu, s):
return scipy.stats.norm.pdf(x, loc=mu, scale=s)
```
%% Cell type:code id: tags:
``` python
def F(x, mu, s):
return scipy.stats.norm.cdf(x, loc=mu, scale=s)
```
%% Cell type:markdown id: tags:
Qualities are set in the interval (0,1) and equally spaced
%% Cell type:code id: tags:
``` python
n = 5
qualities = {i: i/(n+1) for i in range(1, (n+1))}
```
%% Cell type:code id: tags:
``` python
def integrand(x, i, j, s):
return f(x, qualities[i], s)*F(x, qualities[j], s)
def probConfusion(i, j, s):
return scipy.integrate.quad(integrand, -np.inf, np.inf, args=(i, j, s))
```
%% Cell type:code id: tags:
``` python
s0 = 0.0000001
sf = 100
ss = np.linspace(s0, sf, 1000)
confusions = [probConfusion(1, 2, s)[0] for s in ss]
```
%% Cell type:code id: tags:
``` python
plt.plot(ss, confusions)
```
%%%% Output: execute_result
[<matplotlib.lines.Line2D at 0x97bb1d0>]
[<matplotlib.lines.Line2D at 0x22e0f4ad240>]
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
ss = np.linspace(0.0000001, 10, 1000)
confusions = [probConfusion(1, 2, s)[0] for s in ss]
plt.plot(ss, confusions)
```
%%%% Output: execute_result
[<matplotlib.lines.Line2D at 0x9851c88>]
[<matplotlib.lines.Line2D at 0x22e0f548f28>]
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
ss = np.linspace(0.0000001, 10, 1000)
confusions = [[probConfusion(1, j, s)[0] for s in ss] for j in [2, 3, 4, 5]]
for confusion in confusions:
plt.plot(ss, confusion)
```
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
ss = np.linspace(0.0000001, 10, 1000)
confusions = [[probConfusion(1, j, s)[0] for s in ss] for j in [2, 3, 4, 5]]
for confusion in confusions:
plt.plot(ss, confusion)
```
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
ss = np.linspace(0.0000001, 5, 1000)
confusions = [[probConfusion(1, j, s)[0] for s in ss] for j in [2, 3, 4, 5]]
for confusion in confusions:
plt.plot(ss, confusion)
```
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
d = {'sigma':ss, **{'q{}'.format(j): confusions[i] for i, j in enumerate([2, 3, 4, 5])}}
```
%% Cell type:code id: tags:
``` python
df = pd.DataFrame(data=d)
```
%% Cell type:code id: tags:
``` python
df.head()
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
0 1.000000e-07 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00
1 5.005105e-03 4.104751e-122 0.000000e+00 0.000000e+00 0.000000e+00
2 1.001011e-02 1.051201e-31 3.577518e-122 7.451264e-274 0.000000e+00
3 1.501511e-02 5.734691e-15 2.733803e-55 9.343211e-123 2.336542e-219
4 2.002012e-02 4.075678e-09 6.984412e-32 7.149700e-70 3.085665e-124
%% Cell type:markdown id: tags:
### Here want to get some idea of appropriate sigma values - how about checking the cut offs for having at least a 10% chance of confusion for all sites within a range:
%% Cell type:code id: tags:
``` python
testParams_DF = pd.DataFrame(columns=['sigma', 'alpha', 'note'])
```
%% Cell type:markdown id: tags:
Range is whole quality space
%% Cell type:code id: tags:
``` python
df.loc[(df['q2'] >= 0.1) & (df['q3'] >= 0.1) & (df['q4'] >= 0.1) & (df['q5'] >= 0.1)].head()
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
74 0.370370 0.375167 0.262259 0.169892 0.101546
75 0.375375 0.376777 0.265031 0.173131 0.104590
76 0.380380 0.378347 0.267745 0.176322 0.107618
77 0.385385 0.379878 0.270401 0.179466 0.110627
78 0.390390 0.381372 0.273002 0.182563 0.113616
%% Cell type:markdown id: tags:
$\sigma = 0.37$
- q2 : 0.38
- q3 : 0.26
- q4 : 0.17
- q5 : 0.10
%% Cell type:markdown id: tags:
Range is q4 or closer
%% Cell type:code id: tags:
``` python
df.loc[(df['q2'] >= 0.1) & (df['q3'] >= 0.1) & (df['q4'] >= 0.1)].head()
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
56 0.280280 0.337069 0.200187 0.103577 0.046294
57 0.285285 0.339767 0.204346 0.107618 0.049227
58 0.290290 0.342380 0.208409 0.111625 0.052198
59 0.295295 0.344911 0.212380 0.115597 0.055202
60 0.300300 0.347365 0.216260 0.119531 0.058234
%% Cell type:markdown id: tags:
$\sigma=0.28$
- q2 : 0.34
- q3 : 0.20
- q4 : 0.10
- q5 : 0.04
%% Cell type:markdown id: tags:
Range is q3 or closer
%% Cell type:code id: tags:
``` python
df.loc[(df['q2'] >= 0.1) & (df['q3'] >= 0.1)].head()
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
37 0.185185 0.262259 0.101546 0.028119 0.005455
38 0.190190 0.267745 0.107618 0.031517 0.006595
39 0.195195 0.273002 0.113616 0.035049 0.007867
40 0.200200 0.278043 0.119531 0.038698 0.009270
41 0.205205 0.282880 0.125357 0.042451 0.010803
%% Cell type:markdown id: tags:
$\sigma = 0.19$
- q2 : 0.26
- q3 : 0.10
- q4 : 0.03
- q5 : 0.01
%% Cell type:markdown id: tags:
Range is just q2
%% Cell type:code id: tags:
``` python
df.loc[(df['q2'] >= 0.1)].head()
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
19 0.095095 0.107618 0.006595 0.000100 3.576083e-07
20 0.100100 0.119532 0.009270 0.000206 1.242683e-06
21 0.105105 0.131087 0.012463 0.000384 3.644447e-06
22 0.110110 0.142242 0.016153 0.000662 9.294145e-06
23 0.115115 0.152973 0.020303 0.001066 2.110231e-05
%% Cell type:markdown id: tags:
$\sigma = 0.1$
- q2 : 0.11
- q3 : 0.01
- q4 : 0.00
- q5 : 0.00
%% Cell type:code id: tags:
``` python
sns.lineplot(data=df.set_index('sigma'))
```
%%%% Output: execute_result
<matplotlib.axes._subplots.AxesSubplot at 0x5394898>
<matplotlib.axes._subplots.AxesSubplot at 0x22e106ee1d0>
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
sns.lineplot(data=df.where(df['sigma'] <= 1.0).set_index('sigma'))
# plotting the values of sigma
plt.axvline(0.1, alpha=0.3, color='grey')
plt.axvline(0.19, alpha=0.3, color='grey')
plt.axvline(0.28, alpha=0.3, color='grey')
plt.axvline(0.37, alpha=0.3, color='grey')
# plotting P = 0.1
plt.axhline(0.1, alpha=0.3, color='grey')
```
%%%% Output: execute_result
<matplotlib.lines.Line2D at 0x9984320>
<matplotlib.lines.Line2D at 0x22e1083ae80>
%%%% Output: display_data
%% Cell type:code id: tags:
``` python
desProb = 0.1
range_all = df[(df['q2'] >= desProb) & (df['q3'] >= desProb) & (df['q4'] >= desProb) & (df['q5'] >= desProb)].iloc[0]
range_q4 = df[(df['q2'] >= desProb) & (df['q3'] >= desProb) & (df['q4'] >= desProb)].iloc[0]
range_q3 = df[(df['q2'] >= desProb) & (df['q3'] >= desProb)].iloc[0]
range_q2 = df[(df['q2'] >= desProb)].iloc[0]
df_1Ranges = pd.DataFrame(data = [range_all, range_q4, range_q3, range_q2])
```
%% Cell type:code id: tags:
``` python
df_1Ranges
```
%%%% Output: execute_result
sigma q2 q3 q4 q5
74 0.370370 0.375167 0.262259 0.169892 1.015460e-01
56 0.280280 0.337069 0.200187 0.103577 4.629378e-02
37 0.185185 0.262259 0.101546 0.028119 5.454770e-03
19 0.095095 0.107618 0.006595 0.000100 3.576083e-07