From 3e98b2cd9cb8ceb9246b65adc289abae04d12466 Mon Sep 17 00:00:00 2001 From: Lukas Galke Date: Thu, 10 Oct 2024 12:07:23 +0200 Subject: [PATCH] Clean-up --- debug.bash | 3 +- modeling.py | 126 ------------------------ stats.bash | 5 +- stats.py | 273 +++++++--------------------------------------------- 4 files changed, 38 insertions(+), 369 deletions(-) diff --git a/debug.bash b/debug.bash index d4e1a92..b1120af 100755 --- a/debug.bash +++ b/debug.bash @@ -9,7 +9,6 @@ OUTDIR="./results-v1" SEED=123456789 LEXP="data/LearningExp_190501_S5_001_log.txt" -# Iterate over different seeds here echo "Seed: $SEED" echo "Starting run with experiment data: $LEXP" -python3 train.py --as_humans "$LEXP" --seed "$SEED" --debug --iterations 5 --outdir "tmp/" +python3 train.py --as_humans "$LEXP" --seed "$SEED" --debug --iterations 5 --outdir "tmp/" diff --git a/modeling.py b/modeling.py index f1924f7..e07f54b 100644 --- a/modeling.py +++ b/modeling.py @@ -4,7 +4,6 @@ from egg import core -# TODO: https://github.com/pytorch/examples/blob/main/word_language_model/model.py class RelaxedLinear(nn.Linear): @@ -12,12 +11,6 @@ class RelaxedLinear(nn.Linear): """Overwrite to allow second, ignored, input""" def __init__(self, *args, **kwargs): - """TODO: to be defined. - - :*args: TODO - :**kwargs: TODO - - """ super().__init__(*args, **kwargs) def forward(self, x, __aux_input=None): @@ -145,33 +138,6 @@ def tie_weights( raise AssertionError("Could not resolve type mismatch") -class DoubleAgent(nn.Module): - - """Common forward signature for an agent capable of both sending and receiving""" - - def __init__(self): - nn.Module.__init__(self) - self.tie_weights() - - def forward( - self, - sender_input=None, - aux_input=None, - message=None, - receiver_input=None, - lengths=None, - ): - if message is not None: - outputs = self.receiver( - message, input=receiver_input, aux_input=aux_input, lengths=length - ) - else: - assert sender_input is not None - outputs = self.sender(sender_input, aux_input=aux_input) - return outputs - - def tie_weights(self): - tie_weights(self.sender, self.receiver) class MLP(nn.Module): @@ -197,7 +163,6 @@ def __init__( self.drop = nn.Dropout(dropout) def forward(self, x, input=None, aux_input=None): - # TODO check if we want to do something with input/aux_input receivers h = x for i, layer in enumerate(self.layers): @@ -293,12 +258,6 @@ def __init__( straight_through: bool = False, tied_weights: str = "all", ): - """TODO: to be defined. - - :input2hidden: TODO - :hidden2output: TODO - - """ nn.Module.__init__(self) self.input2hidden = input2hidden # MLP / Linear @@ -366,88 +325,3 @@ def forward( sender_input, aux_input=aux_input ) # bsz, seqlen, vocab_size return outputs - - -class MLPDoubleAgent(DoubleAgent): - - """MLP-based double agent""" - - def __init__( - self, - input_dim, - hidden_size, - vocab_size, - max_length, - tie_weights=True, - dropout=0.5, - ): - """TODO: to be defined. - - :l: TODO - - """ - DoubleAgent.__init__(self) - - self.dropout = nn.Dropout(dropout) - # Seeing: scene to latent - self.input2hidden = nn.Linear(input_dim, hidden_size) - - # Writing: latent to message - self.hidden2msg = nn.Linear(hidden_size, vocab_size * max_length) - - # Reading: message to latent - if tie_weights: - print("Tying weights...") - self.msg2hidden = TiedLinear(self.hidden2msg) - else: - self.msg2hidden = nn.Linear(vocab_size * max_length, hidden_size) - - self.init_weights() - - def init_weights(self): - initrange = 0.1 - - # input to hidden - nn.init.uniform_(self.input2hidden.weight, -initrange, initrange) - nn.init.zeros_(self.input2hidden.bias) - - # hidden to message - nn.init.uniform_(self.hidden2msg.weight, -initrange, initrange) - nn.init.zeros_(self.hidden2msg.bias) - - # message to hidden (be sure even if tied) - nn.init.uniform_(self.msg2hidden.weight, -initrange, initrange) - nn.init.zeros_(self.msg2hidden.bias) - - def forward(self, x, aux_input=None): - self.input2hidden(x) - # TODO fill - - raise NotImplementedError - - -class UniformAgentSamplerWithRoleAlternation(nn.Module): - """Extension for egg.core.population to build populations that allow role alternation - >>> game = ... - >>> agents = ... - >>> agents_loss_sampler = RoleAlternationAgentSampler(agents) - >>> game = PopulationGame(game, agents_loss_sampler) - """ - - def __init__(self, agents, losses, seed=1234): - super().__init__() - np.random.seed(seed) - self.agents = nn.ModuleList(agents) - self.losses = list(losses) - - def forward(self): - s_idx, r_idx, l_idx = ( - np.random.choice(len(self.senders)), - np.random.choice(len(self.receivers)), - np.random.choice(len(self.losses)), - ) - return ( - self.agents[s_idx], # Sender - self.agents[r_idx], # Receiver - self.losses[l_idx], - ) diff --git a/stats.bash b/stats.bash index 182947d..a4b6d93 100644 --- a/stats.bash +++ b/stats.bash @@ -1 +1,4 @@ -python3 stats.py --models_subdir models_nested_new -o ~/Documents/project-data/paper__LEADS_easy-to-learn/results-v1-stats-output-v2/ ~/Documents/project-data/paper__LEADS_easy-to-learn/results-v1 +# Helper script to run the stats.py script +# Adjust the path to the results directory +RESULTS_DIR=./results +python3 stats.py -o $RESULTS_DIR --models_subdir statsmodels $RESULTS_DIR diff --git a/stats.py b/stats.py index b000081..972e0a2 100755 --- a/stats.py +++ b/stats.py @@ -1,3 +1,8 @@ +""" +This script calculates the statistics for the memorization and regularization results. +A path to the folder with all results (mem_data.csv and reg_data.csv) is required. +The script will output the results in a subdirectory of the same folder. +""" import argparse import glob import os @@ -7,7 +12,6 @@ mpl.rcParams["figure.dpi"] = 300 - import matplotlib.pyplot as plt plt.rcParams["axes.facecolor"] = "white" @@ -38,9 +42,6 @@ mean_production_similarity, ) -# from statsmodels.genmod.bayes_mixed_glm import BinomialBayesMixedGLM - - # GLOBALS COLORMAP = plt.get_cmap( "copper" @@ -286,28 +287,6 @@ def calc_convergence_score(reg_data: pd.DataFrame, word_column="Input"): convscore_series[group.index] = convscore - # OLD variant (errornous) - # Iterate over items - # Items are a function of Trial in convergence - # rounds = pd.unique(reg_data.index.get_level_values("Round")) - # trials = pd.unique(reg_data.index.get_level_values("Trial")) - # Use Target to make sure despite Trial would be faster to access - # items = pd.unique(reg_data["Target"]) - # for i in tqdm(rounds, desc="Calculate convergence score"): - # for j in trials: - # subset = reg_data.loc[:, i, j] - - # messages = subset["Input"].values - - # # Exactly as in the orig R script: - # # 1. calc conv score with normalized edit distance - # # 2. then calculate 1 - conv score - # # ( we could also do prodsim right away, but better keep same ) - - # convscore = convergence_score(messages, metric="normalized_editdistance") - - # convscore_series.loc[:, i, j] = 1 - convscore - return convscore_series @@ -741,7 +720,17 @@ def save_summary_and_plot(number, results, folder, extra_desc: str = ""): ) as filehandle: print(summary.as_latex(), file=filehandle) + print("---") + print("Making QQ plot to test normality of residuals") + qq_path = os.path.join(folder, f"model_{number}{extra_desc}_qq.png") + qq_plot = sm.qqplot(results.resid, line="45") + plt.tight_layout(pad=1.0) + plt.savefig(qq_path) + print("---") + with plt.style.context("seaborn-paper"): + + # Plot partregress grid fig = plt.figure(figsize=(8, 6)) fig = sm.graphics.plot_partregress_grid(results, fig=fig) @@ -760,32 +749,7 @@ def save_summary_and_plot(number, results, folder, extra_desc: str = ""): def run_model_1(data: pd.DataFrame, epoch=100, outdir="."): - """Final Accuracy - :data: Dataframe holding memorization data - :epoch: The epoch at which the test should be conducted - :returns: None - :outdir: base directory to write outputs to - - Plot 1 Final Accuracy - # model with 1 degree - m1<- glmer(Raw_ACC ~ - c.Structure_Score + - (1|Seed)+(1|Target_Item), - data=memory_test, family="binomial") - - # model with 2 degree (selected) - m1_poly <- glmer(Raw_ACC ~ - poly(c.Structure_Score,2) + - (1|Seed)+(1|Target_Item), - data=memory_test, family="binomial") - - """ - print(f"Running model 1 on data from epoch {epoch}") - - # BinomialBayesMixedGLM needs vc_formulas ? random? - # https://www.statsmodels.org/stable/generated/statsmodels.genmod.bayes_mixed_glm.BinomialBayesMixedGLM.html#statsmodels.genmod.bayes_mixed_glm.BinomialBayesMixedGLM - # Example from page would translate to (1 + Year | Village) - + """Final Accuracy""" subset = data[data.Round == epoch] random = {"a": "0 + C(Producer)", "b": "0 + C(Target)"} @@ -802,12 +766,6 @@ def run_model_1(data: pd.DataFrame, epoch=100, outdir="."): def run_model_2(data: pd.DataFrame, epoch=100, outdir="."): """Final Production Similarity (with nested random intercepts) - - :data: Dataframe holding memorization data - :epoch: The epoch at which the test should be conducted - :returns: None - :outdir: base directory to write outputs to - """ print(f"Running model 2 on data from epoch {epoch}") subset = data[data.Round == epoch] @@ -835,114 +793,22 @@ def make_interaction_plot( x=data["Round"], # x trace=data["StructureScore"], # trace response=data[response_variable], # response - # colors=["red", "blue"], - # markers=["D", "^"], ms=10, ax=ax, plottype="both", ) - # NO RANDOM EFFECTS plt.tight_layout(pad=1.0) plt.savefig( os.path.join(outdir, f"{prefix}{response_variable}_interaction_plot.png") ) -def run_model_3(mem_data: pd.DataFrame): - """Learning trajectory""" - print("Running model 3 -- Learning trajectory") - print(mem_data.columns) - - raise NotImplementedError - - def nested_group_labels(outer: pd.Series, inner: pd.Series): return outer.astype(str) + "_" + inner.astype(str) - -def run_model_4_with_crossed_random_effects(data: pd.DataFrame, outdir="."): - """Learning trajectory PROD SIM to ground truth - Adapted from: https://github.com/statsmodels/statsmodels/blob/main/statsmodels/regression/tests/test_lme.py#L284 - """ - print( - "Running model 4 with more random effects -- Learning trajectory -- PROD SIM to Ground truth " - ) - - vcf = {"item": "0 + C(Target)", "seed": "0 + C(Producer)"} - groups = np.ones(len(data)) - print(data.columns) - model = sm.MixedLM.from_formula( - "ProdSim_GroundTruth ~ scale(StructureScore) * scale(np.log(Round))", - re_formula="0+C(Target)+C(Producer)", - vc_formula=vcf, - groups=groups, - data=data, - ) - # Killed (Out of memory) - results = model.fit() - - save_summary_and_plot(41, results, outdir) - - with plt.style.context("seaborn-paper"): - plt.figure(figsize=(8, 6)) - sns.relplot( - x="Round", - y="ProdSim_GroundTruth", - size="StructureScore", - hue="StructureBin", - kind="line", - data=data, - errorbar=ERRORBAR, - ) - plt.tight_layout(pad=1.0) - plt.savefig(os.path.join(outdir, "model_41_relplot.png")) - - -def run_model_4_with_nested_random_effects(data: pd.DataFrame, outdir="."): - """Learning trajectory PROD SIM to ground truth""" - print( - "Running model 4 with nested random effects -- Learning trajectory -- PROD SIM to Ground truth " - ) - - # vcf = {"item": "0 + C(Target)", "seed": "0 + C(Producer)"} - # groups = nested_group_labels(data['Producer'], data['Target']) - # print(data.columns) - # model = sm.MixedLM.from_formula("ProdSim_GroundTruth ~ scale(StructureScore) * scale(np.log(Round))", - # re_formula="0+C(Target)+C(Producer)", - # vc_formula=vcf, - # groups=groups, - # data=data - # ) - # results = model.fit() - # -> killed - - groups = nested_group_labels(data["Producer"], data["Target"]) - model = sm.MixedLM.from_formula( - "ProdSim_GroundTruth ~ scale(StructureScore) * scale(np.log(Round))", - groups=groups, - data=data, - ) - results = model.fit() - - save_summary_and_plot(42, results, outdir) - - with plt.style.context("seaborn-paper"): - plt.figure(figsize=(8, 6)) - sns.relplot( - x="Round", - y="ProdSim_GroundTruth", - size="StructureScore", - kind="line", - data=data, - errorbar=ERRORBAR, - ) - plt.tight_layout(pad=1.0) - plt.savefig(os.path.join(outdir, "model_42_relplot.png")) - - def run_model_4(data: pd.DataFrame, outdir=".", nested=False): - """Learning trajectory PROD SIM to ground truth""" + """LME 1: Learning trajectory PROD SIM to ground truth""" print("Running model 4 -- Learning trajectory -- PROD SIM to Ground truth ") endog, exog = dmatrices( @@ -974,50 +840,8 @@ def run_model_4(data: pd.DataFrame, outdir=".", nested=False): plt.tight_layout(pad=1.0) plt.savefig(os.path.join(outdir, "model_4_relplot.png")) - -def run_model_6(data: pd.DataFrame, outdir=".", nested=False): - """Learning trajectory GEN SCORE""" - print("Running model 6 -- GenScore trajectory") - - print(data.columns) - data["GenScore_normalized"] = normalize_genscore(data) - - print("Removing NA values") - subset = data[data.GenScore.notna()] - - endog, exog = dmatrices( - "GenScore_normalized ~ scale(StructureScore) * scale(np.log(Round))", # this works - data=subset, - return_type="dataframe", - ) - - if nested: - groups = nested_group_labels(subset["Producer"], subset["Target"]) - else: - groups = subset["Producer"] - - # model = sm.OLS(endog, exog) # Converges nicely, R^2=0.52 or something - model = sm.MixedLM(endog, exog, groups=groups) # converges with linear round - # model = sm.MixedLM(endog, exog, subset["Target"]) # converges with linear round - results = model.fit() - - save_summary_and_plot(6, results, outdir) - with plt.style.context("seaborn-paper"): - plt.figure(figsize=(8, 6)) - sns.relplot( - x="Round", - y="GenScore_normalized", - size="StructureScore", - kind="line", - data=data, - errorbar=ERRORBAR, - ) - plt.tight_layout(pad=1.0) - plt.savefig(os.path.join(outdir, f"model_6_relplot.png")) - - def run_model_6b(data: pd.DataFrame, outdir=".", nested=False): - """Learning trajectory GEN SCORE not normalized""" + """LME 3: Learning trajectory of un-normalized Generalization Score""" print("Running model 6 -- GenScore trajectory") subset = data[data.GenScore.notna()] @@ -1032,9 +856,7 @@ def run_model_6b(data: pd.DataFrame, outdir=".", nested=False): else: groups = subset["Producer"] - # model = sm.OLS(endog, exog) # Converges nicely, R^2=0.52 or something model = sm.MixedLM(endog, exog, groups=groups) # converges with linear round - # model = sm.MixedLM(endog, exog, subset["Target"]) # converges with linear round results = model.fit() save_summary_and_plot("6b", results, outdir, extra_desc="no-norm") @@ -1053,7 +875,7 @@ def run_model_6b(data: pd.DataFrame, outdir=".", nested=False): def run_model_7(data: pd.DataFrame, outdir=".", nested=False): - """Convergence score ~ Structure""" + """LME 4: Convergence score ~ Structure""" print("Running model 7 -- ConvScore trajectory") endog, exog = dmatrices( @@ -1086,7 +908,7 @@ def run_model_7(data: pd.DataFrame, outdir=".", nested=False): def run_model_8(data: pd.DataFrame, outdir=".", nested=False): - """Learning trajectory -- PROD SIM to Humans in Generalization""" + """LME 5: Learning trajectory -- PROD SIM to Humans in Generalization""" print( "Running model 8 -- Learning trajectory -- ProdSim to Humans in Generalization" ) @@ -1128,7 +950,7 @@ def run_model_8(data: pd.DataFrame, outdir=".", nested=False): def run_model_9(data: pd.DataFrame, outdir=".", nested=False): - """Learning trajectory -- PROD SIM to Humans in Memorization""" + """LME 2: Learning trajectory -- PROD SIM to Humans in Memorization""" print( "Running model 8 -- Learning trajectory -- ProdSim to Humans in Generalization" ) @@ -1283,37 +1105,24 @@ def main(): print("Will save models to", model_output_folder) os.makedirs(model_output_folder, exist_ok=True) - # run_model_4_with_crossed_random_effects(mem_data, outdir='/tmp') # -> Killed (OOM) - # run_model_4_with_nested_random_effects(mem_data, outdir='/tmp') # -> With slope per item -> Killed, else ok - - # run_model_1(mem_data) # Does not work yet - # run_model_4(mem_data, outdir=model_output_folder) - # run_model_6(reg_data, outdir=model_output_folder) - # run_model_7(reg_data, outdir=model_output_folder) - # run_model_8(reg_data, outdir=model_output_folder) # Does not converge with scaling, but centering is ok - # run_model_9(mem_data, outdir=model_output_folder) # Needs scaling to converge # FINAL MODELS - # run_model_1(mem_data, epoch=100) # not converged - # run_model_1(mem_data, epoch=50) # not converged + # Added QQ Plots 2024-10-07 + run_model_4(mem_data, outdir=model_output_folder, nested=True) + # run_model_6(reg_data, outdir=model_output_folder, nested=True) + run_model_6b(reg_data, outdir=model_output_folder, nested=True) # Gen score not normalized + + run_model_7(reg_data, outdir=model_output_folder, nested=True) + run_model_8(reg_data, outdir=model_output_folder, nested=True) + run_model_9(mem_data, outdir=model_output_folder, nested=True) - # Added CPPR plots 2022-08-10 + # Supplementary: Prod. Sim to ground truth at a specific epoch # run_model_2(mem_data, epoch=10, outdir=model_output_folder) # run_model_2(mem_data, epoch=40, outdir=model_output_folder) # run_model_2(mem_data, epoch=70, outdir=model_output_folder) # run_model_2(mem_data, epoch=100, outdir=model_output_folder) - # run_model_4(mem_data, outdir=model_output_folder, nested=True) - # run_model_6(reg_data, outdir=model_output_folder, nested=True) - - # TODO 6b updated? - # run_model_6b(reg_data, outdir=model_output_folder, nested=True) - - # run_model_7(reg_data, outdir=model_output_folder, nested=True) - # run_model_8(reg_data, outdir=model_output_folder, nested=True) - # run_model_9(mem_data, outdir=model_output_folder, nested=True) - - # old plots + # Supplementary: Interaction plots # reg_data["GenScore_normalized"] = normalize_genscore(reg_data) # make_interaction_plot(mem_data, 'Correct', outdir=model_output_folder, prefix='mem_') # make_interaction_plot(mem_data, 'ProdSim_GroundTruth', outdir=model_output_folder, prefix='mem_') @@ -1326,32 +1135,16 @@ def main(): print("Calculating human gen scores") human_raw_genscores = calc_generalization_score_for_humans(mem_data, reg_data) - # CUT PLOTS AT 60 + # Main paper: Plots cut at 60 cut = 60 plots_dir = "plots_binned_at_60" os.makedirs(plots_dir, exist_ok=True) - # make_memorization_plots(mem_data, outdir=plots_dir, cut=cut) - # make_memorization_panel(mem_data, outdir=plots_dir, cut=cut) - # make_generalization_plots( - # reg_data, outdir=plots_dir, human_raw_genscores=human_raw_genscores, cut=cut - # ) - # make_generalization_panel( - # reg_data, human_raw_genscores=human_raw_genscores, outdir=plots_dir, cut=cut - # ) make_big_panel(mem_data, reg_data, human_raw_genscores, outdir=plots_dir, cut=cut) - # UNCUT PLOTS (at 100) + # Supplementary: Plots uncut cut = None plots_dir = "plots_binned_at_100" os.makedirs(plots_dir, exist_ok=True) - # make_memorization_plots(mem_data, outdir=plots_dir, cut=cut) - # make_memorization_panel(mem_data, outdir=plots_dir, cut=cut) - # make_generalization_plots( - # reg_data, outdir=plots_dir, human_raw_genscores=human_raw_genscores, cut=cut - # ) - # make_generalization_panel( - # reg_data, human_raw_genscores=human_raw_genscores, outdir=plots_dir, cut=cut - # ) make_big_panel(mem_data, reg_data, human_raw_genscores, outdir=plots_dir, cut=cut)