diff --git a/paper/paper.bib b/paper/paper.bib index 9e490f64..538936e4 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -269,6 +269,41 @@ @article{yondo_review_2018 file = {ScienceDirect Snapshot:/Users/mstoffel/Zotero/storage/U2A59TEM/S0376042117300611.html:text/html}, } +@article{saves_smt_2024, + title = {{SMT} 2.0: {A} {Surrogate} {Modeling} {Toolbox} with a focus on hierarchical and mixed variables {Gaussian} processes}, + volume = {188}, + issn = {0965-9978}, + shorttitle = {{SMT} 2.0}, + url = {https://www.sciencedirect.com/science/article/pii/S096599782300162X}, + doi = {10.1016/j.advengsoft.2023.103571}, + abstract = {The Surrogate Modeling Toolbox (SMT) is an open-source Python package that offers a collection of surrogate modeling methods, sampling techniques, and a set of sample problems. This paper presents SMT 2.0, a major new release of SMT that introduces significant upgrades and new features to the toolbox. This release adds the capability to handle mixed-variable surrogate models and hierarchical variables. These types of variables are becoming increasingly important in several surrogate modeling applications. SMT 2.0 also improves SMT by extending sampling methods, adding new surrogate models, and computing variance and kernel derivatives for Kriging. This release also includes new functions to handle noisy and use multi-fidelity data. To the best of our knowledge, SMT 2.0 is the first open-source surrogate library to propose surrogate models for hierarchical and mixed inputs. This open-source software is distributed under the New BSD license.22https://github.com/SMTorg/SMT.}, + urldate = {2024-12-02}, + journal = {Advances in Engineering Software}, + author = {Saves, Paul and Lafage, Rémi and Bartoli, Nathalie and Diouane, Youssef and Bussemaker, Jasper and Lefebvre, Thierry and Hwang, John T. and Morlier, Joseph and Martins, Joaquim R. R. A.}, + month = feb, + year = {2024}, + keywords = {Gaussian process, Hierarchical and mixed-categorical inputs, Hierarchical problems, Kriging, Meta variables, Surrogate modeling}, + pages = {103571}, + file = {ScienceDirect Snapshot:files/4269/S096599782300162X.html:text/html;Submitted Version:files/4270/Saves et al. - 2024 - SMT 2.0 A Surrogate Modeling Toolbox with a focus on hierarchical and mixed variables Gaussian proc.pdf:application/pdf}, +} + +@article{westermann_surrogate_2019, + title = {Surrogate modelling for sustainable building design – {A} review}, + volume = {198}, + issn = {0378-7788}, + url = {https://www.sciencedirect.com/science/article/pii/S0378778819302877}, + doi = {10.1016/j.enbuild.2019.05.057}, + abstract = {Statistical models can be used as surrogates of detailed simulation models. Their key advantage is that they are evaluated at low computational cost which can remove computational barriers in building performance simulation. This comprehensive review discusses significant publications in sustainable building design research where surrogate modelling was applied. First, we familiarize the reader with the field and begin by explaining the use of surrogate modelling for building design with regard to applications in the conceptual design stage, for sensitivity and uncertainty analysis, and for building design optimisation. This is complemented with practical instructions on the steps required to derive a surrogate model. Next, publications in the field are discussed and significant methodological findings highlighted. We have aggregated 57 studies in a comprehensive table with details on objective, sampling strategy and surrogate model type. Based on the literature major research trends are extracted and useful practical aspects outlined. As surrogate modelling may contribute to many sustainable building design problems, this review summarizes and aggregates past successes, and serves as practical guide to make surrogate modelling accessible for future researchers.}, + urldate = {2024-12-02}, + journal = {Energy and Buildings}, + author = {Westermann, Paul and Evins, Ralph}, + month = sep, + year = {2019}, + keywords = {Building design optimisation, Building performance simulation, Early design, Meta-model, Sensitivity analysis, Surrogate model, Sustainable building design, Uncertainty analysis}, + pages = {170--186}, + file = {ScienceDirect Snapshot:files/4273/S0378778819302877.html:text/html}, +} + @article{bounceur_global_2015, title = {Global sensitivity analysis of the climate–vegetation system to astronomical forcing: an emulator-based approach}, volume = {6}, diff --git a/paper/paper.md b/paper/paper.md index aaa7dbf8..ecd8f89a 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -48,27 +48,27 @@ Simulations are ubiquitous in research and application, but are often too slow a # Statement of need -To understand complex real-world systems, researchers and engineers often construct computer simulations. These can be computationally expensive and take minutes, hours or even days to run. For tasks like optimisation, sensitivity analysis or uncertainty quantification where thousands or even millions of runs are needed, a solution has long been to approximate simulations with efficient emulators, which can be orders of magnitudes faster [@forrester_recent_2009, @kudela_recent_2022]. Emulation is becoming increasingly widespread, ranging from engineering [@yondo_review_2018], biomedical [@strocci_cell_2023] and climate science [@bounceur_global_2015] to agent-based models [@angione_using_2022]. A typical emulation pipeline involves three steps: 1. Sample selection, which involves evaluating the simulation at a small, strategically chosen set of inputs to create a representative dataset., 2. constructing a high-accuracy emulator using that dataset, which involves model selection, hyperparameter optimisation and evaluation and 3. applying the emulator to tasks such as prediction, sensitivity analysis, or optimisation. Building an emulator in particular is a key challenge for non-experts, as it can involve substantial machine learning experimentation, all within an ever increasing ecosystem of models and packages. This puts a substantial burden on practitioners whose main focus is to explore the underlying system, not building the emulator. +To understand complex real-world systems, researchers and engineers often construct computer simulations. These can be computationally expensive and take minutes, hours or even days to run. For tasks like optimisation, sensitivity analysis or uncertainty quantification where thousands or even millions of runs are needed, a solution has long been to approximate simulations with efficient emulators, which can be orders of magnitudes faster [@forrester_recent_2009, @kudela_recent_2022]. Emulation is becoming increasingly widespread, ranging from engineering [@yondo_review_2018], architecture [@westermann_surrogate_2019], biomedical [@strocci_cell_2023] and climate science [@bounceur_global_2015], to agent-based models [@angione_using_2022]. A typical emulation pipeline involves three steps: 1. Evaluating the expensive simulation at a small, strategically chosen set of inputs using techniques such as Latin Hypercube Sampling [@mckay_comparison_1979] to create a representative dataset, 2. constructing a high-accuracy emulator using that dataset, which involves model selection, hyperparameter optimisation and evaluation and 3. applying the emulator to tasks such as prediction, sensitivity analysis, or optimisation. Building an emulator in particular is a key challenge which requires substantial machine learning experimentation within an ever increasing ecosystem of models and packages. This puts a substantial burden on practitioners whose main focus is to explore the underlying system, not building the emulator. -AutoEmulate automates emulator building, with the goal to eventually streamline the whole emulation pipeline. For people new to ML, AutoEmulate compares, optimises and evaluates a range of models to create an efficient emulator for their simulation in just a few lines of code. For experienced surrogate modellers, it provides a reference set of cutting-edge emulators to quickly benchmark new models against. The package includes classic emulators such as Radial Basis Functions and Gaussian Processes, established ML models like Gradient Boosting and Support Vector Machines, as well as experimental deep learning emulators such as [Conditional Neural Processes](https://yanndubs.github.io/Neural-Process-Family/text/Intro.html) [@garnelo_conditional_2018]. AutoEmulate is built to be extensible. Emulators follow the well established [scikit-learn estimator template](https://scikit-learn.org/1.5/developers/develop.html#rolling-your-own-estimator) and deep learning models written in PyTorch [@paszke_pytorch_2019] are supported with little overhead through a skorch [@tietz_skorch_2017] interface. AutoEmulate fills a gap in the current landscape of surrogate modeling tools as it’s both highly accessible for newcomers while providing cutting edge-methods for experienced surrogate modelers. In contrast, existing libraries either focus on lower level implementations of specific models, like GPflow [@matthews_gpflow_2017] and GPytorch [@gardner_gpytorch_2018], provide multiple emulators but require to manually pre-process data, compare emulators and optimise parameters like SMT in Python [@bouhlel_smt_2019] or Surrogates.jl in Julia [@gorissen_surrogates.jl_2010]. +AutoEmulate automates emulator building, with the goal to eventually streamline the whole emulation pipeline. For people new to ML, AutoEmulate compares, optimises and evaluates a range of models to create an efficient emulator for their simulation in just a few lines of code. For experienced surrogate modellers, AutoEmulate provides a reference set of cutting-edge emulators to quickly benchmark new models against. The package includes classic emulators such as Radial Basis Functions and Gaussian Processes, established ML models like Gradient Boosting and Support Vector Machines, as well as experimental deep learning emulators such as [Conditional Neural Processes](https://yanndubs.github.io/Neural-Process-Family/text/Intro.html) [@garnelo_conditional_2018]. AutoEmulate is built to be extensible. Emulators follow the popular [scikit-learn estimator template](https://scikit-learn.org/1.5/developers/develop.html#rolling-your-own-estimator) and deep learning models are supported with little overhead through PyTorch [@paszke_pytorch_2019] with a skorch [@tietz_skorch_2017] interface. AutoEmulate fills a gap in the current landscape of surrogate modeling tools as it’s both highly accessible for newcomers while providing cutting-edge emulators for experienced surrogate modelers. In contrast, existing libraries either focus on lower level implementations of specific models, like GPflow [@matthews_gpflow_2017] and GPytorch [@gardner_gpytorch_2018], provide multiple emulators but require to manually pre-process data, compare emulators and optimise hyperparameters like SMT in Python [@saves_smt_2024] or [Surrogates.jl](https://docs.sciml.ai/Surrogates/latest/) in Julia. # Pipeline -The inputs for AutoEmulate are X and y, where X is a 2D array (e.g. numpy-array, Pandas DataFrame) containing one simulation parameter per column and their values in rows, and y is an array containing the corresponding simulation outputs. A dataset X, y is usually constructed by sampling input parameters X using Latin Hypercube Sampling [@mckay_comparison_1979] and evaluating the simulation on these inputs to obtain outputs y. With X and y, we can create an emulator with AutoEmulate in just a few lines of code. +The inputs for AutoEmulate are `X` and `y`, where `X` is a 2D array (e.g. numpy-array, Pandas DataFrame) containing one simulation parameter per column and their values in rows, and `y` is an array containing the corresponding simulation outputs. A dataset `X`, `y` is usually obtained by constructing a set of parameters `X` using sampling techniques like Latin Hypercube Sampling [@mckay_comparison_1979] and evaluating the simulation on these inputs to obtain outputs `y`. With `X` and `y`, we can create an emulator with AutoEmulate in just a few lines of code. ```python from autoemulate.compare import AutoEmulate ae = AutoEmulate() -ae.setup(X, y) # customise pipeline -ae.compare() # runs the pipeline +ae.setup(X, y) # customise pipeline +ae.compare() # runs the pipeline ``` Under the hood, AutoEmulate runs a complete ML pipeline. It splits the data into training and test sets, standardises inputs, fits a set of user-specified emulators, compares them using cross-validation and optionally optimises hyperparameters using pre-defined search spaces. All these steps can be customised in `setup()`. After running `compare()`, the cross-validation results can be visualised and summarised. ```python -ae.plot_cv() # visualise results -ae.summarise_cv() # metrics for each model +ae.plot_cv() # visualise results +ae.summarise_cv() # cv scores for each model ``` : Average cross-validation scores @@ -84,22 +84,22 @@ ae.summarise_cv() # metrics for each model | LightGBM | lgbm | 0.6044 | 0.4930 | | Second Order Polynomial | sop | 0.8378 | 0.0297 | -After choosing an emulator based on its cross-validation performance, it can be evaluated on the test set, which by default is 20% of the original dataset. AutoEmulate provides various visualisations in addition to the metrics. +After selecting an emulator based on its cross-validation performance, it can be evaluated on the held-out test set. AutoEmulate again calculates RMSE and R² scores, and provides several visualisations to assess the quality of predictions. ```python -emulator = ae.get_model("GaussianProcess") -ae.evaluate(emulator) # calculate test set scores -ae.plot_eval(emulator) # visualise test set predictions +emulator = ae.get_model("GaussianProcess") # select fitted emulator +ae.evaluate(emulator) # calculate test set scores +ae.plot_eval(emulator) # visualise test set predictions ``` ![Test set predictions](eval_2.png) -If the test-set performance is acceptable, the emulator can be refitted on the combined training and test data before applying it. It's now ready to be used as an efficient replacement for the original simulation, and is able to generate tens of thousands of new data points in seconds using predict(). We implemented global sensitivity analysis as a common use-case, which decomposes the variance in the outputs into the contributions of the various simulation parameters and their interactions. +Finally, the emulator can be refitted on the combined training and test set data before applying it. It's now ready to be used as an efficient replacement for the original simulation, and is able to generate tens of thousands of new data points in negligible time using predict(). We also implemented global sensitivity analysis as a common emulator application, which decomposes the variance in the outputs into the contributions of the various simulation parameters and their interactions. ```python -emulator = ae.refit(emulator) # refit using full data -emulator.predict(X) # emulate! -ae.sensitivity_analysis(emulator) # global SA with Sobol indices +emulator = ae.refit(emulator) # refit using full data +emulator.predict(X) # emulate! +ae.sensitivity_analysis(emulator) # global SA with Sobol indices ``` # Acknowledgements