diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 00000000..d6f17740 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper/paper.pdf \ No newline at end of file diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 00000000..752a130b --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,300 @@ + +@misc{duvenaud_structure_2013, + title = {Structure {Discovery} in {Nonparametric} {Regression} through {Compositional} {Kernel} {Search}}, + url = {http://arxiv.org/abs/1302.4922}, + doi = {10.48550/arXiv.1302.4922}, + abstract = {Despite its importance, choosing the structural form of the kernel in nonparametric regression remains a black art. We define a space of kernel structures which are built compositionally by adding and multiplying a small number of base kernels. We present a method for searching over this space of structures which mirrors the scientific discovery process. The learned structures can often decompose functions into interpretable components and enable long-range extrapolation on time-series datasets. Our structure search method outperforms many widely used kernels and kernel combination methods on a variety of prediction tasks.}, + urldate = {2024-08-28}, + publisher = {arXiv}, + author = {Duvenaud, David and Lloyd, James Robert and Grosse, Roger and Tenenbaum, Joshua B. and Ghahramani, Zoubin}, + month = may, + year = {2013}, + note = {arXiv:1302.4922 [cs, stat]}, + keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, G.3, I.2.6, Statistics - Methodology}, + annote = {Comment: 9 pages, 7 figures, To appear in proceedings of the 2013 International Conference on Machine Learning}, + file = {arXiv Fulltext PDF:/Users/mstoffel/Zotero/storage/7MYQSNHZ/Duvenaud et al. - 2013 - Structure Discovery in Nonparametric Regression through Compositional Kernel Search.pdf:application/pdf;arXiv.org Snapshot:/Users/mstoffel/Zotero/storage/7NFEGJMS/1302.html:text/html}, +} + +@inproceedings{bonilla_multi-task_2007, + title = {Multi-task {Gaussian} {Process} {Prediction}}, + volume = {20}, + url = {https://papers.nips.cc/paper_files/paper/2007/hash/66368270ffd51418ec58bd793f2d9b1b-Abstract.html}, + abstract = {In this paper we investigate multi-task learning in the context of Gaussian Pro- cesses (GP). We propose a model that learns a shared covariance function on input-dependent features and a “free-form” covariance matrix over tasks. This al- lows for good flexibility when modelling inter-task dependencies while avoiding the need for large amounts of data for training. We show that under the assump- tion of noise-free observations and a block design, predictions for a given task only depend on its target values and therefore a cancellation of inter-task trans- fer occurs. We evaluate the benefits of our model on two practical applications: a compiler performance prediction problem and an exam score prediction task. Additionally, we make use of GP approximations and properties of our model in order to provide scalability to large data sets.}, + urldate = {2024-08-28}, + booktitle = {Advances in {Neural} {Information} {Processing} {Systems}}, + publisher = {Curran Associates, Inc.}, + author = {Bonilla, Edwin V and Chai, Kian and Williams, Christopher}, + year = {2007}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/CALGMFRQ/Bonilla et al. - 2007 - Multi-task Gaussian Process Prediction.pdf:application/pdf}, +} + +@misc{garnelo_conditional_2018, + title = {Conditional {Neural} {Processes}}, + url = {http://arxiv.org/abs/1807.01613}, + abstract = {Deep neural networks excel at function approximation, yet they are typically trained from scratch for each new function. On the other hand, Bayesian methods, such as Gaussian Processes (GPs), exploit prior knowledge to quickly infer the shape of a new function at test time. Yet GPs are computationally expensive, and it can be hard to design appropriate priors. In this paper we propose a family of neural models, Conditional Neural Processes (CNPs), that combine the benefits of both. CNPs are inspired by the flexibility of stochastic processes such as GPs, but are structured as neural networks and trained via gradient descent. CNPs make accurate predictions after observing only a handful of training data points, yet scale to complex functions and large datasets. We demonstrate the performance and versatility of the approach on a range of canonical machine learning tasks, including regression, classification and image completion.}, + language = {en}, + urldate = {2024-09-16}, + publisher = {arXiv}, + author = {Garnelo, Marta and Rosenbaum, Dan and Maddison, Chris J. and Ramalho, Tiago and Saxton, David and Shanahan, Murray and Teh, Yee Whye and Rezende, Danilo J. and Eslami, S. M. Ali}, + month = jul, + year = {2018}, + note = {arXiv:1807.01613 [cs, stat]}, + keywords = {Computer Science - Machine Learning, Statistics - Machine Learning}, + file = {PDF:/Users/mstoffel/Zotero/storage/7P3Z5VCU/Garnelo et al. - 2018 - Conditional Neural Processes.pdf:application/pdf}, +} + +@article{kudela_recent_2022, + title = {Recent advances and applications of surrogate models for finite element method computations: a review}, + volume = {26}, + issn = {1433-7479}, + shorttitle = {Recent advances and applications of surrogate models for finite element method computations}, + url = {https://doi.org/10.1007/s00500-022-07362-8}, + doi = {10.1007/s00500-022-07362-8}, + abstract = {The utilization of surrogate models to approximate complex systems has recently gained increased popularity. Because of their capability to deal with black-box problems and lower computational requirements, surrogates were successfully utilized by researchers in various engineering and scientific fields. An efficient use of surrogates can bring considerable savings in computational resources and time. Since literature on surrogate modelling encompasses a large variety of approaches, the appropriate choice of a surrogate remains a challenging task. This review discusses significant publications where surrogate modelling for finite element method-based computations was utilized. We familiarize the reader with the subject, explain the function of surrogate modelling, sampling and model validation procedures, and give a description of the different surrogate types. We then discuss main categories where surrogate models are used: prediction, sensitivity analysis, uncertainty quantification, and surrogate-assisted optimization, and give detailed account of recent advances and applications. We review the most widely used and recently developed software tools that are used to apply the discussed techniques with ease. Based on a literature review of 180 papers related to surrogate modelling, we discuss major research trends, gaps, and practical recommendations. As the utilization of surrogate models grows in popularity, this review can function as a guide that makes surrogate modelling more accessible.}, + language = {en}, + number = {24}, + urldate = {2024-11-06}, + journal = {Soft Computing}, + author = {Kudela, Jakub and Matousek, Radomil}, + month = dec, + year = {2022}, + keywords = {Artificial Intelligence, Finite element method, Sensitivity analysis, Surrogate model, Surrogate-assisted optimization, Uncertainty quantification}, + pages = {13709--13733}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/N9J6YA8N/Kudela and Matousek - 2022 - Recent advances and applications of surrogate models for finite element method computations a revie.pdf:application/pdf}, +} + +@article{forrester_recent_2009, + title = {Recent advances in surrogate-based optimization}, + volume = {45}, + issn = {0376-0421}, + url = {https://www.sciencedirect.com/science/article/pii/S0376042108000766}, + doi = {10.1016/j.paerosci.2008.11.001}, + abstract = {The evaluation of aerospace designs is synonymous with the use of long running and computationally intensive simulations. This fuels the desire to harness the efficiency of surrogate-based methods in aerospace design optimization. Recent advances in surrogate-based design methodology bring the promise of efficient global optimization closer to reality. We review the present state of the art of constructing surrogate models and their use in optimization strategies. We make extensive use of pictorial examples and, since no method is truly universal, give guidance as to each method's strengths and weaknesses.}, + number = {1}, + urldate = {2024-11-06}, + journal = {Progress in Aerospace Sciences}, + author = {Forrester, Alexander I. J. and Keane, Andy J.}, + month = jan, + year = {2009}, + pages = {50--79}, + file = {Accepted Version:/Users/mstoffel/Zotero/storage/ZUDUR3QG/Forrester and Keane - 2009 - Recent advances in surrogate-based optimization.pdf:application/pdf;ScienceDirect Snapshot:/Users/mstoffel/Zotero/storage/L5RHBKN8/S0376042108000766.html:text/html}, +} + +@article{alizadeh_managing_2020, + title = {Managing computational complexity using surrogate models: a critical review}, + volume = {31}, + issn = {1435-6066}, + shorttitle = {Managing computational complexity using surrogate models}, + url = {https://doi.org/10.1007/s00163-020-00336-7}, + doi = {10.1007/s00163-020-00336-7}, + abstract = {In simulation-based realization of complex systems, we are forced to address the issue of computational complexity. One critical issue that must be addressed is the approximation of reality using surrogate models to replace expensive simulation models of engineering problems. In this paper, we critically review over 200 papers. We find that a framework for selecting appropriate surrogate modeling methods for a given function with specific requirements has been lacking. Having such a framework for surrogate model users, specifically practitioners in industry, is very important because there is very limited information about the performance of different models before applying them on the problem. Our contribution in this paper is to address this gap by creating practical guidance based on a trade-off among three main drivers, namely, size (how much information is necessary to compute the surrogate model), accuracy (how accurate the surrogate model must be) and computational time (how much time is required for the surrogate modeling process). Using the proposed guidance a huge amount of time is saved by avoiding time-consuming comparisons before selecting the appropriate surrogate model. To make this contribution, we review the state-of-the-art surrogate modeling literature to answer the following three questions: (1) What are the main classes of the design of experiment (DOE) methods, surrogate modeling methods and model-fitting methods based on the requirements of size, computational time, and accuracy? (2) Which surrogate modeling method is suitable based on the critical characteristics of the requirements of size, computational time and accuracy? (3) Which DOE is suitable based on the critical characteristics of the requirements of size, computational time and accuracy? Based on these three characteristics, we find six different qualitative categories for the surrogate models through a critical evaluation of the literature. These categories provide a framework for selecting an efficient surrogate modeling process to assist those who wish to select more appropriate surrogate modeling techniques for a given function. It is also summarized in Table 4 and Figs. 2, 3. MARS, response surface models, and kriging are more appropriate for large problems, acquiring less computation time and high accuracy, respectively. Also, Latin Hypercube, fractional factorial designs and D-Optimal designs are appropriate experimental designs. Our contribution is to propose a qualitative evaluation and a mental model which is based on quantitative results and findings of authors in the published literature. The value of such a framework is in providing practical guide for researchers and practitioners in industry to choose the most appropriate surrogate model based on incomplete information about an engineering design problem. Another contribution is to use three drivers, namely, computational time, accuracy, and problem size instead of using a single measure that authors generally use in the published literature.}, + language = {en}, + number = {3}, + urldate = {2024-11-06}, + journal = {Research in Engineering Design}, + author = {Alizadeh, Reza and Allen, Janet K. and Mistree, Farrokh}, + month = jul, + year = {2020}, + keywords = {Surrogate model, Computational complexity, Design, Meta model, Model selection, Response surface}, + pages = {275--298}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/3WBZ68IK/Alizadeh et al. - 2020 - Managing computational complexity using surrogate models a critical review.pdf:application/pdf}, +} + +@article{pedregosa_scikit-learn_2011, + title = {Scikit-learn: {Machine} learning in {Python}}, + volume = {12}, + shorttitle = {Scikit-learn}, + url = {https://www.jmlr.org/papers/volume12/pedregosa11a/pedregosa11a.pdf?ref=https:/}, + urldate = {2024-11-06}, + journal = {the Journal of machine Learning research}, + author = {Pedregosa, Fabian and Varoquaux, Gaël and Gramfort, Alexandre and Michel, Vincent and Thirion, Bertrand and Grisel, Olivier and Blondel, Mathieu and Prettenhofer, Peter and Weiss, Ron and Dubourg, Vincent}, + year = {2011}, + note = {Publisher: JMLR. org}, + pages = {2825--2830}, + file = {Available Version (via Google Scholar):/Users/mstoffel/Zotero/storage/DQIBAI7L/Pedregosa et al. - 2011 - Scikit-learn Machine learning in Python.pdf:application/pdf}, +} + +@article{bouhlel_python_2019, + title = {A {Python} surrogate modeling framework with derivatives}, + volume = {135}, + url = {https://www.sciencedirect.com/science/article/pii/S0965997818309360}, + urldate = {2024-11-07}, + journal = {Advances in Engineering Software}, + author = {Bouhlel, Mohamed Amine and Hwang, John T. and Bartoli, Nathalie and Lafage, Rémi and Morlier, Joseph and Martins, Joaquim RRA}, + year = {2019}, + note = {Publisher: Elsevier}, + pages = {102662}, + file = {Available Version (via Google Scholar):/Users/mstoffel/Zotero/storage/6CE92NTH/Bouhlel et al. - 2019 - A Python surrogate modeling framework with derivatives.pdf:application/pdf}, +} + +@article{mckay_comparison_1979, + title = {A {Comparison} of {Three} {Methods} for {Selecting} {Values} of {Input} {Variables} in the {Analysis} of {Output} from a {Computer} {Code}}, + volume = {21}, + issn = {0040-1706}, + url = {https://www.jstor.org/stable/1268522}, + doi = {10.2307/1268522}, + abstract = {Two types of sampling plans are examined as alternatives to simple random sampling in Monte Carlo studies. These plans are shown to be improvements over simple random sampling with respect to variance for a class of estimators which includes the sample mean and the empirical distribution function.}, + number = {2}, + urldate = {2024-11-08}, + journal = {Technometrics}, + author = {McKay, M. D. and Beckman, R. J. and Conover, W. J.}, + year = {1979}, + note = {Publisher: [Taylor \& Francis, Ltd., American Statistical Association, American Society for Quality]}, + pages = {239--245}, +} + +@article{gorissen_surrogate_2010, + title = {A surrogate modeling and adaptive sampling toolbox for computer based design}, + volume = {11}, + url = {https://www.jmlr.org/papers/volume11/gorissen10a/gorissen10a.pdf}, + urldate = {2024-11-08}, + journal = {Journal of machine learning research.-Cambridge, Mass.}, + author = {Gorissen, Dirk and Couckuyt, Ivo and Demeester, Piet and Dhaene, Tom and Crombecq, Karel}, + year = {2010}, + pages = {2051--2055}, + file = {Available Version (via Google Scholar):/Users/mstoffel/Zotero/storage/H5PMFBLH/Gorissen et al. - 2010 - A surrogate modeling and adaptive sampling toolbox for computer based design.pdf:application/pdf}, +} + +@misc{kim_attentive_2019, + title = {Attentive {Neural} {Processes}}, + url = {http://arxiv.org/abs/1901.05761}, + abstract = {Neural Processes (NPs) (Garnelo et al 2018a;b) approach regression by learning to map a context set of observed input-output pairs to a distribution over regression functions. Each function models the distribution of the output given an input, conditioned on the context. NPs have the benefit of fitting observed data efficiently with linear complexity in the number of context input-output pairs, and can learn a wide family of conditional distributions; they learn predictive distributions conditioned on context sets of arbitrary size. Nonetheless, we show that NPs suffer a fundamental drawback of underfitting, giving inaccurate predictions at the inputs of the observed data they condition on. We address this issue by incorporating attention into NPs, allowing each input location to attend to the relevant context points for the prediction. We show that this greatly improves the accuracy of predictions, results in noticeably faster training, and expands the range of functions that can be modelled.}, + urldate = {2024-11-11}, + publisher = {arXiv}, + author = {Kim, Hyunjik and Mnih, Andriy and Schwarz, Jonathan and Garnelo, Marta and Eslami, Ali and Rosenbaum, Dan and Vinyals, Oriol and Teh, Yee Whye}, + month = jul, + year = {2019}, + note = {arXiv:1901.05761}, + keywords = {Computer Science - Machine Learning, Statistics - Machine Learning}, + file = {Preprint PDF:/Users/mstoffel/Zotero/storage/IS64EU7A/Kim et al. - 2019 - Attentive Neural Processes.pdf:application/pdf;Snapshot:/Users/mstoffel/Zotero/storage/K57PNDVL/1901.html:text/html}, +} + +@article{gardner_gpytorch_2018, + title = {Gpytorch: {Blackbox} matrix-matrix gaussian process inference with gpu acceleration}, + volume = {31}, + shorttitle = {Gpytorch}, + url = {https://proceedings.neurips.cc/paper/2018/hash/27e8e17134dd7083b050476733207ea1-Abstract.html}, + urldate = {2024-11-18}, + journal = {Advances in neural information processing systems}, + author = {Gardner, Jacob and Pleiss, Geoff and Weinberger, Kilian Q. and Bindel, David and Wilson, Andrew G.}, + year = {2018}, +} + +@article{matthews_gpflow_2017, + title = {{GPflow}: {A} {Gaussian} process library using {TensorFlow}}, + volume = {18}, + shorttitle = {{GPflow}}, + url = {https://www.jmlr.org/papers/v18/16-537.html}, + number = {40}, + urldate = {2024-11-18}, + journal = {Journal of Machine Learning Research}, + author = {Matthews, Alexander G. de G. and Van Der Wilk, Mark and Nickson, Tom and Fujii, Keisuke and Boukouvalas, Alexis and Le, Pablo and Ghahramani, Zoubin and Hensman, James}, + year = {2017}, + pages = {1--6}, + file = {Available Version (via Google Scholar):/Users/mstoffel/Zotero/storage/748FHRT5/Matthews et al. - 2017 - GPflow A Gaussian process library using TensorFlow.pdf:application/pdf}, +} + +@misc{noauthor_notitle_nodate, +} + +@book{tietz_skorch_2017, + title = {skorch: {A} scikit-learn compatible neural network library that wraps {PyTorch}}, + url = {https://skorch.readthedocs.io/en/stable/}, + author = {Tietz, Marian and Fan, Thomas J. and Nouri, Daniel and Bossan, Benjamin and {skorch Developers}}, + month = jul, + year = {2017}, +} + +@article{strocchi_cell_2023, + title = {Cell to whole organ global sensitivity analysis on a four-chamber heart electromechanics model using {Gaussian} processes emulators}, + volume = {19}, + issn = {1553-7358}, + url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1011257}, + doi = {10.1371/journal.pcbi.1011257}, + abstract = {Cardiac pump function arises from a series of highly orchestrated events across multiple scales. Computational electromechanics can encode these events in physics-constrained models. However, the large number of parameters in these models has made the systematic study of the link between cellular, tissue, and organ scale parameters to whole heart physiology challenging. A patient-specific anatomical heart model, or digital twin, was created. Cellular ionic dynamics and contraction were simulated with the Courtemanche-Land and the ToR-ORd-Land models for the atria and the ventricles, respectively. Whole heart contraction was coupled with the circulatory system, simulated with CircAdapt, while accounting for the effect of the pericardium on cardiac motion. The four-chamber electromechanics framework resulted in 117 parameters of interest. The model was broken into five hierarchical sub-models: tissue electrophysiology, ToR-ORd-Land model, Courtemanche-Land model, passive mechanics and CircAdapt. For each sub-model, we trained Gaussian processes emulators (GPEs) that were then used to perform a global sensitivity analysis (GSA) to retain parameters explaining 90\% of the total sensitivity for subsequent analysis. We identified 45 out of 117 parameters that were important for whole heart function. We performed a GSA over these 45 parameters and identified the systemic and pulmonary peripheral resistance as being critical parameters for a wide range of volumetric and hemodynamic cardiac indexes across all four chambers. We have shown that GPEs provide a robust method for mapping between cellular properties and clinical measurements. This could be applied to identify parameters that can be calibrated in patient-specific models or digital twins, and to link cellular function to clinical indexes.}, + language = {en}, + number = {6}, + urldate = {2024-11-18}, + journal = {PLOS Computational Biology}, + author = {Strocchi, Marina and Longobardi, Stefano and Augustin, Christoph M. and Gsell, Matthias A. F. and Petras, Argyrios and Rinaldi, Christopher A. and Vigmond, Edward J. and Plank, Gernot and Oates, Chris J. and Wilkinson, Richard D. and Niederer, Steven A.}, + month = jun, + year = {2023}, + note = {Publisher: Public Library of Science}, + keywords = {Cardiac atria, Cardiac ventricles, Electrophysiology, Heart, Pericardium, Stiffness, Systolic pressure, Veins}, + pages = {e1011257}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/3BQ64JL9/Strocchi et al. - 2023 - Cell to whole organ global sensitivity analysis on a four-chamber heart electromechanics model using.pdf:application/pdf}, +} + +@incollection{mack_surrogate_2007, + address = {Berlin, Heidelberg}, + title = {Surrogate {Model}-{Based} {Optimization} {Framework}: {A} {Case} {Study} in {Aerospace} {Design}}, + isbn = {978-3-540-49774-5}, + shorttitle = {Surrogate {Model}-{Based} {Optimization} {Framework}}, + url = {https://doi.org/10.1007/978-3-540-49774-5_14}, + language = {en}, + urldate = {2024-11-18}, + booktitle = {Evolutionary {Computation} in {Dynamic} and {Uncertain} {Environments}}, + publisher = {Springer}, + author = {Mack, Yolanda and Goel, Tushar and Shyy, Wei and Haftka, Raphael}, + editor = {Yang, Shengxiang and Ong, Yew-Soon and Jin, Yaochu}, + year = {2007}, + doi = {10.1007/978-3-540-49774-5_14}, + pages = {323--342}, +} + +@article{yondo_review_2018, + title = {A review on design of experiments and surrogate models in aircraft real-time and many-query aerodynamic analyses}, + volume = {96}, + issn = {0376-0421}, + url = {https://www.sciencedirect.com/science/article/pii/S0376042117300611}, + doi = {10.1016/j.paerosci.2017.11.003}, + abstract = {Full scale aerodynamic wind tunnel testing, numerical simulation of high dimensional (full-order) aerodynamic models or flight testing are some of the fundamental but complex steps in the various design phases of recent civil transport aircrafts. Current aircraft aerodynamic designs have increase in complexity (multidisciplinary, multi-objective or multi-fidelity) and need to address the challenges posed by the nonlinearity of the objective functions and constraints, uncertainty quantification in aerodynamic problems or the restrained computational budgets. With the aim to reduce the computational burden and generate low-cost but accurate models that mimic those full order models at different values of the design variables, Recent progresses have witnessed the introduction, in real-time and many-query analyses, of surrogate-based approaches as rapid and cheaper to simulate models. In this paper, a comprehensive and state-of-the art survey on common surrogate modeling techniques and surrogate-based optimization methods is given, with an emphasis on models selection and validation, dimensionality reduction, sensitivity analyses, constraints handling or infill and stopping criteria. Benefits, drawbacks and comparative discussions in applying those methods are described. Furthermore, the paper familiarizes the readers with surrogate models that have been successfully applied to the general field of fluid dynamics, but not yet in the aerospace industry. Additionally, the review revisits the most popular sampling strategies used in conducting physical and simulation-based experiments in aircraft aerodynamic design. Attractive or smart designs infrequently used in the field and discussions on advanced sampling methodologies are presented, to give a glance on the various efficient possibilities to a priori sample the parameter space. Closing remarks foster on future perspectives, challenges and shortcomings associated with the use of surrogate models by aircraft industrial aerodynamicists, despite their increased interest among the research communities.}, + urldate = {2024-11-18}, + journal = {Progress in Aerospace Sciences}, + author = {Yondo, Raul and Andrés, Esther and Valero, Eusebio}, + month = jan, + year = {2018}, + keywords = {Aerodynamics design, Aircraft aerodynamics, Industrial optimization, Sampling strategies, Surrogate models, Surrogate-based optimization}, + pages = {23--61}, + file = {ScienceDirect Snapshot:/Users/mstoffel/Zotero/storage/U2A59TEM/S0376042117300611.html:text/html}, +} + +@article{bounceur_global_2015, + title = {Global sensitivity analysis of the climate–vegetation system to astronomical forcing: an emulator-based approach}, + volume = {6}, + issn = {2190-4979}, + shorttitle = {Global sensitivity analysis of the climate–vegetation system to astronomical forcing}, + url = {https://esd.copernicus.org/articles/6/205/2015/}, + doi = {10.5194/esd-6-205-2015}, + abstract = {A global sensitivity analysis is performed to describe the effects of astronomical forcing on the climate–vegetation system simulated by the model of intermediate complexity LOVECLIM in interglacial conditions. The methodology relies on the estimation of sensitivity measures, using a Gaussian process emulator as a fast surrogate of the climate model, calibrated on a set of well-chosen experiments. The outputs considered are the annual mean temperature and precipitation and the growing degree days (GDD). The experiments were run on two distinct land surface schemes to estimate the importance of vegetation feedbacks on climate variance. This analysis provides a spatial description of the variance due to the factors and their combinations, in the form of "fingerprints" obtained from the covariance indices. The results are broadly consistent with the current under-standing of Earth's climate response to the astronomical forcing. In particular, precession and obliquity are found to contribute in LOVECLIM equally to GDD in the Northern Hemisphere, and the effect of obliquity on the response of Southern Hemisphere temperature dominates precession effects. Precession dominates precipitation changes in subtropical areas. Compared to standard approaches based on a small number of simulations, the methodology presented here allows us to identify more systematically regions susceptible to experiencing rapid climate change in response to the smooth astronomical forcing change. In particular, we find that using interactive vegetation significantly enhances the expected rates of climate change, specifically in the Sahel (up to 50\% precipitation change in 1000 years) and in the Canadian Arctic region (up to 3° in 1000 years). None of the tested astronomical configurations were found to induce multiple steady states, but, at low obliquity, we observed the development of an oscillatory pattern that has already been reported in LOVECLIM. Although the mathematics of the analysis are fairly straightforward, the emulation approach still requires considerable care in its implementation. We discuss the effect of the choice of length scales and the type of emulator, and estimate uncertainties associated with specific computational aspects, to conclude that the principal component emulator is a good option for this kind of application.}, + language = {English}, + number = {1}, + urldate = {2024-11-18}, + journal = {Earth System Dynamics}, + author = {Bounceur, N. and Crucifix, M. and Wilkinson, R. D.}, + month = may, + year = {2015}, + note = {Publisher: Copernicus GmbH}, + pages = {205--224}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/U99HNEI2/Bounceur et al. - 2015 - Global sensitivity analysis of the climate–vegetation system to astronomical forcing an emulator-ba.pdf:application/pdf}, +} + +@article{angione_using_2022, + title = {Using machine learning as a surrogate model for agent-based simulations}, + volume = {17}, + issn = {1932-6203}, + url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0263150}, + doi = {10.1371/journal.pone.0263150}, + abstract = {In this proof-of-concept work, we evaluate the performance of multiple machine-learning methods as surrogate models for use in the analysis of agent-based models (ABMs). Analysing agent-based modelling outputs can be challenging, as the relationships between input parameters can be non-linear or even chaotic even in relatively simple models, and each model run can require significant CPU time. Surrogate modelling, in which a statistical model of the ABM is constructed to facilitate detailed model analyses, has been proposed as an alternative to computationally costly Monte Carlo methods. Here we compare multiple machine-learning methods for ABM surrogate modelling in order to determine the approaches best suited as a surrogate for modelling the complex behaviour of ABMs. Our results suggest that, in most scenarios, artificial neural networks (ANNs) and gradient-boosted trees outperform Gaussian process surrogates, currently the most commonly used method for the surrogate modelling of complex computational models. ANNs produced the most accurate model replications in scenarios with high numbers of model runs, although training times were longer than the other methods. We propose that agent-based modelling would benefit from using machine-learning methods for surrogate modelling, as this can facilitate more robust sensitivity analyses for the models while also reducing CPU time consumption when calibrating and analysing the simulation.}, + language = {en}, + number = {2}, + urldate = {2024-11-18}, + journal = {PLOS ONE}, + author = {Angione, Claudio and Silverman, Eric and Yaneske, Elisabeth}, + month = feb, + year = {2022}, + note = {Publisher: Public Library of Science}, + keywords = {Behavior, Simulation and modeling, Machine learning, Agent-based modeling, Artificial neural networks, Decision trees, Neural networks, Support vector machines}, + pages = {e0263150}, + file = {Full Text PDF:/Users/mstoffel/Zotero/storage/MQQPTDNY/Angione et al. - 2022 - Using machine learning as a surrogate model for agent-based simulations.pdf:application/pdf}, +} diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 00000000..cef9002b --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,96 @@ +--- +title: "AutoEmulate: A Python Package for semi-automated emulation" +tags: + - Python + - Surrogate modelling + - Emulation + - Simulation + - Machine Learning + - Gaussian Processes + - Neural Processes +authors: + - name: Martin A. Stoffel + orcid: 0000-0003-4030-3543 + affiliation: 1 + corresponding: true + - name: Bryan M. Li + orcid: 0000-0003-3144-4838 + affiliation: 1, 2 + - name: Kalle Westerling + orcid: 0000-0002-2014-332X + affiliation: 1 + - name: Sophie Arana + orcid: 0000-0002-2014-332X + affiliation: 1 + - name: Max Balmus + orcid: 0000-0002-6003-0178 + affiliation: 1, 3 + - name: Eric Daub + orcid: + affiliation: 1 + - name: Steve Niederer + orcid: 0000-0002-4612-6982 + affiliation: 1, 3 +affiliations: + - name: The Alan Turing Institute, London, United Kingdom + index: 1 + - name: University of Edinburgh, Edinburgh, United Kingdom + index: 2 + - name: Imperial College London, London, United Kingdom + index: 3 +date: 28 November 2024 +bibliography: paper.bib +--- + +# Summary + +Simulations are ubiquitous in research and application, but are often too slow and computationally expensive to deeply explore the underlying system. One solution is to create efficient emulators (also surrogate- or meta-models) to approximate simulations, but this requires substantial expertise. Here, we present AutoEmulate, a low-code, AutoML-style python package for emulation. AutoEmulate makes it easy to fit and compare emulators, abstracting away the need for extensive machine learning (ML) experimentation. The package includes a range of emulators, from Gaussian Processes, Support Vector Machines and Gradient Boosting Models to novel, experimental deep learning emulators such as Neural Processes (Garnelo et al., 2018). AutoEmulate also implements global sensitivity analysis as a common emulator application, and we aim to add other applications in the future. Finally, AutoEmulate is designed to be easy to contribute to by being modular, integrated with the scikit-learn ecosystem (Pedregosa et al., 2011), and well documented. We aim to iterate based on user feedback to make AutoEmulate a tool for end-to-end emulation across fields. + +# Statement of need + +To understand complex real-world systems, researchers and engineers often construct computer simulations. These can be computationally expensive and take minutes, hours or even days to run. For tasks like optimisation, sensitivity analysis or uncertainty quantification where thousands or even millions of runs are needed, a solution has long been to approximate simulations with efficient emulators, which can be orders of magnitudes faster (Forrester & Keane, 2009; Kudela & Matousek, 2022). Emulation is becoming increasingly widespread, ranging from engineering (Yondo et al., 2018), biomedical (Strocchi et al., 2023) and climate science (Bounceur et al., 2015) to agent-based models (Angione et al., 2022). A typical emulation pipeline involves three steps: 1. Sample selection, which involves evaluating the simulation at a small, strategically chosen set of inputs to create a representative dataset., 2. constructing a high-accuracy emulator using that dataset, which involves model selection, hyperparameter optimisation and evaluation and 3. applying the emulator to tasks such as prediction, sensitivity analysis, or optimisation. Building an emulator in particular is a key challenge for non-experts, as it can involve substantial machine learning experimentation, all within an ever increasing ecosystem of models and packages. This puts a substantial burden on practitioners whose main focus is to explore the underlying system, not building the emulator. + +AutoEmulate automates emulator building, with the goal to eventually streamline the whole emulation pipeline. For people new to ML, AutoEmulate compares, optimises and evaluates a range of models to create an efficient emulator for their simulation in just a few lines of code. For experienced surrogate modellers, it provides a reference set of cutting-edge emulators to quickly benchmark new models against. The package includes classic emulators such as Radial Basis Functions and Gaussian Processes, established ML models like Gradient Boosting and Support Vector Machines, as well as experimental deep learning emulators such as Conditional Neural Processes (Garnelo et al., 2018). AutoEmulate is built to be extensible. New emulators can be added using the scikit-learn estimator template and deep learning models with PyTorch (Paszke et al. 2019) are supported with little overhead through a skorch (Tietz et al., 2017) interface. AutoEmulate fills a gap in the current landscape of surrogate modeling tools as it’s both highly accessible for newcomers while providing cutting edge-methods for experienced surrogate modelers. In contrast, existing libraries either focus on lower level implementations of specific models, like GPflow (Matthews et al., 2017) and GPytorch (Gardner et al., 2018), provide multiple emulators but require to manually pre-process data, compare emulators and optimise parameters like SMT in Python (Bouhlel et al., 2019) or Surrogates.jl in Julia, or have a higher level of automation but are written in a commercial language like SUMO toolbox in Matlab (Gorissen et al., 2010). + +# Pipeline + +The minimal input for AutoEmulate are X, y, where X is a 2D array (e.g. numpy-array, Pandas DataFrame) containing one simulation parameter per column and their values in rows, and y is an array containing the corresponding simulation outputs, where y can be either single or multi-output. After a dataset X, y has been constructed by evaluating the original simulation, we can create an emulator with AutoEmulate in just three lines of code: + +```python +from autoemulate.compare import AutoEmulate + +# creating an emulator +ae = AutoEmulate() +ae.setup(X, y) # allows to customise pipeline +emulator = ae.compare() # compares emulators & returns +``` + +Under the hood, AutoEmulate runs a complete ML pipeline. It splits the data into training and test sets, standardises inputs, fits a set of user-specified emulators, compares them using cross-validation and optionally optimises hyperparameters using pre-defined search spaces. It then returns the emulator with the highest average cross-validation R^2 score. The results can then easily be summarised and visualised. + +```python +# cross-validation results +ae.summarise_cv() # cv metrics for each model +ae.plot_cv() # visualise best cv fold per model +``` + +After choosing an emulator based on its cross-validation performance, it can be evaluated on the test set, which by default is 20% of the original dataset. If the test-set performance is acceptable, the emulator can be refitted on the combined training and test data before applying it. + +```python +# evaluating the emulator +ae.evaluate(emulator) # test set scores +emulator = ae.refit(emulator) # refit using full data +``` + +The emulator can now be used as an efficient replacement for the original simulation by generating tens of thousands of new data points in milliseconds using predict(). We’ve also implemented global sensitivity analysis, a common use-case for emulators, which decomposes the variance in the output(s) into the contributions of the various simulation parameters and their interactions. + +```python +# application +emulator.predict(X) # generate new samples +ae.sensitivity_analysis(emulator) # global SA with Sobol indices +``` + +# Acknowledgements + +We thank the Turing Research and Innovation Cluster in Digital Twins for supporting this work. We also thank Keith Worden, Ieva Kazlauskaite, Rosie Williams, Robert Arthern, Peter Yatsyshin, Christopher Burr, James Byrne for discussions and Marina Strocci, Zack Xuereb Conti, Richard Wilkinson for also providing datasets. + +# References \ No newline at end of file