From b37d689b21470e6c4a10c6521f63f56cdcf2d420 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 1 May 2023 10:56:38 -0700 Subject: [PATCH] DOC: Move ecosystem.rst to getting started section (#52944) * DOC: Move ecosystem.rst to getting started section * Also error on ipython warnings * Migrate existing ecosystem to web and fix references --- doc/data/tips.csv | 245 ---------- doc/make.py | 2 +- doc/source/conf.py | 1 - doc/source/development/extending.rst | 4 +- doc/source/ecosystem.rst | 619 ------------------------ doc/source/user_guide/basics.rst | 2 +- doc/source/user_guide/scale.rst | 2 +- doc/source/user_guide/visualization.rst | 4 +- doc/source/whatsnew/v0.13.1.rst | 2 +- doc/source/whatsnew/v0.23.0.rst | 3 +- doc/source/whatsnew/v0.24.0.rst | 2 +- web/pandas/community/ecosystem.md | 166 ++++++- 12 files changed, 173 insertions(+), 879 deletions(-) delete mode 100644 doc/data/tips.csv delete mode 100644 doc/source/ecosystem.rst diff --git a/doc/data/tips.csv b/doc/data/tips.csv deleted file mode 100644 index 856a65a69e6475..00000000000000 --- a/doc/data/tips.csv +++ /dev/null @@ -1,245 +0,0 @@ -total_bill,tip,sex,smoker,day,time,size -16.99,1.01,Female,No,Sun,Dinner,2 -10.34,1.66,Male,No,Sun,Dinner,3 -21.01,3.5,Male,No,Sun,Dinner,3 -23.68,3.31,Male,No,Sun,Dinner,2 -24.59,3.61,Female,No,Sun,Dinner,4 -25.29,4.71,Male,No,Sun,Dinner,4 -8.77,2.0,Male,No,Sun,Dinner,2 -26.88,3.12,Male,No,Sun,Dinner,4 -15.04,1.96,Male,No,Sun,Dinner,2 -14.78,3.23,Male,No,Sun,Dinner,2 -10.27,1.71,Male,No,Sun,Dinner,2 -35.26,5.0,Female,No,Sun,Dinner,4 -15.42,1.57,Male,No,Sun,Dinner,2 -18.43,3.0,Male,No,Sun,Dinner,4 -14.83,3.02,Female,No,Sun,Dinner,2 -21.58,3.92,Male,No,Sun,Dinner,2 -10.33,1.67,Female,No,Sun,Dinner,3 -16.29,3.71,Male,No,Sun,Dinner,3 -16.97,3.5,Female,No,Sun,Dinner,3 -20.65,3.35,Male,No,Sat,Dinner,3 -17.92,4.08,Male,No,Sat,Dinner,2 -20.29,2.75,Female,No,Sat,Dinner,2 -15.77,2.23,Female,No,Sat,Dinner,2 -39.42,7.58,Male,No,Sat,Dinner,4 -19.82,3.18,Male,No,Sat,Dinner,2 -17.81,2.34,Male,No,Sat,Dinner,4 -13.37,2.0,Male,No,Sat,Dinner,2 -12.69,2.0,Male,No,Sat,Dinner,2 -21.7,4.3,Male,No,Sat,Dinner,2 -19.65,3.0,Female,No,Sat,Dinner,2 -9.55,1.45,Male,No,Sat,Dinner,2 -18.35,2.5,Male,No,Sat,Dinner,4 -15.06,3.0,Female,No,Sat,Dinner,2 -20.69,2.45,Female,No,Sat,Dinner,4 -17.78,3.27,Male,No,Sat,Dinner,2 -24.06,3.6,Male,No,Sat,Dinner,3 -16.31,2.0,Male,No,Sat,Dinner,3 -16.93,3.07,Female,No,Sat,Dinner,3 -18.69,2.31,Male,No,Sat,Dinner,3 -31.27,5.0,Male,No,Sat,Dinner,3 -16.04,2.24,Male,No,Sat,Dinner,3 -17.46,2.54,Male,No,Sun,Dinner,2 -13.94,3.06,Male,No,Sun,Dinner,2 -9.68,1.32,Male,No,Sun,Dinner,2 -30.4,5.6,Male,No,Sun,Dinner,4 -18.29,3.0,Male,No,Sun,Dinner,2 -22.23,5.0,Male,No,Sun,Dinner,2 -32.4,6.0,Male,No,Sun,Dinner,4 -28.55,2.05,Male,No,Sun,Dinner,3 -18.04,3.0,Male,No,Sun,Dinner,2 -12.54,2.5,Male,No,Sun,Dinner,2 -10.29,2.6,Female,No,Sun,Dinner,2 -34.81,5.2,Female,No,Sun,Dinner,4 -9.94,1.56,Male,No,Sun,Dinner,2 -25.56,4.34,Male,No,Sun,Dinner,4 -19.49,3.51,Male,No,Sun,Dinner,2 -38.01,3.0,Male,Yes,Sat,Dinner,4 -26.41,1.5,Female,No,Sat,Dinner,2 -11.24,1.76,Male,Yes,Sat,Dinner,2 -48.27,6.73,Male,No,Sat,Dinner,4 -20.29,3.21,Male,Yes,Sat,Dinner,2 -13.81,2.0,Male,Yes,Sat,Dinner,2 -11.02,1.98,Male,Yes,Sat,Dinner,2 -18.29,3.76,Male,Yes,Sat,Dinner,4 -17.59,2.64,Male,No,Sat,Dinner,3 -20.08,3.15,Male,No,Sat,Dinner,3 -16.45,2.47,Female,No,Sat,Dinner,2 -3.07,1.0,Female,Yes,Sat,Dinner,1 -20.23,2.01,Male,No,Sat,Dinner,2 -15.01,2.09,Male,Yes,Sat,Dinner,2 -12.02,1.97,Male,No,Sat,Dinner,2 -17.07,3.0,Female,No,Sat,Dinner,3 -26.86,3.14,Female,Yes,Sat,Dinner,2 -25.28,5.0,Female,Yes,Sat,Dinner,2 -14.73,2.2,Female,No,Sat,Dinner,2 -10.51,1.25,Male,No,Sat,Dinner,2 -17.92,3.08,Male,Yes,Sat,Dinner,2 -27.2,4.0,Male,No,Thur,Lunch,4 -22.76,3.0,Male,No,Thur,Lunch,2 -17.29,2.71,Male,No,Thur,Lunch,2 -19.44,3.0,Male,Yes,Thur,Lunch,2 -16.66,3.4,Male,No,Thur,Lunch,2 -10.07,1.83,Female,No,Thur,Lunch,1 -32.68,5.0,Male,Yes,Thur,Lunch,2 -15.98,2.03,Male,No,Thur,Lunch,2 -34.83,5.17,Female,No,Thur,Lunch,4 -13.03,2.0,Male,No,Thur,Lunch,2 -18.28,4.0,Male,No,Thur,Lunch,2 -24.71,5.85,Male,No,Thur,Lunch,2 -21.16,3.0,Male,No,Thur,Lunch,2 -28.97,3.0,Male,Yes,Fri,Dinner,2 -22.49,3.5,Male,No,Fri,Dinner,2 -5.75,1.0,Female,Yes,Fri,Dinner,2 -16.32,4.3,Female,Yes,Fri,Dinner,2 -22.75,3.25,Female,No,Fri,Dinner,2 -40.17,4.73,Male,Yes,Fri,Dinner,4 -27.28,4.0,Male,Yes,Fri,Dinner,2 -12.03,1.5,Male,Yes,Fri,Dinner,2 -21.01,3.0,Male,Yes,Fri,Dinner,2 -12.46,1.5,Male,No,Fri,Dinner,2 -11.35,2.5,Female,Yes,Fri,Dinner,2 -15.38,3.0,Female,Yes,Fri,Dinner,2 -44.3,2.5,Female,Yes,Sat,Dinner,3 -22.42,3.48,Female,Yes,Sat,Dinner,2 -20.92,4.08,Female,No,Sat,Dinner,2 -15.36,1.64,Male,Yes,Sat,Dinner,2 -20.49,4.06,Male,Yes,Sat,Dinner,2 -25.21,4.29,Male,Yes,Sat,Dinner,2 -18.24,3.76,Male,No,Sat,Dinner,2 -14.31,4.0,Female,Yes,Sat,Dinner,2 -14.0,3.0,Male,No,Sat,Dinner,2 -7.25,1.0,Female,No,Sat,Dinner,1 -38.07,4.0,Male,No,Sun,Dinner,3 -23.95,2.55,Male,No,Sun,Dinner,2 -25.71,4.0,Female,No,Sun,Dinner,3 -17.31,3.5,Female,No,Sun,Dinner,2 -29.93,5.07,Male,No,Sun,Dinner,4 -10.65,1.5,Female,No,Thur,Lunch,2 -12.43,1.8,Female,No,Thur,Lunch,2 -24.08,2.92,Female,No,Thur,Lunch,4 -11.69,2.31,Male,No,Thur,Lunch,2 -13.42,1.68,Female,No,Thur,Lunch,2 -14.26,2.5,Male,No,Thur,Lunch,2 -15.95,2.0,Male,No,Thur,Lunch,2 -12.48,2.52,Female,No,Thur,Lunch,2 -29.8,4.2,Female,No,Thur,Lunch,6 -8.52,1.48,Male,No,Thur,Lunch,2 -14.52,2.0,Female,No,Thur,Lunch,2 -11.38,2.0,Female,No,Thur,Lunch,2 -22.82,2.18,Male,No,Thur,Lunch,3 -19.08,1.5,Male,No,Thur,Lunch,2 -20.27,2.83,Female,No,Thur,Lunch,2 -11.17,1.5,Female,No,Thur,Lunch,2 -12.26,2.0,Female,No,Thur,Lunch,2 -18.26,3.25,Female,No,Thur,Lunch,2 -8.51,1.25,Female,No,Thur,Lunch,2 -10.33,2.0,Female,No,Thur,Lunch,2 -14.15,2.0,Female,No,Thur,Lunch,2 -16.0,2.0,Male,Yes,Thur,Lunch,2 -13.16,2.75,Female,No,Thur,Lunch,2 -17.47,3.5,Female,No,Thur,Lunch,2 -34.3,6.7,Male,No,Thur,Lunch,6 -41.19,5.0,Male,No,Thur,Lunch,5 -27.05,5.0,Female,No,Thur,Lunch,6 -16.43,2.3,Female,No,Thur,Lunch,2 -8.35,1.5,Female,No,Thur,Lunch,2 -18.64,1.36,Female,No,Thur,Lunch,3 -11.87,1.63,Female,No,Thur,Lunch,2 -9.78,1.73,Male,No,Thur,Lunch,2 -7.51,2.0,Male,No,Thur,Lunch,2 -14.07,2.5,Male,No,Sun,Dinner,2 -13.13,2.0,Male,No,Sun,Dinner,2 -17.26,2.74,Male,No,Sun,Dinner,3 -24.55,2.0,Male,No,Sun,Dinner,4 -19.77,2.0,Male,No,Sun,Dinner,4 -29.85,5.14,Female,No,Sun,Dinner,5 -48.17,5.0,Male,No,Sun,Dinner,6 -25.0,3.75,Female,No,Sun,Dinner,4 -13.39,2.61,Female,No,Sun,Dinner,2 -16.49,2.0,Male,No,Sun,Dinner,4 -21.5,3.5,Male,No,Sun,Dinner,4 -12.66,2.5,Male,No,Sun,Dinner,2 -16.21,2.0,Female,No,Sun,Dinner,3 -13.81,2.0,Male,No,Sun,Dinner,2 -17.51,3.0,Female,Yes,Sun,Dinner,2 -24.52,3.48,Male,No,Sun,Dinner,3 -20.76,2.24,Male,No,Sun,Dinner,2 -31.71,4.5,Male,No,Sun,Dinner,4 -10.59,1.61,Female,Yes,Sat,Dinner,2 -10.63,2.0,Female,Yes,Sat,Dinner,2 -50.81,10.0,Male,Yes,Sat,Dinner,3 -15.81,3.16,Male,Yes,Sat,Dinner,2 -7.25,5.15,Male,Yes,Sun,Dinner,2 -31.85,3.18,Male,Yes,Sun,Dinner,2 -16.82,4.0,Male,Yes,Sun,Dinner,2 -32.9,3.11,Male,Yes,Sun,Dinner,2 -17.89,2.0,Male,Yes,Sun,Dinner,2 -14.48,2.0,Male,Yes,Sun,Dinner,2 -9.6,4.0,Female,Yes,Sun,Dinner,2 -34.63,3.55,Male,Yes,Sun,Dinner,2 -34.65,3.68,Male,Yes,Sun,Dinner,4 -23.33,5.65,Male,Yes,Sun,Dinner,2 -45.35,3.5,Male,Yes,Sun,Dinner,3 -23.17,6.5,Male,Yes,Sun,Dinner,4 -40.55,3.0,Male,Yes,Sun,Dinner,2 -20.69,5.0,Male,No,Sun,Dinner,5 -20.9,3.5,Female,Yes,Sun,Dinner,3 -30.46,2.0,Male,Yes,Sun,Dinner,5 -18.15,3.5,Female,Yes,Sun,Dinner,3 -23.1,4.0,Male,Yes,Sun,Dinner,3 -15.69,1.5,Male,Yes,Sun,Dinner,2 -19.81,4.19,Female,Yes,Thur,Lunch,2 -28.44,2.56,Male,Yes,Thur,Lunch,2 -15.48,2.02,Male,Yes,Thur,Lunch,2 -16.58,4.0,Male,Yes,Thur,Lunch,2 -7.56,1.44,Male,No,Thur,Lunch,2 -10.34,2.0,Male,Yes,Thur,Lunch,2 -43.11,5.0,Female,Yes,Thur,Lunch,4 -13.0,2.0,Female,Yes,Thur,Lunch,2 -13.51,2.0,Male,Yes,Thur,Lunch,2 -18.71,4.0,Male,Yes,Thur,Lunch,3 -12.74,2.01,Female,Yes,Thur,Lunch,2 -13.0,2.0,Female,Yes,Thur,Lunch,2 -16.4,2.5,Female,Yes,Thur,Lunch,2 -20.53,4.0,Male,Yes,Thur,Lunch,4 -16.47,3.23,Female,Yes,Thur,Lunch,3 -26.59,3.41,Male,Yes,Sat,Dinner,3 -38.73,3.0,Male,Yes,Sat,Dinner,4 -24.27,2.03,Male,Yes,Sat,Dinner,2 -12.76,2.23,Female,Yes,Sat,Dinner,2 -30.06,2.0,Male,Yes,Sat,Dinner,3 -25.89,5.16,Male,Yes,Sat,Dinner,4 -48.33,9.0,Male,No,Sat,Dinner,4 -13.27,2.5,Female,Yes,Sat,Dinner,2 -28.17,6.5,Female,Yes,Sat,Dinner,3 -12.9,1.1,Female,Yes,Sat,Dinner,2 -28.15,3.0,Male,Yes,Sat,Dinner,5 -11.59,1.5,Male,Yes,Sat,Dinner,2 -7.74,1.44,Male,Yes,Sat,Dinner,2 -30.14,3.09,Female,Yes,Sat,Dinner,4 -12.16,2.2,Male,Yes,Fri,Lunch,2 -13.42,3.48,Female,Yes,Fri,Lunch,2 -8.58,1.92,Male,Yes,Fri,Lunch,1 -15.98,3.0,Female,No,Fri,Lunch,3 -13.42,1.58,Male,Yes,Fri,Lunch,2 -16.27,2.5,Female,Yes,Fri,Lunch,2 -10.09,2.0,Female,Yes,Fri,Lunch,2 -20.45,3.0,Male,No,Sat,Dinner,4 -13.28,2.72,Male,No,Sat,Dinner,2 -22.12,2.88,Female,Yes,Sat,Dinner,2 -24.01,2.0,Male,Yes,Sat,Dinner,4 -15.69,3.0,Male,Yes,Sat,Dinner,3 -11.61,3.39,Male,No,Sat,Dinner,2 -10.77,1.47,Male,No,Sat,Dinner,2 -15.53,3.0,Male,Yes,Sat,Dinner,2 -10.07,1.25,Male,No,Sat,Dinner,2 -12.6,1.0,Male,Yes,Sat,Dinner,2 -32.83,1.17,Male,Yes,Sat,Dinner,2 -35.83,4.67,Female,No,Sat,Dinner,3 -29.03,5.92,Male,No,Sat,Dinner,3 -27.18,2.0,Female,Yes,Sat,Dinner,2 -22.67,2.0,Male,Yes,Sat,Dinner,2 -17.82,1.75,Male,No,Sat,Dinner,2 -18.78,3.0,Female,No,Thur,Dinner,2 diff --git a/doc/make.py b/doc/make.py index f5bf170c6274dd..de298ce0475b19 100755 --- a/doc/make.py +++ b/doc/make.py @@ -321,7 +321,7 @@ def main(): help=( "filename (relative to the 'source' folder) of section or method name to " "compile, e.g. 'development/contributing.rst', " - "'ecosystem.rst', 'pandas.DataFrame.join'" + "'pandas.DataFrame.join'" ), ) argparser.add_argument( diff --git a/doc/source/conf.py b/doc/source/conf.py index 790a58aa37f182..ac318c5230275a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -467,7 +467,6 @@ } -ipython_warning_is_error = False ipython_execlines = [ "import numpy as np", "import pandas as pd", diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index b829cfced69620..f74eacb6b861d2 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -60,7 +60,7 @@ Now users can access your methods using the ``geo`` namespace: This can be a convenient way to extend pandas objects without subclassing them. If you write a custom accessor, make a pull request adding it to our -:ref:`ecosystem` page. +`ecosystem `_ page. We highly recommend validating the data in your accessor's ``__init__``. In our ``GeoAccessor``, we validate that the data contains the expected columns, @@ -91,7 +91,7 @@ objects). Many methods like :func:`pandas.isna` will dispatch to the extension type's implementation. If you're building a library that implements the interface, please publicize it -on :ref:`ecosystem.extensions`. +on `the ecosystem page `_. The interface consists of two classes. diff --git a/doc/source/ecosystem.rst b/doc/source/ecosystem.rst deleted file mode 100644 index 165be2a0925354..00000000000000 --- a/doc/source/ecosystem.rst +++ /dev/null @@ -1,619 +0,0 @@ -:orphan: - -.. _ecosystem: - -{{ header }} - -**************** -pandas ecosystem -**************** - -Increasingly, packages are being built on top of pandas to address specific needs -in data preparation, analysis and visualization. -This is encouraging because it means pandas is not only helping users to handle -their data tasks but also that it provides a better starting point for developers to -build powerful and more focused data tools. -The creation of libraries that complement pandas' functionality also allows pandas -development to remain focused around it's original requirements. - -This is an inexhaustive list of projects that build on pandas in order to provide -tools in the PyData space. For a list of projects that depend on pandas, -see the -`GitHub network dependents for pandas `_ -or `search pypi for pandas `_. - -We'd like to make it easier for users to find these projects, if you know of other -substantial projects that you feel should be on this list, please let us know. - -.. _ecosystem.data_cleaning_and_validation: - -Data cleaning and validation ----------------------------- - -`Pyjanitor `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Pyjanitor provides a clean API for cleaning data, using method chaining. - -`Pandera `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Pandera provides a flexible and expressive API for performing data validation on dataframes -to make data processing pipelines more readable and robust. -Dataframes contain information that pandera explicitly validates at runtime. This is useful in -production-critical data pipelines or reproducible research settings. - -`pandas-path `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since Python 3.4, `pathlib `_ has been -included in the Python standard library. Path objects provide a simple -and delightful way to interact with the file system. The pandas-path package enables the -Path API for pandas through a custom accessor ``.path``. Getting just the filenames from -a series of full file paths is as simple as ``my_files.path.name``. Other convenient operations like -joining paths, replacing file extensions, and checking if files exist are also available. - -.. _ecosystem.stats: - -Statistics and machine learning -------------------------------- - -`pandas-tfrecords `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Easy saving pandas dataframe to tensorflow tfrecords format and reading tfrecords to pandas. - -`Statsmodels `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Statsmodels is the prominent Python "statistics and econometrics library" and it has -a long-standing special relationship with pandas. Statsmodels provides powerful statistics, -econometrics, analysis and modeling functionality that is out of pandas' scope. -Statsmodels leverages pandas objects as the underlying data container for computation. - -`sklearn-pandas `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Use pandas DataFrames in your `scikit-learn `__ -ML pipeline. - -`Featuretools `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Featuretools is a Python library for automated feature engineering built on top of pandas. It excels at transforming temporal and relational datasets into feature matrices for machine learning using reusable feature engineering "primitives". Users can contribute their own primitives in Python and share them with the rest of the community. - -`Compose `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Compose is a machine learning tool for labeling data and prediction engineering. It allows you to structure the labeling process by parameterizing prediction problems and transforming time-driven relational data into target values with cutoff times that can be used for supervised learning. - -`STUMPY `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -STUMPY is a powerful and scalable Python library for modern time series analysis. -At its core, STUMPY efficiently computes something called a -`matrix profile `__, -which can be used for a wide variety of time series data mining tasks. - -.. _ecosystem.visualization: - -Visualization -------------- - -`Pandas has its own Styler class for table visualization `_, and while -:ref:`pandas also has built-in support for data visualization through charts with matplotlib `, -there are a number of other pandas-compatible libraries. - -`Altair `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Altair is a declarative statistical visualization library for Python. -With Altair, you can spend more time understanding your data and its -meaning. Altair's API is simple, friendly and consistent and built on -top of the powerful Vega-Lite JSON specification. This elegant -simplicity produces beautiful and effective visualizations with a -minimal amount of code. Altair works with pandas DataFrames. - - -`Bokeh `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Bokeh is a Python interactive visualization library for large datasets that natively uses -the latest web technologies. Its goal is to provide elegant, concise construction of novel -graphics in the style of Protovis/D3, while delivering high-performance interactivity over -large data to thin clients. - -`Pandas-Bokeh `__ provides a high level API -for Bokeh that can be loaded as a native pandas plotting backend via - -.. code:: python - - pd.set_option("plotting.backend", "pandas_bokeh") - -It is very similar to the matplotlib plotting backend, but provides interactive -web-based charts and maps. - - -`Seaborn `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Seaborn is a Python visualization library based on -`matplotlib `__. It provides a high-level, dataset-oriented -interface for creating attractive statistical graphics. The plotting functions -in seaborn understand pandas objects and leverage pandas grouping operations -internally to support concise specification of complex visualizations. Seaborn -also goes beyond matplotlib and pandas with the option to perform statistical -estimation while plotting, aggregating across observations and visualizing the -fit of statistical models to emphasize patterns in a dataset. - -`plotnine `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Hadley Wickham's `ggplot2 `__ is a foundational exploratory visualization package for the R language. -Based on `"The Grammar of Graphics" `__ it -provides a powerful, declarative and extremely general way to generate bespoke plots of any kind of data. -Various implementations to other languages are available. -A good implementation for Python users is `has2k1/plotnine `__. - -`IPython vega `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`IPython Vega `__ leverages `Vega -`__ to create plots within Jupyter Notebook. - -`Plotly `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`Plotly’s `__ `Python API `__ enables interactive figures and web shareability. Maps, 2D, 3D, and live-streaming graphs are rendered with WebGL and `D3.js `__. The library supports plotting directly from a pandas DataFrame and cloud-based collaboration. Users of `matplotlib, ggplot for Python, and Seaborn `__ can convert figures into interactive web-based plots. Plots can be drawn in `IPython Notebooks `__ , edited with R or MATLAB, modified in a GUI, or embedded in apps and dashboards. Plotly is free for unlimited sharing, and has `offline `__, or `on-premise `__ accounts for private use. - -`Lux `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`Lux `__ is a Python library that facilitates fast and easy experimentation with data by automating the visual data exploration process. To use Lux, simply add an extra import alongside pandas: - -.. code:: python - - import lux - import pandas as pd - - df = pd.read_csv("data.csv") - df # discover interesting insights! - -By printing out a dataframe, Lux automatically `recommends a set of visualizations `__ that highlights interesting trends and patterns in the dataframe. Users can leverage any existing pandas commands without modifying their code, while being able to visualize their pandas data structures (e.g., DataFrame, Series, Index) at the same time. Lux also offers a `powerful, intuitive language `__ that allow users to create `Altair `__, `matplotlib `__, or `Vega-Lite `__ visualizations without having to think at the level of code. - -`Qtpandas `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spun off from the main pandas library, the `qtpandas `__ -library enables DataFrame visualization and manipulation in PyQt4 and PySide applications. - -`D-Tale `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -D-Tale is a lightweight web client for visualizing pandas data structures. It -provides a rich spreadsheet-style grid which acts as a wrapper for a lot of -pandas functionality (query, sort, describe, corr...) so users can quickly -manipulate their data. There is also an interactive chart-builder using Plotly -Dash allowing users to build nice portable visualizations. D-Tale can be -invoked with the following command - -.. code:: python - - import dtale - - dtale.show(df) - -D-Tale integrates seamlessly with Jupyter notebooks, Python terminals, Kaggle -& Google Colab. Here are some demos of the `grid `__. - -`hvplot `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -hvPlot is a high-level plotting API for the PyData ecosystem built on `HoloViews `__. -It can be loaded as a native pandas plotting backend via - -.. code:: python - - pd.set_option("plotting.backend", "hvplot") - -.. _ecosystem.ide: - -IDE ---- - -`IPython `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -IPython is an interactive command shell and distributed computing -environment. IPython tab completion works with pandas methods and also -attributes like DataFrame columns. - -`Jupyter Notebook / Jupyter Lab `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Jupyter Notebook is a web application for creating Jupyter notebooks. -A Jupyter notebook is a JSON document containing an ordered list -of input/output cells which can contain code, text, mathematics, plots -and rich media. -Jupyter notebooks can be converted to a number of open standard output formats -(HTML, HTML presentation slides, LaTeX, PDF, ReStructuredText, Markdown, -Python) through 'Download As' in the web interface and ``jupyter convert`` -in a shell. - -pandas DataFrames implement ``_repr_html_`` and ``_repr_latex`` methods -which are utilized by Jupyter Notebook for displaying -(abbreviated) HTML or LaTeX tables. LaTeX output is properly escaped. -(Note: HTML tables may or may not be -compatible with non-HTML Jupyter output formats.) - -See :ref:`Options and Settings ` and -:ref:`Available Options ` -for pandas ``display.`` settings. - -`Quantopian/qgrid `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -qgrid is "an interactive grid for sorting and filtering -DataFrames in IPython Notebook" built with SlickGrid. - -`Spyder `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spyder is a cross-platform PyQt-based IDE combining the editing, analysis, -debugging and profiling functionality of a software development tool with the -data exploration, interactive execution, deep inspection and rich visualization -capabilities of a scientific environment like MATLAB or Rstudio. - -Its `Variable Explorer `__ -allows users to view, manipulate and edit pandas ``Index``, ``Series``, -and ``DataFrame`` objects like a "spreadsheet", including copying and modifying -values, sorting, displaying a "heatmap", converting data types and more. -pandas objects can also be renamed, duplicated, new columns added, -copied/pasted to/from the clipboard (as TSV), and saved/loaded to/from a file. -Spyder can also import data from a variety of plain text and binary files -or the clipboard into a new pandas DataFrame via a sophisticated import wizard. - -Most pandas classes, methods and data attributes can be autocompleted in -Spyder's `Editor `__ and -`IPython Console `__, -and Spyder's `Help pane `__ can retrieve -and render Numpydoc documentation on pandas objects in rich text with Sphinx -both automatically and on-demand. - - -.. _ecosystem.api: - -API ---- - -`pandas-datareader `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``pandas-datareader`` is a remote data access library for pandas (PyPI:``pandas-datareader``). -It is based on functionality that was located in ``pandas.io.data`` and ``pandas.io.wb`` but was -split off in v0.19. -See more in the `pandas-datareader docs `_: - -The following data feeds are available: - - * Google Finance - * Tiingo - * Morningstar - * IEX - * Robinhood - * Enigma - * Quandl - * FRED - * Fama/French - * World Bank - * OECD - * Eurostat - * TSP Fund Data - * Nasdaq Trader Symbol Definitions - * Stooq Index Data - * MOEX Data - -`Quandl/Python `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Quandl API for Python wraps the Quandl REST API to return -pandas DataFrames with timeseries indexes. - -`Pydatastream `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -PyDatastream is a Python interface to the -`Refinitiv Datastream (DWS) `__ -REST API to return indexed pandas DataFrames with financial data. -This package requires valid credentials for this API (non free). - -`pandaSDMX `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -pandaSDMX is a library to retrieve and acquire statistical data -and metadata disseminated in -`SDMX `_ 2.1, an ISO-standard -widely used by institutions such as statistics offices, central banks, -and international organisations. pandaSDMX can expose datasets and related -structural metadata including data flows, code-lists, -and data structure definitions as pandas Series -or MultiIndexed DataFrames. - -`fredapi `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -fredapi is a Python interface to the `Federal Reserve Economic Data (FRED) `__ -provided by the Federal Reserve Bank of St. Louis. It works with both the FRED database and ALFRED database that -contains point-in-time data (i.e. historic data revisions). fredapi provides a wrapper in Python to the FRED -HTTP API, and also provides several convenient methods for parsing and analyzing point-in-time data from ALFRED. -fredapi makes use of pandas and returns data in a Series or DataFrame. This module requires a FRED API key that -you can obtain for free on the FRED website. - -`dataframe_sql `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``dataframe_sql`` is a Python package that translates SQL syntax directly into -operations on pandas DataFrames. This is useful when migrating from a database to -using pandas or for users more comfortable with SQL looking for a way to interface -with pandas. - - -.. _ecosystem.domain: - -Domain specific ---------------- - -`Geopandas `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Geopandas extends pandas data objects to include geographic information which support -geometric operations. If your work entails maps and geographical coordinates, and -you love pandas, you should take a close look at Geopandas. - -`staircase `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -staircase is a data analysis package, built upon pandas and numpy, for modelling and -manipulation of mathematical step functions. It provides a rich variety of arithmetic -operations, relational operations, logical operations, statistical operations and -aggregations for step functions defined over real numbers, datetime and timedelta domains. - - -`xarray `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -xarray brings the labeled data power of pandas to the physical sciences by -providing N-dimensional variants of the core pandas data structures. It aims to -provide a pandas-like and pandas-compatible toolkit for analytics on -multi-dimensional arrays, rather than the tabular data for which pandas excels. - - -.. _ecosystem.io: - -IO --- - -`BCPandas `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -BCPandas provides high performance writes from pandas to Microsoft SQL Server, -far exceeding the performance of the native ``df.to_sql`` method. Internally, it uses -Microsoft's BCP utility, but the complexity is fully abstracted away from the end user. -Rigorously tested, it is a complete replacement for ``df.to_sql``. - -`Deltalake `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Deltalake python package lets you access tables stored in -`Delta Lake `__ natively in Python without the need to use Spark or -JVM. It provides the ``delta_table.to_pyarrow_table().to_pandas()`` method to convert -any Delta table into Pandas dataframe. - - -.. _ecosystem.out-of-core: - -Out-of-core ------------ - -`Blaze `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Blaze provides a standard API for doing computations with various -in-memory and on-disk backends: NumPy, pandas, SQLAlchemy, MongoDB, PyTables, -PySpark. - -`Cylon `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Cylon is a fast, scalable, distributed memory parallel runtime with a pandas -like Python DataFrame API. ”Core Cylon” is implemented with C++ using Apache -Arrow format to represent the data in-memory. Cylon DataFrame API implements -most of the core operators of pandas such as merge, filter, join, concat, -group-by, drop_duplicates, etc. These operators are designed to work across -thousands of cores to scale applications. It can interoperate with pandas -DataFrame by reading data from pandas or converting data to pandas so users -can selectively scale parts of their pandas DataFrame applications. - -.. code:: python - - from pycylon import read_csv, DataFrame, CylonEnv - from pycylon.net import MPIConfig - - # Initialize Cylon distributed environment - config: MPIConfig = MPIConfig() - env: CylonEnv = CylonEnv(config=config, distributed=True) - - df1: DataFrame = read_csv('/tmp/csv1.csv') - df2: DataFrame = read_csv('/tmp/csv2.csv') - - # Using 1000s of cores across the cluster to compute the join - df3: Table = df1.join(other=df2, on=[0], algorithm="hash", env=env) - - print(df3) - -`Dask `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Dask is a flexible parallel computing library for analytics. Dask -provides a familiar ``DataFrame`` interface for out-of-core, parallel and distributed computing. - -`Dask-ML `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Dask-ML enables parallel and distributed machine learning using Dask alongside existing machine learning libraries like Scikit-Learn, XGBoost, and TensorFlow. - -`Ibis `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Ibis offers a standard way to write analytics code, that can be run in multiple engines. It helps in bridging the gap between local Python environments (like pandas) and remote storage and execution systems like Hadoop components (like HDFS, Impala, Hive, Spark) and SQL databases (Postgres, etc.). - - -`Koalas `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Koalas provides a familiar pandas DataFrame interface on top of Apache Spark. It enables users to leverage multi-cores on one machine or a cluster of machines to speed up or scale their DataFrame code. - -`Modin `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``modin.pandas`` DataFrame is a parallel and distributed drop-in replacement -for pandas. This means that you can use Modin with existing pandas code or write -new code with the existing pandas API. Modin can leverage your entire machine or -cluster to speed up and scale your pandas workloads, including traditionally -time-consuming tasks like ingesting data (``read_csv``, ``read_excel``, -``read_parquet``, etc.). - -.. code:: python - - # import pandas as pd - import modin.pandas as pd - - df = pd.read_csv("big.csv") # use all your cores! - -`Odo `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Odo provides a uniform API for moving data between different formats. It uses -pandas own ``read_csv`` for CSV IO and leverages many existing packages such as -PyTables, h5py, and pymongo to move data between non pandas formats. Its graph -based approach is also extensible by end users for custom formats that may be -too specific for the core of odo. - -`Pandarallel `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Pandarallel provides a simple way to parallelize your pandas operations on all your CPUs by changing only one line of code. -If also displays progress bars. - -.. code:: python - - from pandarallel import pandarallel - - pandarallel.initialize(progress_bar=True) - - # df.apply(func) - df.parallel_apply(func) - - -`Vaex `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Increasingly, packages are being built on top of pandas to address specific needs in data preparation, analysis and visualization. Vaex is a Python library for Out-of-Core DataFrames (similar to pandas), to visualize and explore big tabular datasets. It can calculate statistics such as mean, sum, count, standard deviation etc, on an N-dimensional grid up to a billion (10\ :sup:`9`) objects/rows per second. Visualization is done using histograms, density plots and 3d volume rendering, allowing interactive exploration of big data. Vaex uses memory mapping, zero memory copy policy and lazy computations for best performance (no memory wasted). - - * vaex.from_pandas - * vaex.to_pandas_df - -.. _ecosystem.extensions: - -Extension data types --------------------- - -pandas provides an interface for defining -:ref:`extension types ` to extend NumPy's type -system. The following libraries implement that interface to provide types not -found in NumPy or pandas, which work well with pandas' data containers. - -`Cyberpandas`_ -~~~~~~~~~~~~~~ - -Cyberpandas provides an extension type for storing arrays of IP Addresses. These -arrays can be stored inside pandas' Series and DataFrame. - -`Pandas-Genomics`_ -~~~~~~~~~~~~~~~~~~ - -Pandas-Genomics provides extension types, extension arrays, and extension accessors for working with genomics data - -`Pint-Pandas`_ -~~~~~~~~~~~~~~ - -`Pint-Pandas `_ provides an extension type for -storing numeric arrays with units. These arrays can be stored inside pandas' -Series and DataFrame. Operations between Series and DataFrame columns which -use pint's extension array are then units aware. - -`Text Extensions for Pandas`_ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`Text Extensions for Pandas `_ -provides extension types to cover common data structures for representing natural language -data, plus library integrations that convert the outputs of popular natural language -processing libraries into Pandas DataFrames. - -.. _ecosystem.accessors: - -Accessors ---------- - -A directory of projects providing -:ref:`extension accessors `. This is for users to -discover new accessors and for library authors to coordinate on the namespace. - -================== ============ ==================================== =============================================================================== -Library Accessor Classes Description -================== ============ ==================================== =============================================================================== -`cyberpandas`_ ``ip`` ``Series`` Provides common operations for working with IP addresses. -`pdvega`_ ``vgplot`` ``Series``, ``DataFrame`` Provides plotting functions from the Altair_ library. -`pandas-genomics`_ ``genomics`` ``Series``, ``DataFrame`` Provides common operations for quality control and analysis of genomics data. -`pandas_path`_ ``path`` ``Index``, ``Series`` Provides `pathlib.Path`_ functions for Series. -`pint-pandas`_ ``pint`` ``Series``, ``DataFrame`` Provides units support for numeric Series and DataFrames. -`composeml`_ ``slice`` ``DataFrame`` Provides a generator for enhanced data slicing. -`datatest`_ ``validate`` ``Series``, ``DataFrame``, ``Index`` Provides validation, differences, and acceptance managers. -`woodwork`_ ``ww`` ``Series``, ``DataFrame`` Provides physical, logical, and semantic data typing information for Series and DataFrames. -`staircase`_ ``sc`` ``Series`` Provides methods for querying, aggregating and plotting step functions -================== ============ ==================================== =============================================================================== - -.. _cyberpandas: https://cyberpandas.readthedocs.io/en/latest -.. _pdvega: https://altair-viz.github.io/pdvega/ -.. _Altair: https://altair-viz.github.io/ -.. _pandas-genomics: https://pandas-genomics.readthedocs.io/en/latest/ -.. _pandas_path: https://github.com/drivendataorg/pandas-path/ -.. _pathlib.Path: https://docs.python.org/3/library/pathlib.html -.. _pint-pandas: https://github.com/hgrecco/pint-pandas -.. _composeml: https://github.com/alteryx/compose -.. _datatest: https://datatest.readthedocs.io/en/stable/ -.. _woodwork: https://github.com/alteryx/woodwork -.. _staircase: https://www.staircase.dev/ - -Development tools ------------------ - -`pandas-stubs `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -While pandas repository is partially typed, the package itself doesn't expose this information for external use. -Install pandas-stubs to enable basic type coverage of pandas API. - -Learn more by reading through :issue:`14468`, :issue:`26766`, :issue:`28142`. - -See installation and usage instructions on the `GitHub page `__. - -`Hamilton `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Hamilton is a declarative dataflow framework that came out of Stitch Fix. It was designed to help one manage a Pandas code base, specifically with respect to feature engineering for machine learning models. - -It prescibes an opinionated paradigm, that ensures all code is: - -* unit testable -* integration testing friendly -* documentation friendly -* transformation logic is reusable, as it is decoupled from the context of where it is used. -* integratable with runtime data quality checks. - -This helps one to scale your pandas code base, at the same time, keeping maintenance costs low. - -For more information, see `documentation `__. diff --git a/doc/source/user_guide/basics.rst b/doc/source/user_guide/basics.rst index 989d5128d7d085..34965a2a6d7110 100644 --- a/doc/source/user_guide/basics.rst +++ b/doc/source/user_guide/basics.rst @@ -2022,7 +2022,7 @@ does not support timezone-aware datetimes). pandas and third-party libraries *extend* NumPy's type system in a few places. This section describes the extensions pandas has made internally. See :ref:`extending.extension-types` for how to write your own extension that -works with pandas. See :ref:`ecosystem.extensions` for a list of third-party +works with pandas. See `the ecosystem page `_ for a list of third-party libraries that have implemented an extension. The following table lists all of pandas extension types. For methods requiring ``dtype`` diff --git a/doc/source/user_guide/scale.rst b/doc/source/user_guide/scale.rst index 261852692ab2ff..b66ddcd33a684d 100644 --- a/doc/source/user_guide/scale.rst +++ b/doc/source/user_guide/scale.rst @@ -248,7 +248,7 @@ Use other libraries pandas is just one library offering a DataFrame API. Because of its popularity, pandas' API has become something of a standard that other libraries implement. The pandas documentation maintains a list of libraries implementing a DataFrame API -in :ref:`our ecosystem page `. +in `the ecosystem page `_. For example, `Dask`_, a parallel computing library, has `dask.dataframe`_, a pandas-like API for working with larger than memory datasets in parallel. Dask diff --git a/doc/source/user_guide/visualization.rst b/doc/source/user_guide/visualization.rst index 1f198944793485..67799edf96ce22 100644 --- a/doc/source/user_guide/visualization.rst +++ b/doc/source/user_guide/visualization.rst @@ -23,7 +23,7 @@ We use the standard convention for referencing the matplotlib API: plt.close("all") We provide the basics in pandas to easily create decent looking plots. -See the :ref:`ecosystem ` section for visualization +See `the ecosystem page `_ for visualization libraries that go beyond the basics documented here. .. note:: @@ -1827,7 +1827,7 @@ This would be more or less equivalent to: The backend module can then use other visualization tools (Bokeh, Altair, hvplot,...) to generate the plots. Some libraries implementing a backend for pandas are listed -on the ecosystem :ref:`ecosystem.visualization` page. +on `the ecosystem page `_. Developers guide can be found at https://pandas.pydata.org/docs/dev/development/extending.html#plotting-backends diff --git a/doc/source/whatsnew/v0.13.1.rst b/doc/source/whatsnew/v0.13.1.rst index 249b9555b7fd4b..6a9ade92a8b1d2 100644 --- a/doc/source/whatsnew/v0.13.1.rst +++ b/doc/source/whatsnew/v0.13.1.rst @@ -17,7 +17,7 @@ Highlights include: - Will intelligently limit display precision for datetime/timedelta formats. - Enhanced Panel :meth:`~pandas.Panel.apply` method. - Suggested tutorials in new :ref:`Tutorials` section. -- Our pandas ecosystem is growing, We now feature related projects in a new :ref:`Pandas Ecosystem` section. +- Our pandas ecosystem is growing, We now feature related projects in a new `ecosystem page `_ section. - Much work has been taking place on improving the docs, and a new :ref:`Contributing` section has been added. - Even though it may only be of interest to devs, we <3 our new CI status page: `ScatterCI `__. diff --git a/doc/source/whatsnew/v0.23.0.rst b/doc/source/whatsnew/v0.23.0.rst index 9f24bc8e8ec507..5aba5ec061e8b8 100644 --- a/doc/source/whatsnew/v0.23.0.rst +++ b/doc/source/whatsnew/v0.23.0.rst @@ -236,8 +236,7 @@ array are respected: dtype: bool For more, see the :ref:`extension types ` -documentation. If you build an extension array, publicize it on our -:ref:`ecosystem page `. +documentation. If you build an extension array, publicize it on `the ecosystem page `_. .. _cyberpandas: https://cyberpandas.readthedocs.io/en/latest/ diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 8b850c7da37f39..fa4ae8f25cc05f 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -161,7 +161,7 @@ See :ref:`Dtypes ` and :ref:`Attributes and Underlying Data `, including -extension arrays registered by :ref:`3rd party libraries `. +extension arrays registered by `3rd party libraries `_. See the :ref:`dtypes docs ` for more on extension arrays. .. ipython:: python diff --git a/web/pandas/community/ecosystem.md b/web/pandas/community/ecosystem.md index 935ef12694a524..ebd1fc198dfcfe 100644 --- a/web/pandas/community/ecosystem.md +++ b/web/pandas/community/ecosystem.md @@ -8,9 +8,10 @@ developers to build powerful and more focused data tools. The creation of libraries that complement pandas' functionality also allows pandas development to remain focused around its original requirements. -This is an inexhaustive list of projects that build on pandas in order -to provide tools in the PyData space. For a list of projects that depend -on pandas, see the [libraries.io usage page for +This is an community-maintained list of projects that build on pandas in order +to provide tools in the PyData space. The pandas core development team does not necessarily endorse any particular project on this list or have any knowledge of the maintenance status of any particular library. + +For a more complete list of projects that depend on pandas, see the [libraries.io usage page for pandas](https://libraries.io/pypi/pandas/usage) or [search pypi for pandas](https://pypi.org/search/?q=pandas). @@ -20,6 +21,10 @@ please let us know. ## Statistics and machine learning +### [pandas-tfrecords](https://pypi.org/project/pandas-tfrecords/) + +Easy saving pandas dataframe to tensorflow tfrecords format and reading tfrecords to pandas. + ### [Statsmodels](https://www.statsmodels.org/) Statsmodels is the prominent Python "statistics and econometrics @@ -49,6 +54,13 @@ It allows you to structure the labeling process by parameterizing prediction problems and transforming time-driven relational data into target values with cutoff times that can be used for supervised learning. +### [STUMPY](https://github.com/TDAmeritrade/stumpy) + +STUMPY is a powerful and scalable Python library for modern time series analysis. +At its core, STUMPY efficiently computes something called a +`matrix profile `__, +which can be used for a wide variety of time series data mining tasks. + ## Visualization ### [Altair](https://altair-viz.github.io/) @@ -124,6 +136,20 @@ is free for unlimited sharing, and has [on-premise](https://plot.ly/product/enterprise/) accounts for private use. +### [Lux](https://github.com/lux-org/lux) + +Lux is a Python library that facilitates fast and easy experimentation with data by automating the visual data exploration process. To use Lux, simply add an extra import alongside pandas: + +```python +import lux +import pandas as pd + +df = pd.read_csv("data.csv") +df # discover interesting insights! +``` + +By printing out a dataframe, Lux automatically [recommends a set of visualizations](https://raw.githubusercontent.com/lux-org/lux-resources/master/readme_img/demohighlight.gif) that highlights interesting trends and patterns in the dataframe. Users can leverage any existing pandas commands without modifying their code, while being able to visualize their pandas data structures (e.g., DataFrame, Series, Index) at the same time. Lux also offers a [powerful, intuitive language](https://lux-api.readthedocs.io/en/latest/source/guide/vis.html>) that allow users to create Altair, matplotlib, or Vega-Lite visualizations without having to think at the level of code. + ### [QtPandas](https://github.com/draperjames/qtpandas) Spun off from the main pandas library, the @@ -131,6 +157,33 @@ Spun off from the main pandas library, the DataFrame visualization and manipulation in PyQt4 and PySide applications. +### [D-Tale](https://github.com/man-group/dtale) + +D-Tale is a lightweight web client for visualizing pandas data structures. It +provides a rich spreadsheet-style grid which acts as a wrapper for a lot of +pandas functionality (query, sort, describe, corr...) so users can quickly +manipulate their data. There is also an interactive chart-builder using Plotly +Dash allowing users to build nice portable visualizations. D-Tale can be +invoked with the following command + +```python +import dtale + +dtale.show(df) +``` + +D-Tale integrates seamlessly with Jupyter notebooks, Python terminals, Kaggle +& Google Colab. Here are some demos of the [grid](http://alphatechadmin.pythonanywhere.com/dtale/main/1). + +### [hvplot](https://hvplot.holoviz.org/index.html) + +hvPlot is a high-level plotting API for the PyData ecosystem built on `HoloViews `__. +It can be loaded as a native pandas plotting backend via + +```python +pd.set_option("plotting.backend", "hvplot") +``` + ## IDE ### [IPython](https://ipython.org/documentation.html) @@ -252,6 +305,13 @@ point-in-time data from ALFRED. fredapi makes use of pandas and returns data in a Series or DataFrame. This module requires a FRED API key that you can obtain for free on the FRED website. +### [dataframe_sql](https://github.com/zbrookle/dataframe_sql) + +``dataframe_sql`` is a Python package that translates SQL syntax directly into +operations on pandas DataFrames. This is useful when migrating from a database to +using pandas or for users more comfortable with SQL looking for a way to interface +with pandas. + ## Domain specific ### [Geopandas](https://github.com/geopandas/geopandas) @@ -261,6 +321,13 @@ which support geometric operations. If your work entails maps and geographical coordinates, and you love pandas, you should take a close look at Geopandas. +### [staricase](https://github.com/staircase-dev/staircase) + +staircase is a data analysis package, built upon pandas and numpy, for modelling and +manipulation of mathematical step functions. It provides a rich variety of arithmetic +operations, relational operations, logical operations, statistical operations and +aggregations for step functions defined over real numbers, datetime and timedelta domains. + ### [xarray](https://github.com/pydata/xarray) xarray brings the labeled data power of pandas to the physical sciences @@ -269,6 +336,22 @@ It aims to provide a pandas-like and pandas-compatible toolkit for analytics on multi-dimensional arrays, rather than the tabular data for which pandas excels. +## IO + +### [BCPandas](https://github.com/yehoshuadimarsky/bcpandas) + +BCPandas provides high performance writes from pandas to Microsoft SQL Server, +far exceeding the performance of the native ``df.to_sql`` method. Internally, it uses +Microsoft's BCP utility, but the complexity is fully abstracted away from the end user. +Rigorously tested, it is a complete replacement for ``df.to_sql``. + +### [Deltalake](https://pypi.org/project/deltalake) + +Deltalake python package lets you access tables stored in +`Delta Lake `__ natively in Python without the need to use Spark or +JVM. It provides the ``delta_table.to_pyarrow_table().to_pandas()`` method to convert +any Delta table into Pandas dataframe. + ## Out-of-core ### [Blaze](https://blaze.pydata.org/) @@ -277,6 +360,34 @@ Blaze provides a standard API for doing computations with various in-memory and on-disk backends: NumPy, Pandas, SQLAlchemy, MongoDB, PyTables, PySpark. +### [Cylon](https://cylondata.org/) + +Cylon is a fast, scalable, distributed memory parallel runtime with a pandas +like Python DataFrame API. ”Core Cylon” is implemented with C++ using Apache +Arrow format to represent the data in-memory. Cylon DataFrame API implements +most of the core operators of pandas such as merge, filter, join, concat, +group-by, drop_duplicates, etc. These operators are designed to work across +thousands of cores to scale applications. It can interoperate with pandas +DataFrame by reading data from pandas or converting data to pandas so users +can selectively scale parts of their pandas DataFrame applications. + +```python +from pycylon import read_csv, DataFrame, CylonEnv +from pycylon.net import MPIConfig + +# Initialize Cylon distributed environment +config: MPIConfig = MPIConfig() +env: CylonEnv = CylonEnv(config=config, distributed=True) + +df1: DataFrame = read_csv('/tmp/csv1.csv') +df2: DataFrame = read_csv('/tmp/csv2.csv') + +# Using 1000s of cores across the cluster to compute the join +df3: Table = df1.join(other=df2, on=[0], algorithm="hash", env=env) + +print(df3) +``` + ### [Dask](https://docs.dask.org) Dask is a flexible parallel computing library for analytics. Dask @@ -289,12 +400,33 @@ Dask-ML enables parallel and distributed machine learning using Dask alongside existing machine learning libraries like Scikit-Learn, XGBoost, and TensorFlow. +### [Ibis](https://ibis-project.org/docs/) + +Ibis offers a standard way to write analytics code, that can be run in multiple engines. It helps in bridging the gap between local Python environments (like pandas) and remote storage and execution systems like Hadoop components (like HDFS, Impala, Hive, Spark) and SQL databases (Postgres, etc.). + + ### [Koalas](https://koalas.readthedocs.io/en/latest/) Koalas provides a familiar pandas DataFrame interface on top of Apache Spark. It enables users to leverage multi-cores on one machine or a cluster of machines to speed up or scale their DataFrame code. +### [Modin](https://github.com/modin-project/modin) + +The ``modin.pandas`` DataFrame is a parallel and distributed drop-in replacement +for pandas. This means that you can use Modin with existing pandas code or write +new code with the existing pandas API. Modin can leverage your entire machine or +cluster to speed up and scale your pandas workloads, including traditionally +time-consuming tasks like ingesting data (``read_csv``, ``read_excel``, +``read_parquet``, etc.). + +```python +# import pandas as pd +import modin.pandas as pd + +df = pd.read_csv("big.csv") # use all your cores! +``` + ### [Odo](http://odo.pydata.org) Odo provides a uniform API for moving data between different formats. It @@ -303,6 +435,20 @@ packages such as PyTables, h5py, and pymongo to move data between non pandas formats. Its graph based approach is also extensible by end users for custom formats that may be too specific for the core of odo. +### [Pandarallel](https://github.com/nalepae/pandarallel) + +Pandarallel provides a simple way to parallelize your pandas operations on all your CPUs by changing only one line of code. +If also displays progress bars. + +```python +from pandarallel import pandarallel + +pandarallel.initialize(progress_bar=True) + +# df.apply(func) +df.parallel_apply(func) +``` + ### [Ray](https://docs.ray.io/en/latest/data/modin/index.html) Pandas on Ray is an early stage DataFrame library that wraps Pandas and @@ -342,6 +488,13 @@ wasted). Pyjanitor provides a clean API for cleaning data, using method chaining. +### [Pandera](https://pandera.readthedocs.io/en/stable/) + +Pandera provides a flexible and expressive API for performing data validation on dataframes +to make data processing pipelines more readable and robust. +Dataframes contain information that pandera explicitly validates at runtime. This is useful in +production-critical data pipelines or reproducible research settings. + ### [Engarde](https://engarde.readthedocs.io/en/latest/) Engarde is a lightweight library used to explicitly state your @@ -373,6 +526,10 @@ These arrays can be stored inside pandas' Series and DataFrame. Operations between Series and DataFrame columns which use pint's extension array are then units aware. +### [Text Extensions](https://ibm.biz/text-extensions-for-pandas) + +Text Extensions for Pandas provides extension types to cover common data structures for representing natural language data, plus library integrations that convert the outputs of popular natural language processing libraries into Pandas DataFrames. + ## Accessors A directory of projects providing @@ -387,6 +544,9 @@ authors to coordinate on the namespace. | [pandas_path](https://github.com/drivendataorg/pandas-path/) | `path` | `Index`, `Series` | | [pint-pandas](https://github.com/hgrecco/pint-pandas) | `pint` | `Series`, `DataFrame` | | [composeml](https://github.com/alteryx/compose) | `slice` | `DataFrame` | + | [datatest](https://datatest.readthedocs.io/en/stable/) | `validate` | `Series`, `DataFrame` | + | [composeml](https://github.com/alteryx/compose) | `slice` | `DataFrame` | + | [staircase](https://www.staircase.dev/) | `sc` | `Series`, `DataFrame` | | [woodwork](https://github.com/alteryx/woodwork) | `slice` | `Series`, `DataFrame` | ## Development tools