diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml new file mode 100644 index 0000000..31fddd3 --- /dev/null +++ b/.github/workflows/contributors.yml @@ -0,0 +1,52 @@ +name: Monthly contributor report +on: + workflow_dispatch: + schedule: + - cron: '3 2 1 * *' + +permissions: + contents: read + +jobs: + contributor_report: + name: contributor report + runs-on: ubuntu-latest + permissions: + issues: write + + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Get start and end dates + shell: bash + run: | + # Some date before the start of the Herb.jl project + start_date=2023-01-01 + + # Calculate the last day of the previous month + end_date=$(date -d "$start_date +1 month -1 day" +%Y-%m-%d) + + #Set an environment variable with the date range + echo "START_DATE=$start_date" >> "$GITHUB_ENV" + echo "END_DATE=$end_date" >> "$GITHUB_ENV" + + - name: Run contributor action + uses: github/contributors@v1 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + START_DATE: ${{ env.START_DATE }} + END_DATE: ${{ env.END_DATE }} + ORGANIZATION: Herb-AI + SPONSOR_INFO: "true" + LINK_TO_PROFILE: "true" + + - name: Move contributors file to repo directory + shell: bash + run: mv contibutors.md ${{ github.repository }}/CONTRIBUTORS.md + + - name: Add and commit CONTRIBUTORS.md + uses: EndBug/add-and-commit@v9 + with: + message: "Monthly contributors report" + default_author: github_actor diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 268c42f..c9ba3d5 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -14,6 +14,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade jupyterlab nbconvert - uses: julia-actions/setup-julia@v1 with: version: '1' diff --git a/Project.toml b/Project.toml index ce5f4b2..80bcf85 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Herb" uuid = "c09c6b7f-4f63-49de-90d9-97a3563c0f4a" authors = ["Jaap de Jong ", "Tilman Hinnerichs ", "Sebastijan Dumancic "] -version = "0.2.0" +version = "0.3.0" [deps] HerbConstraints = "1fa96474-3206-4513-b4fa-23913f296dfc" @@ -13,11 +13,11 @@ HerbSpecification = "6d54aada-062f-46d8-85cf-a1ceaf058a06" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] -HerbConstraints = "^0.1.1" -HerbCore = "^0.2.0" +HerbConstraints = "^0.2.0" +HerbCore = "^0.3.0" +HerbGrammar = "^0.3.0" HerbInterpret = "^0.1.2" -HerbGrammar = "^0.2.1" -HerbSearch = "^0.2.0" +HerbSearch = "^0.3.0" HerbSpecification = "^0.1.0" julia = "^1.8" diff --git a/docs/Project.toml b/docs/Project.toml index 2fdf779..06eab32 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -8,6 +8,7 @@ HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7" HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7" HerbSearch = "3008d8e8-f9aa-438a-92ed-26e9c7b4829f" HerbSpecification = "6d54aada-062f-46d8-85cf-a1ceaf058a06" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" [compat] Documenter = "1" diff --git a/docs/make.jl b/docs/make.jl index b6c685a..51e5ecd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -9,6 +9,18 @@ using HerbInterpret using HerbCore using HerbSpecification +# Use jupyter.nbconver to convert notebooks to markdown +using PyCall +jupyter = pyimport("jupyterlab") +nbconvert = pyimport("nbconvert") +all_notebooks = readdir("docs/src/tutorials/") +for file in all_notebooks + if occursin("ipynb", file) + path = "docs/src/tutorials/" * file + run(`jupyter nbconvert --to markdown $path`) + end +end + makedocs( modules=[HerbConstraints, HerbSearch, HerbGrammar, HerbSpecification, HerbInterpret, HerbCore], authors="PONYs", @@ -24,7 +36,9 @@ makedocs( "A more verbose getting started with Herb.jl" => "tutorials/getting_started_with_herb.md", "Defining Grammars in Herb.jl" => "tutorials/defining_grammars.md", "Advanced Search Procedures" => "tutorials/advanced_search.md" - "Top Down Iterator" => "tutorials/TopDown.md" + "Top Down Iterator" => "tutorials/TopDown.md", + "Getting started with Constraints" => "tutorials/getting_started_with_constraints.md", + "Working with custom interpreters" => "tutorials/working_with_interpreters.md" ], "Sub-Modules" => [ "HerbCore.jl" => "HerbCore/index.md", diff --git a/docs/src/tutorials/defining_grammars.ipynb b/docs/src/tutorials/defining_grammars.ipynb index c0828fe..8bbfb79 100644 --- a/docs/src/tutorials/defining_grammars.ipynb +++ b/docs/src/tutorials/defining_grammars.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Defining Grammars in Herb.jl\n", + "# Defining Grammars in Herb.jl using HerbGrammar\n", "\n", "The program space in Herb.jl is defined using a grammar. \n", "This notebook demonstrates how such a grammar can be created. \n", @@ -15,17 +15,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Setup" + "### Setup\n", + "First, we import the necessary Herb packages." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "include(\"../../src/Herb.jl\") # this can be deleted when we have modules\n", - "using ..Herb" + "using HerbGrammar, HerbConstraints" ] }, { @@ -39,30 +39,16 @@ "The grammar is defined using the `@cfgrammar` macro. \n", "This macro converts the grammar definition in the form of a Julia expression into Herb's internal grammar representation. \n", "Macro's are executed during compilation.\n", - "If you want to load a grammar during execution, have a look at the `Herb.HerbGrammar.expr2cfgrammar` function." + "If you want to load a grammar during execution, have a look at the `HerbGrammar.expr2cfgrammar` function." ] }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 1\n", - "2: Int = 2\n", - "3: Int = 3\n", - "4: Int = Int * Int\n", - "5: Int = Int + Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₁ = Herb.HerbGrammar.@cfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₁ = HerbGrammar.@cfgrammar begin\n", " Int = 1\n", " Int = 2\n", " Int = 3\n", @@ -81,32 +67,11 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int * Int\n", - "12: Int = Int + Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₂ = Herb.HerbGrammar.@cfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₂ = HerbGrammar.@cfgrammar begin\n", " Int = |(0:9)\n", " Int = Int * Int\n", " Int = Int + Int\n", @@ -122,27 +87,11 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 2\n", - "3: Int = 4\n", - "4: Int = 6\n", - "5: Int = 8\n", - "6: Int = Int * Int\n", - "7: Int = Int + Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₃ = Herb.HerbGrammar.@cfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₃ = HerbGrammar.@cfgrammar begin\n", " Int = |([0, 2, 4, 6, 8])\n", " Int = Int * Int\n", " Int = Int + Int\n", @@ -158,33 +107,11 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int * Int\n", - "12: Int = Int + Int\n", - "13: Int = x\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₄ = Herb.HerbGrammar.@cfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₄ = HerbGrammar.@cfgrammar begin\n", " Int = |(0:9)\n", " Int = Int * Int\n", " Int = Int + Int\n", @@ -203,36 +130,13 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int * Int\n", - "12: Int = Int + Int\n", - "13: Int = f(Int)\n", - "14: Int = x\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "f(a) = a + 1\n", "\n", - "g₅ = Herb.HerbGrammar.@cfgrammar begin\n", + "g₅ = HerbGrammar.@cfgrammar begin\n", " Int = |(0:9)\n", " Int = Int * Int\n", " Int = Int + Int\n", @@ -251,35 +155,13 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = a\n", - "12: Int = Int + Int\n", - "13: Int = Int × Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "×(a, b) = a * b\n", "\n", - "g₆ = Herb.HerbGrammar.@cfgrammar begin\n", + "g₆ = HerbGrammar.@cfgrammar begin\n", " Int = |(0:9)\n", " Int = a\n", " Int = Int + Int\n", @@ -304,34 +186,11 @@ }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "13-element Vector{Tuple{Int64, Any}}:\n", - " (1, 0)\n", - " (2, 1)\n", - " (3, 2)\n", - " (4, 3)\n", - " (5, 4)\n", - " (6, 5)\n", - " (7, 6)\n", - " (8, 7)\n", - " (9, 8)\n", - " (10, 9)\n", - " (11, :(Int + Int))\n", - " (12, :(Int * Int))\n", - " (13, :x)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₇ = Herb.HerbGrammar.@cfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₇ = HerbGrammar.@cfgrammar begin\n", " Int = |(0:9)\n", " Int = Int + Int\n", " Int = Int * Int\n", @@ -359,40 +218,20 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "true" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.isterminal(g₇, 1)" + "HerbGrammar.isterminal(g₇, 1)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "false" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.isterminal(g₇, 11)" + "HerbGrammar.isterminal(g₇, 11)" ] }, { @@ -406,21 +245,11 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - ":Int" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.return_type(g₇, 11)" + "HerbGrammar.return_type(g₇, 11)" ] }, { @@ -435,42 +264,20 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2-element Vector{Symbol}:\n", - " :Int\n", - " :Int" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.child_types(g₇, 11)" + "HerbGrammar.child_types(g₇, 11)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.nchildren(g₇, 11)" + "HerbGrammar.nchildren(g₇, 11)" ] }, { @@ -484,22 +291,11 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1-element Vector{Symbol}:\n", - " :Int" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.nonterminals(g₇)" + "HerbGrammar.nonterminals(g₇)" ] }, { @@ -518,34 +314,11 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int + Int\n", - "12: Int = Int * Int\n", - "13: Int = x\n", - "14: Int = Int - Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Herb.HerbGrammar.add_rule!(g₇, :(Int = Int - Int))" + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HerbGrammar.add_rule!(g₇, :(Int = Int - Int))" ] }, { @@ -567,65 +340,20 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: nothing = nothing\n", - "12: Int = Int * Int\n", - "13: Int = x\n", - "14: Int = Int - Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Herb.HerbGrammar.remove_rule!(g₇, 11)" + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HerbGrammar.remove_rule!(g₇, 11)" ] }, { "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int * Int\n", - "12: Int = x\n", - "13: Int = Int - Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Herb.HerbGrammar.cleanup_removed_rules!(g₇)" + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HerbGrammar.cleanup_removed_rules!(g₇)" ] }, { @@ -641,33 +369,11 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int + Int\n", - "12: Int = Int * Int\n", - "13: Int = x\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₈ = Herb.HerbGrammar.@csgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₈ = HerbGrammar.@csgrammar begin\n", " Int = |(0:9)\n", " Int = Int + Int\n", " Int = Int * Int\n", @@ -693,22 +399,11 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1-element Vector{Main.Herb.HerbCore.Constraint}:\n", - " Main.Herb.HerbConstraints.ComesAfter(1, [9])" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbGrammar.addconstraint!(g₈, Herb.HerbConstraints.ComesAfter(1, [9]))" + "HerbGrammar.addconstraint!(g₈, HerbConstraints.ComesAfter(1, [9]))" ] }, { @@ -724,52 +419,11 @@ }, { "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.07692307692307693 : 1\n", - "0.07692307692307693 : 2\n", - "0.07692307692307693 : 3\n", - "0.07692307692307693 : 4\n", - "0.07692307692307693 : 5\n", - "0.07692307692307693 : 6\n", - "0.07692307692307693 : 7\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n", - "┌ Warning: Requesting probability in a non-probabilistic grammar.\n", - "│ Uniform distribution is assumed.\n", - "└ @ Main.Herb.HerbGrammar d:\\GitHub\\Herb\\HerbGrammar.jl\\src\\grammar_base.jl:155\n" - ] - } - ], - "source": [ - "g₉ = Herb.HerbGrammar.@pcfgrammar begin\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₉ = HerbGrammar.@pcfgrammar begin\n", " 0.4 : Int = |(0:9)\n", " 0.2 : Int = Int + Int\n", " 0.1 : Int = Int * Int\n", @@ -777,7 +431,7 @@ "end\n", "\n", "for r ∈ 1:length(g₃.rules)\n", - " p = Herb.HerbGrammar.probability(g₈, r)\n", + " p = HerbGrammar.probability(g₈, r)\n", "\n", " println(\"$p : $r\")\n", "end" @@ -811,44 +465,22 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "Herb.HerbGrammar.store_cfg(\"demo.txt\", g₇)" + "HerbGrammar.store_cfg(\"demo.txt\", g₇)" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": { "scrolled": false }, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int * Int\n", - "12: Int = x\n", - "13: Int = Int - Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Herb.HerbGrammar.read_cfg(\"demo.txt\")" + "outputs": [], + "source": [ + "HerbGrammar.read_cfg(\"demo.txt\")" ] }, { @@ -865,82 +497,36 @@ }, { "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int + Int\n", - "12: Int = Int * Int\n", - "13: Int = x\n", - ", Main.Herb.HerbCore.Constraint[Main.Herb.HerbConstraints.ComesAfter(1, [9])])" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Herb.HerbGrammar.store_csg(\"demo.grammar\", \"demo.constraints\", g₈)\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HerbGrammar.store_csg(\"demo.grammar\", \"demo.constraints\", g₈)\n", "g₈, g₈.constraints" ] }, { "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1: Int = 0\n", - "2: Int = 1\n", - "3: Int = 2\n", - "4: Int = 3\n", - "5: Int = 4\n", - "6: Int = 5\n", - "7: Int = 6\n", - "8: Int = 7\n", - "9: Int = 8\n", - "10: Int = 9\n", - "11: Int = Int + Int\n", - "12: Int = Int * Int\n", - "13: Int = x\n", - ", Main.Herb.HerbCore.Constraint[Main.Herb.HerbConstraints.ComesAfter(1, [9])])" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g₉ = Herb.HerbGrammar.read_csg(\"demo.grammar\", \"demo.constraints\")\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "g₉ = HerbGrammar.read_csg(\"demo.grammar\", \"demo.constraints\")\n", "g₉, g₉.constraints" ] } ], "metadata": { "kernelspec": { - "display_name": "Julia 1.8.5", + "display_name": "Julia 1.9.4", "language": "julia", - "name": "julia-1.8" + "name": "julia-1.9" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.8.5" + "version": "1.9.4" } }, "nbformat": 4, diff --git a/docs/src/tutorials/defining_grammars.md b/docs/src/tutorials/defining_grammars.md index c8e2548..6223b12 100644 --- a/docs/src/tutorials/defining_grammars.md +++ b/docs/src/tutorials/defining_grammars.md @@ -1,10 +1,12 @@ -# Defining Grammars in Herb.jl using `HerbGrammar` +# Defining Grammars in Herb.jl using HerbGrammar The program space in Herb.jl is defined using a grammar. This notebook demonstrates how such a grammar can be created. There are multiple kinds of grammars, but they can all be defined in a very similar way. ### Setup +First, we import the necessary Herb packages. + ```julia using HerbGrammar, HerbConstraints @@ -435,8 +437,8 @@ HerbGrammar.addconstraint!(g₈, HerbConstraints.ComesAfter(1, [9])) ``` - 1-element Vector{Main.HerbCore.Constraint}: - Main.HerbConstraints.ComesAfter(1, [9]) + 1-element Vector{HerbCore.Constraint}: + ComesAfter(1, [9]) ### Probabilistic grammars @@ -472,25 +474,25 @@ end ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 ┌ Warning: Requesting probability in a non-probabilistic grammar. │ Uniform distribution is assumed. - └ @ Main.HerbGrammar d:\GitHub\HerbGrammar.jl\src\grammar_base.jl:155 + └ @ HerbGrammar /Users/issahanou/.julia/packages/HerbGrammar/x0E9w/src/grammar_base.jl:155 The numbers before each rule represent the probability assigned to that rule. @@ -561,7 +563,7 @@ g₈, g₈.constraints 11: Int = Int + Int 12: Int = Int * Int 13: Int = x - , Main.HerbCore.Constraint[Main.HerbConstraints.ComesAfter(1, [9])]) + , HerbCore.Constraint[ComesAfter(1, [9])]) @@ -584,5 +586,5 @@ g₉, g₉.constraints 11: Int = Int + Int 12: Int = Int * Int 13: Int = x - , Main.HerbCore.Constraint[Main.HerbConstraints.ComesAfter(1, [9])]) + , HerbCore.Constraint[ComesAfter(1, [9])]) diff --git a/docs/src/tutorials/constraints.ipynb b/docs/src/tutorials/getting_started_with_constraints.ipynb similarity index 66% rename from docs/src/tutorials/constraints.ipynb rename to docs/src/tutorials/getting_started_with_constraints.ipynb index 624f169..2bb63b3 100644 --- a/docs/src/tutorials/constraints.ipynb +++ b/docs/src/tutorials/getting_started_with_constraints.ipynb @@ -27,23 +27,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Int = 1\n", - "2: Int = x\n", - "3: Int = -Int\n", - "4: Int = Int + Int\n", - "5: Int = Int * Int\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "using HerbCore, HerbGrammar, HerbConstraints, HerbSearch\n", "\n", @@ -70,30 +56,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "x\n", - "-1\n", - "-x\n", - "1 * 1\n", - "-(-1)\n", - "1x\n", - "-(-x)\n", - "x * x\n", - "x * 1\n", - "x + 1\n", - "x + x\n", - "1 + x\n", - "1 + 1\n" - ] - } - ], + "outputs": [], "source": [ "clearconstraints!(grammar)\n", "iter = BFSIterator(grammar, :Int, max_size=3)\n", @@ -120,26 +85,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "x\n", - "-1\n", - "-x\n", - "x * 1\n", - "x * x\n", - "x + x\n", - "x + 1\n", - "1 + 1\n", - "1 + x\n" - ] - } - ], + "outputs": [], "source": [ "one = 1\n", "x = 2\n", @@ -171,18 +119,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "154\n", - "106\n" - ] - } - ], + "outputs": [], "source": [ "#this constraint forbids A+A and A*A\n", "constraint = Forbidden(DomainRuleNode(BitVector((0, 0, 0, 1, 1)), [VarNode(:A), VarNode(:A)]))\n", @@ -212,25 +151,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x\n", - "-x\n", - "-(-x)\n", - "1x\n", - "x * x\n", - "x * 1\n", - "x + 1\n", - "x + x\n", - "1 + x\n" - ] - } - ], + "outputs": [], "source": [ "clearconstraints!(grammar)\n", "addconstraint!(grammar, Contains(2)) #rule 2 should be used in the program\n", @@ -252,18 +175,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x * x\n", - "-(x * x)\n" - ] - } - ], + "outputs": [], "source": [ "clearconstraints!(grammar)\n", "addconstraint!(grammar, ContainsSubtree(RuleNode(times, [RuleNode(x), RuleNode(x)]))) #x*x should be in the program tree\n", @@ -292,28 +206,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "x\n", - "-1\n", - "-x\n", - "1 * 1\n", - "-(-1)\n", - "1x\n", - "-(-x)\n", - "x * x\n", - "x + x\n", - "1 + x\n", - "1 + 1\n" - ] - } - ], + "outputs": [], "source": [ "clearconstraints!(grammar)\n", "\n", @@ -355,26 +250,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "x\n", - "-1\n", - "-x\n", - "1 * 1\n", - "-(-1)\n", - "1x\n", - "-(-x)\n", - "x * x\n", - "x + x\n" - ] - } - ], + "outputs": [], "source": [ "constraint = ForbiddenSequence([plus, one], ignore_if=[times])\n", "addconstraint!(grammar, constraint)\n", @@ -411,19 +289,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "HerbConstraints.on_new_node" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "\"\"\"\n", "Forbids the consecutive application of the specified rule.\n", @@ -480,19 +348,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "HerbConstraints.propagate!" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "\"\"\"\n", "Forbids the consecutive application of the specified rule at path `path`.\n", @@ -564,19 +422,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "HerbConstraints.shouldschedule" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "\n", "\"\"\"\n", @@ -597,308 +445,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "x\n", - "-1\n", - "-x\n", - "1 * 1\n", - "1x\n", - "x * x\n", - "x * 1\n", - "x + 1\n", - "x + x\n", - "1 + x\n", - "1 + 1\n", - "1 * -1\n", - "1 * -x\n", - "x * -x\n", - "x * -1\n", - "x + -1\n", - "x + -x\n", - "1 + -x\n", - "-(1 * 1)\n", - "1 + -1\n", - "-(1x)\n", - "-(x * x)\n", - "-(x * 1)\n", - "-((x + 1))\n", - "-((x + x))\n", - "-1 * 1\n", - "-((1 + x))\n", - "-1 * x\n", - "-((1 + 1))\n", - "-x * x\n", - "-x * 1\n", - "-x + 1\n", - "-x + x\n", - "-1 + x\n", - "-1 + 1\n", - "(1 + 1) * 1\n", - "-(1 * -1)\n", - "(1 + 1) * x\n", - "-(1 * -x)\n", - "(1 + x) * x\n", - "-(x * -x)\n", - "(1 + x) * 1\n", - "-(x * -1)\n", - "(x + x) * 1\n", - "-((x + -1))\n", - "(x + x) * x\n", - "-((x + -x))\n", - "(x + 1) * x\n", - "-((1 + -x))\n", - "(x + 1) * 1\n", - "-((1 + -1))\n", - "x * 1 + 1\n", - "x * 1 + x\n", - "x * x + x\n", - "x * x + 1\n", - "1x + 1\n", - "1x + x\n", - "-(-1 * 1)\n", - "1 * 1 + x\n", - "-(-1 * x)\n", - "1 * 1 + 1\n", - "-(-x * x)\n", - "-(-x * 1)\n", - "-((-x + 1))\n", - "1 * (1 + 1)\n", - "-((-x + x))\n", - "1 * (1 + x)\n", - "-((-1 + x))\n", - "1 * (x + x)\n", - "-((-1 + 1))\n", - "1 * (x + 1)\n", - "x * (x + 1)\n", - "x * (x + x)\n", - "x * (1 + x)\n", - "x * (1 + 1)\n", - "-1 * -1\n", - "x + 1 * 1\n", - "-1 * -x\n", - "x + 1x\n", - "-x * -x\n", - "x + x * x\n", - "-x * -1\n", - "x + x * 1\n", - "-x + -1\n", - "1 + x * 1\n", - "-x + -x\n", - "1 + x * x\n", - "-1 + -x\n", - "1 + 1x\n", - "-1 + -1\n", - "1 + 1 * 1\n", - "-1 * (1 + 1)\n", - "1 * (1 + -1)\n", - "-1 * (1 + x)\n", - "1 * (1 + -x)\n", - "-1 * (x + x)\n", - "1 * (x + -x)\n", - "-1 * (x + 1)\n", - "1 * (x + -1)\n", - "-x * (x + 1)\n", - "x * (x + -1)\n", - "-x * (x + x)\n", - "x * (x + -x)\n", - "-x * (1 + x)\n", - "x * (1 + -x)\n", - "-x * (1 + 1)\n", - "x * (1 + -1)\n", - "-x + 1 * 1\n", - "x + 1 * -1\n", - "-x + 1x\n", - "x + 1 * -x\n", - "-x + x * x\n", - "x + x * -x\n", - "-x + x * 1\n", - "x + x * -1\n", - "-1 + x * 1\n", - "1 + x * -1\n", - "-1 + x * x\n", - "1 + x * -x\n", - "-1 + 1x\n", - "1 + 1 * -x\n", - "-1 + 1 * 1\n", - "1 + 1 * -1\n", - "1 * -(1 * 1)\n", - "(1 + -1) * 1\n", - "1 * -(1x)\n", - "(1 + -1) * x\n", - "1 * -(x * x)\n", - "(1 + -x) * x\n", - "1 * -(x * 1)\n", - "(1 + -x) * 1\n", - "1 * -((x + 1))\n", - "(x + -x) * 1\n", - "1 * -((x + x))\n", - "(x + -x) * x\n", - "1 * -((1 + x))\n", - "(x + -1) * x\n", - "1 * -((1 + 1))\n", - "(x + -1) * 1\n", - "x * -((1 + 1))\n", - "x * -1 + 1\n", - "x * -((1 + x))\n", - "x * -1 + x\n", - "x * -((x + x))\n", - "x * -x + x\n", - "x * -((x + 1))\n", - "x * -x + 1\n", - "x * -(x * 1)\n", - "1 * -x + 1\n", - "x * -(x * x)\n", - "1 * -x + x\n", - "x * -(1x)\n", - "1 * -1 + x\n", - "x * -(1 * 1)\n", - "1 * -1 + 1\n", - "x + -(1 * 1)\n", - "x + -(1x)\n", - "1 * (-1 + 1)\n", - "x + -(x * x)\n", - "1 * (-1 + x)\n", - "x + -(x * 1)\n", - "1 * (-x + x)\n", - "x + -((x + 1))\n", - "1 * (-x + 1)\n", - "x + -((x + x))\n", - "x * (-x + 1)\n", - "x + -((1 + x))\n", - "x * (-x + x)\n", - "x + -((1 + 1))\n", - "x * (-1 + x)\n", - "1 + -((1 + 1))\n", - "x * (-1 + 1)\n", - "1 + -((1 + x))\n", - "x + -1 * 1\n", - "1 + -((x + x))\n", - "x + -1 * x\n", - "1 + -((x + 1))\n", - "x + -x * x\n", - "1 + -(x * 1)\n", - "x + -x * 1\n", - "1 + -(x * x)\n", - "1 + -x * 1\n", - "1 + -(1x)\n", - "1 + -x * x\n", - "1 + -(1 * 1)\n", - "1 + -1 * x\n", - "1 + -1 * 1\n", - "-(1 * 1) * 1\n", - "-(1 * 1) * x\n", - "-((1 + 1) * 1)\n", - "-(1x) * x\n", - "-((1 + 1) * x)\n", - "-(1x) * 1\n", - "-((1 + x) * x)\n", - "-(x * x) * 1\n", - "-((1 + x) * 1)\n", - "-(x * x) * x\n", - "-((x + x) * 1)\n", - "-(x * 1) * x\n", - "-((x + x) * x)\n", - "-(x * 1) * 1\n", - "-((x + 1) * x)\n", - "-((x + 1)) * 1\n", - "-((x + 1) * 1)\n", - "-((x + 1)) * x\n", - "-((x * 1 + 1))\n", - "-((x + x)) * x\n", - "-((x * 1 + x))\n", - "-((x + x)) * 1\n", - "-((x * x + x))\n", - "-((1 + x)) * 1\n", - "-((x * x + 1))\n", - "-((1 + x)) * x\n", - "-((1x + 1))\n", - "-((1 + 1)) * x\n", - "-((1x + x))\n", - "-((1 + 1)) * 1\n", - "-((1 * 1 + x))\n", - "-((1 + 1)) + 1\n", - "-((1 * 1 + 1))\n", - "-((1 + 1)) + x\n", - "-((1 + x)) + x\n", - "-(-1 * -1)\n", - "-((1 + x)) + 1\n", - "-(-1 * -x)\n", - "-((x + x)) + 1\n", - "-(-x * -x)\n", - "-((x + x)) + x\n", - "-(-x * -1)\n", - "-((x + 1)) + x\n", - "-((-x + -1))\n", - "-((x + 1)) + 1\n", - "-((-x + -x))\n", - "-(x * 1) + 1\n", - "-((-1 + -x))\n", - "-(x * 1) + x\n", - "-((-1 + -1))\n", - "-(x * x) + x\n", - "-(x * x) + 1\n", - "(-1 + 1) * 1\n", - "-(1x) + 1\n", - "(-1 + 1) * x\n", - "-(1x) + x\n", - "(-1 + x) * x\n", - "-(1 * 1) + x\n", - "(-1 + x) * 1\n", - "-(1 * 1) + 1\n", - "(-x + x) * 1\n", - "(-x + x) * x\n", - "(1 + 1) * -1\n", - "(-x + 1) * x\n", - "(1 + 1) * -x\n", - "(-x + 1) * 1\n", - "(1 + x) * -x\n", - "-x * 1 + 1\n", - "(1 + x) * -1\n", - "-x * 1 + x\n", - "(x + x) * -1\n", - "-x * x + x\n", - "(x + x) * -x\n", - "-x * x + 1\n", - "(x + 1) * -x\n", - "-1 * x + 1\n", - "(x + 1) * -1\n", - "-1 * x + x\n", - "x * 1 + -1\n", - "-1 * 1 + x\n", - "x * 1 + -x\n", - "-1 * 1 + 1\n", - "x * x + -x\n", - "x * x + -1\n", - "-(1 * (1 + 1))\n", - "1x + -1\n", - "-(1 * (1 + x))\n", - "1x + -x\n", - "-(1 * (x + x))\n", - "1 * 1 + -x\n", - "-(1 * (x + 1))\n", - "1 * 1 + -1\n", - "-(x * (x + 1))\n", - "-(x * (x + x))\n", - "-(x * (1 + x))\n", - "-(x * (1 + 1))\n", - "-((x + 1 * 1))\n", - "-((x + 1x))\n", - "-((x + x * x))\n", - "-((x + x * 1))\n", - "-((1 + x * 1))\n", - "-((1 + x * x))\n", - "-((1 + 1x))\n", - "-((1 + 1 * 1))\n" - ] - } - ], + "outputs": [], "source": [ "clearconstraints!(grammar)\n", "\n", @@ -916,15 +465,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Julia 1.8.5", + "display_name": "Julia 1.9.4", "language": "julia", - "name": "julia-1.8" + "name": "julia-1.9" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.8.5" + "version": "1.9.4" } }, "nbformat": 4, diff --git a/docs/src/tutorials/constraints.md b/docs/src/tutorials/getting_started_with_constraints.md similarity index 99% rename from docs/src/tutorials/constraints.md rename to docs/src/tutorials/getting_started_with_constraints.md index 08d9265..7c95e90 100644 --- a/docs/src/tutorials/constraints.md +++ b/docs/src/tutorials/getting_started_with_constraints.md @@ -32,7 +32,7 @@ end 3: Int = -Int 4: Int = Int + Int 5: Int = Int * Int - + ### Working with constraints @@ -66,7 +66,7 @@ end x + x 1 + x 1 + 1 - + Upon inspection, we can already see some redundant programs, like `1 * 1` and `-(-1)`. To eliminate these redundant programs, we will set up some constraints that prevent these patterns from appearing. Then we will create another iteratator to enumerate all programs that satisfy the defined grammar constraints. @@ -106,7 +106,7 @@ end x + 1 1 + 1 1 + x - + ### Forbidden Constraint @@ -135,7 +135,7 @@ println(length(iter)) 154 106 - + ### Contains Constraint @@ -163,7 +163,7 @@ end x + 1 x + x 1 + x - + ### Contains Subtree Constraint @@ -182,7 +182,7 @@ end x * x -(x * x) - + ### Ordered Constraint @@ -224,7 +224,7 @@ end x + x 1 + x 1 + 1 - + ### Forbidden Sequence Constraint @@ -269,7 +269,7 @@ end -(-x) x * x x + x - + ### Custom Constraint @@ -735,4 +735,4 @@ end -((1 + x * x)) -((1 + 1x)) -((1 + 1 * 1)) - + diff --git a/docs/src/tutorials/getting_started_with_herb.ipynb b/docs/src/tutorials/getting_started_with_herb.ipynb index 4d3f2fc..45abcbb 100644 --- a/docs/src/tutorials/getting_started_with_herb.ipynb +++ b/docs/src/tutorials/getting_started_with_herb.ipynb @@ -22,12 +22,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "include(\"../../src/Herb.jl\") # these are only until we get modules going\n", - "using ..Herb" + "using HerbGrammar, HerbSpecification, HerbSearch, HerbInterpret, HerbConstraints" ] }, { @@ -37,9 +36,9 @@ "source": [ "### Defining the program space\n", "\n", - "Next, we start by creating a grammar. We define a context-free grammar which is just a simple set of production rules for defining combinations of terminal symbols (in our case real numbers). \n", + "Next, we start by creating a grammar. We define a context-free grammar (cfg) as a [`HerbGrammar.ContextSpecificGrammar`](@ref) without any constraints. A cfg is just a simple set of production rules for defining combinations of terminal symbols (in our case real numbers). \n", "\n", - "Contrary, we could define a context-sensitive grammar, when the production rules only hold in a certain context. However, for more information on this, please see `example2_defining_grammars.ipynb`.\n", + "Contrary, we could define a context-sensitive grammar, when the production rules only hold in a certain context. However, for more information on this, please see our tutorial on [defining grammars](defining_grammars.md).\n", "\n", "For now, we specify a simple grammar for dealing with integers and explain all the rules individually:\n", "\n", @@ -54,34 +53,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1: Real = 0\n", - "2: Real = 1\n", - "3: Real = 2\n", - "4: Real = 3\n", - "5: Real = 4\n", - "6: Real = 5\n", - "7: Real = 6\n", - "8: Real = 7\n", - "9: Real = 8\n", - "10: Real = 9\n", - "11: Real = x\n", - "12: Real = Real + Real\n", - "13: Real = Real - Real\n", - "14: Real = Real * Real\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "g = Herb.HerbGrammar.@cfgrammar begin\n", + "g = HerbGrammar.@cfgrammar begin\n", " Real = |(0:9)\n", " Real = x\n", " Real = Real + Real\n", @@ -116,27 +92,12 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "5-element Vector{Main.Herb.HerbSpecification.IOExample}:\n", - " Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 1), 8)\n", - " Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 2), 11)\n", - " Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 3), 14)\n", - " Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 4), 17)\n", - " Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 5), 20)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Create input-output examples\n", - "examples = [Herb.HerbSpecification.IOExample(Dict(:x => x), 3x+5) for x ∈ 1:5]" + "examples = [HerbSpecification.IOExample(Dict(:x => x), 3x+5) for x ∈ 1:5]" ] }, { @@ -151,21 +112,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Main.Herb.HerbSpecification.Problem(Main.Herb.HerbSpecification.Example[Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 1), 8), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 2), 11), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 3), 14), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 4), 17), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 5), 20)], \"example\")" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "problem = Herb.HerbSpecification.Problem(examples, \"example\")" + "problem = HerbSpecification.Problem(\"example\", examples)" ] }, { @@ -181,46 +132,42 @@ "Let us consider the case where we also have a ternary if-then-else operator and standard boolean operators in our grammar: we could synthesize the program `x ≤ 5 ? 3x+5 : 0`. \n", "This program satisfies all our examples, but we don't expect it to generalize very well.\n", "\n", + "To search through a program space, we first need to define a [`HerbSearch.ProgramIterator`](@ref), which can be instantiated with different iterators, for now we use a simple [`HerbSearch.BFSIterator`](@ref). For more advanced search methods check out our tutorial on [advanced search](.advanced_search.md). For more information about iterators, check out our tutorial on [working with interpreters](.working_with_interpreters.md). \n", + "\n", "In general, we assume that a smaller program is more general than a larger program. \n", "Therefore, we search for the smallest program in our grammar that satisfies our examples. \n", "This can be done using a breadth-first search over the program/search space.\n", "\n", "This search is very basic; it makes use of an enumeration technique, where we enumerate programs one-by-one until we find a program that matches our examples. The search procedure has a built-in default evaluator to verify the candidate programs with the given input. The search procedure also has a built-in search procedure using breadth-first search. \n", "\n", - "So, we only need to give our grammar and the problem to our search procedure, along with a starting `Symbol`, in our case a `Real`." + "So, we only need to give our grammar and the problem to our search procedure, along with a starting `Symbol`, in our case a `Real`. " ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - ":(x * 3 + 5)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "Herb.HerbSearch.search(g, problem, :Real)" + "iterator = BFSIterator(g, :Real)" ] }, { - "attachments": {}, - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "As you can see, the search procedure found the correct program!" + "synth(problem, iterator)" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, - "source": [] + "source": [ + "As you can see, the search procedure found the correct program!" + ] }, { "attachments": {}, @@ -231,27 +178,20 @@ "\n", "In the previous case, we used the built-ins of the search procedure. However, we can also give a custom enumerator to the search procedure and define a few more values.\n", "\n", - "We first define a new problem to test with, we are looking for the programs that can compute the value`167`. We immediately pass the examples to the problem and then set up the new search.\n", + "We first define a new problem to test with, we are looking for the programs that can compute the value `167`. We immediately pass the examples to the problem and then set up the new search.\n", "\n", "Search is done by passing the grammar, the problem and the starting point like before. We now also specify the enumeration function to be used, and now we use depth-first search. Then, we give the maximum depth of the programs we want to search for `(3)`, the maximum number of nodes in the Abstract Syntax Tree that exists during search `(10)`, and the maximum time in seconds allowed for the search." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "nothing" - ] - } - ], + "outputs": [], "source": [ - "problem = Herb.HerbSpecification.Problem([Herb.HerbSpecification.IOExample(Dict(:x => x), 168) for x ∈ 1:5], \"example2\")\n", - "expr = Herb.HerbSearch.search(g, problem, :Real, enumerator=Herb.HerbSearch.get_dfs_enumerator, max_depth=4, max_size=30, max_time=180)\n", + "problem = HerbSpecification.Problem(\"example2\", [HerbSpecification.IOExample(Dict(:x => x), 168) for x ∈ 1:5])\n", + "iterator = HerbSearch.BFSIterator(g, :Real, max_depth=4, max_size=30, max_time=180)\n", + "expr = HerbSearch.synth(problem, iterator)\n", "print(expr)" ] }, @@ -260,7 +200,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We see that our synthesizer can find a program to construct the value `168`, though a fun experiment would be trying to get the value `167`, what do you think would happen? If you want you can try below.\n", + "We see that our synthesizer can find a program to construct the value `168`, though a fun experiment would be trying to get the value `167`, what do you think would happen? You can try below, using the same iterator.\n", "\n", "In any case, this concludes our first introduction to the `Herb.jl` program synthesis framework. You can see more examples in this repository, or explore yourself. Enjoy!" ] @@ -271,23 +211,23 @@ "metadata": {}, "outputs": [], "source": [ - "problem = Herb.HerbSpecification.Problem([Herb.HerbSpecification.IOExample(Dict(:x => x), 167) for x ∈ 1:5], \"example2\")\n", - "expr = Herb.HerbSearch.search(g, problem, :Real)\n", + "problem = HerbSpecification.Problem(\"example3\", [HerbSpecification.IOExample(Dict(:x => x), 167) for x ∈ 1:5])\n", + "expr = HerbSearch.synth(problem, iterator)\n", "print(expr)" ] } ], "metadata": { "kernelspec": { - "display_name": "Julia 1.8.5", + "display_name": "Julia 1.9.4", "language": "julia", - "name": "julia-1.8" + "name": "julia-1.9" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.8.5" + "version": "1.9.4" } }, "nbformat": 4, diff --git a/docs/src/tutorials/getting_started_with_herb.md b/docs/src/tutorials/getting_started_with_herb.md index 591549e..3467ee9 100644 --- a/docs/src/tutorials/getting_started_with_herb.md +++ b/docs/src/tutorials/getting_started_with_herb.md @@ -8,14 +8,14 @@ First, we start with the setup. We need to access to all the function in the Her ```julia -using Herb +using HerbGrammar, HerbSpecification, HerbSearch, HerbInterpret, HerbConstraints ``` ### Defining the program space -Next, we start by creating a grammar. We define a context-free grammar which is just a simple set of production rules for defining combinations of terminal symbols (in our case real numbers). +Next, we start by creating a grammar. We define a context-free grammar (cfg) as a [`HerbGrammar.ContextSpecificGrammar`](@ref) without any constraints. A cfg is just a simple set of production rules for defining combinations of terminal symbols (in our case real numbers). -Contrary, we could define a context-sensitive grammar, when the production rules only hold in a certain context. However, for more information on this, please see `example2_defining_grammars.ipynb`. +Contrary, we could define a context-sensitive grammar, when the production rules only hold in a certain context. However, for more information on this, please see our tutorial on [defining grammars](defining_grammars.md). For now, we specify a simple grammar for dealing with integers and explain all the rules individually: @@ -29,7 +29,7 @@ If you run this cell, you can see all the rules rolled out. ```julia -g = Herb.HerbGrammar.@cfgrammar begin +g = HerbGrammar.@cfgrammar begin Real = |(0:9) Real = x Real = Real + Real @@ -71,16 +71,16 @@ In the cell below we automatically generate some examples for `x` assigning valu ```julia # Create input-output examples -examples = [Herb.HerbSpecification.IOExample(Dict(:x => x), 3x+5) for x ∈ 1:5] +examples = [HerbSpecification.IOExample(Dict(:x => x), 3x+5) for x ∈ 1:5] ``` - 5-element Vector{Main.Herb.HerbSpecification.IOExample}: - Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 1), 8) - Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 2), 11) - Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 3), 14) - Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 4), 17) - Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 5), 20) + 5-element Vector{IOExample}: + IOExample(Dict{Symbol, Any}(:x => 1), 8) + IOExample(Dict{Symbol, Any}(:x => 2), 11) + IOExample(Dict{Symbol, Any}(:x => 3), 14) + IOExample(Dict{Symbol, Any}(:x => 4), 17) + IOExample(Dict{Symbol, Any}(:x => 5), 20) Now that we have some input-output examples, we can define the problem. @@ -89,11 +89,11 @@ For now, this is irrelevant, and you can give the program any name you like. ```julia -problem = Herb.HerbSpecification.Problem(examples, "example") +problem = HerbSpecification.Problem("example", examples) ``` - Main.Herb.HerbSpecification.Problem(Main.Herb.HerbSpecification.Example[Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 1), 8), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 2), 11), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 3), 14), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 4), 17), Main.Herb.HerbSpecification.IOExample(Dict{Symbol, Any}(:x => 5), 20)], "example") + Problem{Vector{IOExample}}("example", IOExample[IOExample(Dict{Symbol, Any}(:x => 1), 8), IOExample(Dict{Symbol, Any}(:x => 2), 11), IOExample(Dict{Symbol, Any}(:x => 3), 14), IOExample(Dict{Symbol, Any}(:x => 4), 17), IOExample(Dict{Symbol, Any}(:x => 5), 20)]) ### Searching @@ -104,51 +104,77 @@ Of course, our problem is underdefined as there might be multiple programs that Let us consider the case where we also have a ternary if-then-else operator and standard boolean operators in our grammar: we could synthesize the program `x ≤ 5 ? 3x+5 : 0`. This program satisfies all our examples, but we don't expect it to generalize very well. +To search through a program space, we first need to define a [`HerbSearch.ProgramIterator`](@ref), which can be instantiated with different iterators, for now we use a simple [`HerbSearch.BFSIterator`](@ref). For more advanced search methods check out our tutorial on [advanced search](.advanced_search.md). For more information about iterators, check out our tutorial on [working with interpreters](.working_with_interpreters.md). + In general, we assume that a smaller program is more general than a larger program. Therefore, we search for the smallest program in our grammar that satisfies our examples. This can be done using a breadth-first search over the program/search space. This search is very basic; it makes use of an enumeration technique, where we enumerate programs one-by-one until we find a program that matches our examples. The search procedure has a built-in default evaluator to verify the candidate programs with the given input. The search procedure also has a built-in search procedure using breadth-first search. -So, we only need to give our grammar and the problem to our search procedure, along with a starting `Symbol`, in our case a `Real`. +So, we only need to give our grammar and the problem to our search procedure, along with a starting `Symbol`, in our case a `Real`. ```julia -Herb.HerbSearch.search(g, problem, :Real) +iterator = BFSIterator(g, :Real) ``` - :(x * 3 + 5) + BFSIterator(1: Real = 0 + 2: Real = 1 + 3: Real = 2 + 4: Real = 3 + 5: Real = 4 + 6: Real = 5 + 7: Real = 6 + 8: Real = 7 + 9: Real = 8 + 10: Real = 9 + 11: Real = x + 12: Real = Real + Real + 13: Real = Real - Real + 14: Real = Real * Real + , :Real, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807) -As you can see, the search procedure found the correct program! + +```julia +synth(problem, iterator) +``` + (12{14{11,4}6}, optimal_program) + + +As you can see, the search procedure found the correct program! ### Defining the search procedure In the previous case, we used the built-ins of the search procedure. However, we can also give a custom enumerator to the search procedure and define a few more values. -We first define a new problem to test with, we are looking for the programs that can compute the value`167`. We immediately pass the examples to the problem and then set up the new search. +We first define a new problem to test with, we are looking for the programs that can compute the value `167`. We immediately pass the examples to the problem and then set up the new search. Search is done by passing the grammar, the problem and the starting point like before. We now also specify the enumeration function to be used, and now we use depth-first search. Then, we give the maximum depth of the programs we want to search for `(3)`, the maximum number of nodes in the Abstract Syntax Tree that exists during search `(10)`, and the maximum time in seconds allowed for the search. ```julia -problem = Herb.HerbSpecification.Problem([Herb.HerbSpecification.IOExample(Dict(:x => x), 168) for x ∈ 1:5], "example2") -expr = Herb.HerbSearch.search(g, problem, :Real, enumerator=Herb.HerbSearch.get_dfs_enumerator, max_depth=4, max_size=30, max_time=180) +problem = HerbSpecification.Problem("example2", [HerbSpecification.IOExample(Dict(:x => x), 168) for x ∈ 1:5]) +iterator = HerbSearch.BFSIterator(g, :Real, max_depth=4, max_size=30, max_time=180) +expr = HerbSearch.synth(problem, iterator) print(expr) ``` - nothing + (14{7,14{5,8}}, optimal_program) -We see that our synthesizer can find a program to construct the value `168`, though a fun experiment would be trying to get the value `167`, what do you think would happen? If you want you can try below. +We see that our synthesizer can find a program to construct the value `168`, though a fun experiment would be trying to get the value `167`, what do you think would happen? You can try below, using the same iterator. In any case, this concludes our first introduction to the `Herb.jl` program synthesis framework. You can see more examples in this repository, or explore yourself. Enjoy! ```julia -problem = Herb.HerbSpecification.Problem([Herb.HerbSpecification.IOExample(Dict(:x => x), 167) for x ∈ 1:5], "example2") -expr = Herb.HerbSearch.search(g, problem, :Real) -println(expr) +problem = HerbSpecification.Problem("example3", [HerbSpecification.IOExample(Dict(:x => x), 167) for x ∈ 1:5]) +expr = HerbSearch.synth(problem, iterator) +print(expr) ``` + + (12{14{7,14{10,4}}6}, optimal_program) diff --git a/docs/src/tutorials/working_with_interpreters.html b/docs/src/tutorials/working_with_interpreters.html new file mode 100644 index 0000000..e4a9971 --- /dev/null +++ b/docs/src/tutorials/working_with_interpreters.html @@ -0,0 +1,17 @@ + + + + + + + +
\ No newline at end of file diff --git a/docs/src/tutorials/working_with_interpreters.ipynb b/docs/src/tutorials/working_with_interpreters.ipynb new file mode 100644 index 0000000..ce05015 --- /dev/null +++ b/docs/src/tutorials/working_with_interpreters.ipynb @@ -0,0 +1,271 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using the Julia interpreter\n", + "\n", + "To know how good a candidate program is, program synthesisers execute them. The easiest way to execute a program is to rely on Julia itself. To leverage the Julia interpreter, you only have to ensure that your programs are valid Julia expressions. \n", + "\n", + "First, let's import the necessary packages" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "using HerbGrammar, HerbInterpret" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Now, assume the following grammar." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1: Number = 1\n", + "2: Number = 2\n", + "3: Number = x\n", + "4: Number = Number + Number\n", + "5: Number = Number * Number\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "g = @cfgrammar begin\n", + " Number = |(1:2)\n", + " Number = x\n", + " Number = Number + Number\n", + " Number = Number * Number\n", + " end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's construct a program `x+3`, which would correspond to the following `RuleNode` representation" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4{3,1}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "myprog = RuleNode(4,[RuleNode(3),RuleNode(1)])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To run this program, we have to convert it into a Julia expression, which we can do in the following way:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ":(x + 1)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "myprog_julia = rulenode2expr(myprog, g)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we have a valid Julia expression, but we are still missing one key ingredient: we have to inform the interpreter about the special symbols. In our case, these are `:x` and `:+`. To do so, we need to create a symbol table, which is nothing more than a dictionary mapping symbols to their values:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Dict{Symbol, Any} with 2 entries:\n", + " :+ => +\n", + " :x => 2" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "symboltable = Dict{Symbol,Any}(:x => 2, :+ => +)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can execute our program through the defaul interpreter available in `HerbInterpret`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "interpret(symboltable, myprog_julia)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And that's it!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Defining a custom interpreter\n", + "\n", + "A disadvantage of the default Julia interpreter is that it needs to traverse abstract syntax tree twice -- once to convert it into a Julia expression, and the second time to execute that expression. Program execution is regularly the most consuming part of the entire pipeline and, by eliminating one of these steps, we can cut the runtime in half.\n", + "\n", + "We can define an interpreter that works directly over `RuleNode`s. \n", + "Consider the scenario in which we want to write programs for robot navigation: imagine a 2D world in which the robot can move around and pick up a ball. The programs we could write direct the robot to go up, down, left, and right. For convenience, the programming language also offers conditionals and loops:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "grammar_robots = @csgrammar begin\n", + " Start = Sequence #1\n", + "\n", + " Sequence = Operation #2\n", + " Sequence = (Operation; Sequence) #3\n", + " Operation = Transformation #4\n", + " Operation = ControlStatement #5\n", + "\n", + " Transformation = moveRight() | moveDown() | moveLeft() | moveUp() | drop() | grab() #6\n", + " ControlStatement = IF(Condition, Sequence, Sequence) #12\n", + " ControlStatement = WHILE(Condition, Sequence) #13\n", + "\n", + " Condition = atTop() | atBottom() | atLeft() | atRight() | notAtTop() | notAtBottom() | notAtLeft() | notAtRight() #14\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This grammar specifies a simple sequential program with instructions for the robot. A couple of example programs:\n", + " - `moveRight(); moveLeft(); drop()`\n", + " - WHILE(notAtTop(), moveUp())\n", + "\n", + "The idea behind this programming language is that the program specifies a set of transformations over a state of the robot world. Thus, a program can only be executed over a particular state. In this case, the state represents the size of the 2D world, the current position of a robot, the current position of a ball, and whether the robot is currently holding a ball. The execution of a particular instruction acts as a state transformation: each instruction takes a state as an input, transforms it, and passes it to the subsequent instruction. For example, execution of the program `moveRight(); moveLeft(); drop()` would proceed as:\n", + " 1. take an input state, \n", + " 2. pass it to the `moveRight()` instruction,\n", + " 3. pass the output of `moveRight()` to `moveLeft()` instructions,\n", + " 4. pass the output of `moveLeft()` to `drop()`,\n", + " 5. return the output of `drop()`.\n", + "\n", + "\n", + " The following is only one possible way to implement a custom interpreter, but it demonstrates a general template that can always be followed.\n", + "\n", + " We want to implement the following function, which would take in a program in the form of a `RuleNode`, a grammar, and a starting state, and return the state obtained after executing the program:\n", + "\n", + " interpret(prog::AbstractRuleNode, grammar::ContextSensitiveGrammar, state::RobotState)::RobotState\n", + "\n", + " As `RuleNode`s only store indices of derivation rules from the grammar, not the functions themselves, we will first pull the function call associated with every derivation rule. In Julia, this is indicated by the top-level symbol of the rules. For example, the top-level symbol for the derivation rule 6 is `:moveRight`; for rule 12, that is `:IF`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The remaining functions follow a similar idea. (You can see the full implementation of this interpreter [here](https://github.com/Herb-AI/HerbBenchmarks.jl/blob/new-robots/src/data/Robots_2020/robots_primitives.jl))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PLUTO_PROJECT_TOML_CONTENTS = \"\"\"\n", + "[deps]\n", + "HerbGrammar = \"4ef9e186-2fe5-4b24-8de7-9f7291f24af7\"\n", + "HerbInterpret = \"5bbddadd-02c5-4713-84b8-97364418cca7\"\n", + "\n", + "[compat]\n", + "HerbGrammar = \"~0.3.0\"\n", + "HerbInterpret = \"~0.1.3\"\n", + "\"\"\"" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.4", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/src/tutorials/working_with_interpreters.jl b/docs/src/tutorials/working_with_interpreters.jl new file mode 100644 index 0000000..66767a4 --- /dev/null +++ b/docs/src/tutorials/working_with_interpreters.jl @@ -0,0 +1,767 @@ +### A Pluto.jl notebook ### +# v0.19.43 + +using Markdown +using InteractiveUtils + +# ╔═╡ e0a7076c-9345-40ef-a26e-99e8bad31463 +begin + using HerbGrammar + using HerbInterpret +end + +# ╔═╡ 55719688-3940-11ef-1f29-f51dea064ff3 +md"# Using the Julia interpreter + +To know how good a candidate program is, program synthesisers execute them. The easiest way to execute a program is to rely on Julia itself. To leverage the Julia interpreter, you only have to ensure that your programs are valid Julia expressions. + +For example, assume the following grammar. +" + +# ╔═╡ 39eaa982-ba88-49b9-ad52-076a169d0439 +g = @cfgrammar begin + Number = |(1:2) + Number = x + Number = Number + Number + Number = Number * Number + end + +# ╔═╡ 2478d5a4-a11e-42aa-87dd-d97a3fa5d378 +md" +Let's construct a program `x+3`, which would correspond to the following `RuleNode` representation +" + +# ╔═╡ 1e15898e-568c-4211-ba00-27de61806aeb +myprog = RuleNode(4,[RuleNode(3),RuleNode(1)]) + +# ╔═╡ d43a2094-b215-4d6c-b6d8-8d32fe8898d6 +md" +To run this program, we have to convert it into a Julia expression, which we can do in the following way: +" + +# ╔═╡ a77621ce-1749-4e16-b3dc-f5312cc5ee73 +myprog_julia = rulenode2expr(myprog, g) + +# ╔═╡ 48997ff5-a492-4bfd-8c64-d935433228c0 +md" +Now we have a valid Julia expression, but we are still missing one key ingredient: we have to inform the interpreter about the special symbols. In our case, these are `:x` and `:+`. To do so, we need to create a symbol table, which is nothing more than a dictionary mapping symbols to their values: +" + +# ╔═╡ 3e4f7ed5-bfba-4aa0-9f43-2978a9082054 +symboltable = Dict{Symbol,Any}(:x => 2, :+ => +) + +# ╔═╡ 2a6f6456-2937-4520-8427-8d7595076ec5 +md" +Now we can execute our program through the defaul interpreter available in `HerbInterpret`: +" + +# ╔═╡ c0dcdbc8-2355-4f4d-85c2-ec37bfeab226 +interpret(symboltable, myprog_julia) + +# ╔═╡ b3ac6903-8513-40cc-91ee-8ae7beb08d1d +md"And that's it!" + +# ╔═╡ f765b471-74f8-4e17-8e1e-e556d88eb84b +md"# Defining a custom interpreter + +A disadvantage of the default Julia interpreter is that it needs to traverse abstract syntax tree twice -- once to convert it into a Julia expression, and the second time to execute that expression. Program execution is regularly the most consuming part of the entire pipeline and, by eliminating one of these steps, we can cut the runtime in half. + +We can define an interpreter that works directly over `RuleNode`s. +Consider the scenario in which we want to write programs for robot navigation: imagine a 2D world in which the robot can move around and pick up a ball. The programs we could write direct the robot to go up, down, left, and right. For convenience, the programming language also offers conditionals and loops: +" + +# ╔═╡ 1b251d0f-3a77-494f-a359-d8dc33ad5d44 +grammar_robots = @csgrammar begin + Start = Sequence #1 + + Sequence = Operation #2 + Sequence = (Operation; Sequence) #3 + Operation = Transformation #4 + Operation = ControlStatement #5 + + Transformation = moveRight() | moveDown() | moveLeft() | moveUp() | drop() | grab() #6 + ControlStatement = IF(Condition, Sequence, Sequence) #12 + ControlStatement = WHILE(Condition, Sequence) #13 + + Condition = atTop() | atBottom() | atLeft() | atRight() | notAtTop() | notAtBottom() | notAtLeft() | notAtRight() #14 +end + +# ╔═╡ aff77be9-365f-4672-bbd4-07f23528e32e + md" + This grammar specifies a simple sequential program with instructions for the robot. A couple of example programs: + - `moveRight(); moveLeft(); drop()` + - WHILE(notAtTop(), moveUp()) + +The idea behind this programming language is that the program specifies a set of transformations over a state of the robot world. Thus, a program can only be executed over a particular state. In this case, the state represents the size of the 2D world, the current position of a robot, the current position of a ball, and whether the robot is currently holding a ball. The execution of a particular instruction acts as a state transformation: each instruction takes a state as an input, transforms it, and passes it to the subsequent instruction. For example, execution of the program `moveRight(); moveLeft(); drop()` would proceed as: + 1. take an input state, + 2. pass it to the `moveRight()` instruction, + 3. pass the output of `moveRight()` to `moveLeft()` instructions, + 4. pass the output of `moveLeft()` to `drop()`, + 5. return the output of `drop()`. + + + The following is only one possible way to implement a custom interpreter, but it demonstrates a general template that can always be followed. + + We want to implement the following function, which would take in a program in the form of a `RuleNode`, a grammar, and a starting state, and return the state obtained after executing the program: + + interpret(prog::AbstractRuleNode, grammar::ContextSensitiveGrammar, state::RobotState)::RobotState + + As `RuleNode`s only store indices of derivation rules from the grammar, not the functions themselves, we will first pull the function call associated with every derivation rule. In Julia, this is indicated by the top-level symbol of the rules. For example, the top-level symbol for the derivation rule 6 is `:moveRight`; for rule 12, that is `:IF`. + " + +# ╔═╡ c99860ae-644d-4b72-b176-413095cd9b2c + + +# ╔═╡ 0be36600-9528-46b6-96e6-515e2c338f57 + + +# ╔═╡ 7f960487-8268-4b45-8bed-4e073fbbbb4e + + +# ╔═╡ 4a733a80-e98b-4f87-adf4-f72a16afd0c1 +md"" + +# ╔═╡ b55fa52d-75cc-4311-8387-1e6c2682282f + + +# ╔═╡ 204e7a43-ebc7-4caf-af46-7502be14a314 + + +# ╔═╡ 767cfc7d-de5a-4e86-9198-b99db4760634 + + +# ╔═╡ bd2b8c2d-c9a4-40b5-afcd-214daad9eccd + + +# ╔═╡ f7202e23-56e8-4611-9cde-633156e7e4da + + +# ╔═╡ 9db3b242-a14b-4932-80cd-75c54a21946b + + +# ╔═╡ ceb78d01-8e80-4356-95ad-bac889bc3fc1 + + +# ╔═╡ 1f700607-3cdf-43bf-91f2-72de3c9abc85 +md" +The remaining functions follow a similar idea. (You can see the full implementation of this interpreter [here](https://github.com/Herb-AI/HerbBenchmarks.jl/blob/new-robots/src/data/Robots_2020/robots_primitives.jl))." + +# ╔═╡ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7" +HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7" + +[compat] +HerbGrammar = "~0.3.0" +HerbInterpret = "~0.1.3" +""" + +# ╔═╡ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.1" +manifest_format = "2.0" +project_hash = "3d463631c2622b04eb5c06a35c60329ce68a7044" + +[[deps.AbstractTrees]] +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.5" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.ArnoldiMethod]] +deps = ["LinearAlgebra", "Random", "StaticArrays"] +git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" +uuid = "ec485272-7323-5ecc-a04f-4719b315124d" +version = "0.4.0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9e2a6b69137e6969bab0152632dcb3bc108c8bdd" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+1" + +[[deps.Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "a2f1c8c668c8e3cb4cca4e57a8efdb09067bb3fd" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.18.0+2" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "b1c55339b7c6c350ee89f2c1604299660525b248" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.15.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.2+0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.6.2+0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"] +git-tree-sha1 = "db16beca600632c95fc8aca29890d83788dd8b23" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.96+0" + +[[deps.FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "5c1d8ae0efc6c2e7b1fc502cbe25def8f661b7bc" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.13.2+0" + +[[deps.Gettext_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.21.0+0" + +[[deps.Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "7c82e6a6cd34e9d935e9aa4051b66c6ff3af59ba" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.80.2+0" + +[[deps.Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.14+0" + +[[deps.Graphs]] +deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "ebd18c326fa6cee1efb7da9a3b45cf69da2ed4d9" +uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" +version = "1.11.2" + +[[deps.HarfBuzz_ICU_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "HarfBuzz_jll", "ICU_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "6ccbc4fdf65c8197738c2d68cc55b74b19c97ac2" +uuid = "655565e8-fb53-5cb3-b0cd-aec1ca0647ea" +version = "2.8.1+0" + +[[deps.HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.8.1+1" + +[[deps.HerbCore]] +git-tree-sha1 = "923877c2715b8166d7ba9f9be2136d70eed87725" +uuid = "2b23ba43-8213-43cb-b5ea-38c12b45bd45" +version = "0.3.0" + +[[deps.HerbGrammar]] +deps = ["AbstractTrees", "DataStructures", "HerbCore", "Serialization", "TreeView"] +git-tree-sha1 = "b4cbf9712dbb3ab281ff4ed517d3cd6bc12f3078" +uuid = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7" +version = "0.3.0" + +[[deps.HerbInterpret]] +deps = ["HerbCore", "HerbGrammar", "HerbSpecification"] +git-tree-sha1 = "9e19b4ee5f29eb8bb9b1049524728b38e878eca2" +uuid = "5bbddadd-02c5-4713-84b8-97364418cca7" +version = "0.1.3" + +[[deps.HerbSpecification]] +git-tree-sha1 = "5385b81e40c3cd62aeea591319896148036863c9" +uuid = "6d54aada-062f-46d8-85cf-a1ceaf058a06" +version = "0.1.0" + +[[deps.ICU_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "20b6765a3016e1fca0c9c93c80d50061b94218b7" +uuid = "a51ab1cf-af8e-5615-a023-bc2c838bba6b" +version = "69.1.0+0" + +[[deps.Inflate]] +git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.5" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.5.0" + +[[deps.JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "c84a835e1a09b289ffcd2271bf2a337bbdda6637" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "3.0.3+0" + +[[deps.LERC_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "bf36f528eec6634efc60d7ec062008f171071434" +uuid = "88015f11-f218-50d7-93a8-a6af411a945d" +version = "3.0.0+1" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "d986ce2d884d49126836ea94ed5bfb0f12679713" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "15.0.7+0" + +[[deps.LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "70c5da094887fd2cae843b8db33920bac4b6f07d" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.2+0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.2+1" + +[[deps.Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll"] +git-tree-sha1 = "9fd170c4bbfd8b935fdc5f8b7aa33532c991a673" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.11+0" + +[[deps.Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "fbb1f2bef882392312feb1ede3615ddc1e9b99ed" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.49.0+0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+0" + +[[deps.Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0c4f9c4f1a50d8f35048fa0532dabbadf702f81e" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.40.1+0" + +[[deps.Libtiff_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "3eb79b0ca5764d4799c06699573fd8f533259713" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.4.0+0" + +[[deps.Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "5ee6203157c120d79034c748a2acba45b82b8807" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.40.1+0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LittleCMS_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pkg"] +git-tree-sha1 = "110897e7db2d6836be22c18bffd9422218ee6284" +uuid = "d3a379c0-f9a3-5b72-a4c0-6bf4d2e8af0f" +version = "2.12.0+0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.13" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenJpeg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libtiff_jll", "LittleCMS_jll", "Pkg", "libpng_jll"] +git-tree-sha1 = "76374b6e7f632c130e78100b166e5a48464256f8" +uuid = "643b3616-a352-519d-856d-80112ee9badc" +version = "2.4.0+0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "a12e56c72edee3ce6b96667745e6cbbe5498f200" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "1.1.23+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+0" + +[[deps.Pixman_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] +git-tree-sha1 = "35621f10a7531bc8fa58f74610b1bfb70a3cfc6b" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.43.4+0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" + +[[deps.Poppler_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "OpenJpeg_jll", "Pkg", "libpng_jll"] +git-tree-sha1 = "02148a0cb2532f22c0589ceb75c110e168fb3d1f" +uuid = "9c32591e-4766-534b-9725-b71a8799265b" +version = "21.9.0+0" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "eeafab08ae20c62c44c8399ccb9354a04b80db50" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.7" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TikzGraphs]] +deps = ["Graphs", "LaTeXStrings", "TikzPictures"] +git-tree-sha1 = "e8f41ed9a2cabf6699d9906c195bab1f773d4ca7" +uuid = "b4f28e30-c73f-5eaf-a395-8a9db949a742" +version = "1.4.0" + +[[deps.TikzPictures]] +deps = ["LaTeXStrings", "Poppler_jll", "Requires", "tectonic_jll"] +git-tree-sha1 = "79e2d29b216ef24a0f4f905532b900dcf529aa06" +uuid = "37f6aa50-8035-52d0-81c2-5a1d08754b2d" +version = "3.5.0" + +[[deps.TreeView]] +deps = ["CommonSubexpressions", "Graphs", "MacroTools", "TikzGraphs"] +git-tree-sha1 = "41ddcefb625f2ab0f4d9f2081c2da1af2ccbbf8b" +uuid = "39424ebd-4cf3-5550-a685-96706a953f40" +version = "0.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "d9717ce3518dc68a99e6b96300813760d887a01d" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.13.1+0" + +[[deps.XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "a54ee957f4c86b526460a720dbc882fa5edcbefc" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.41+0" + +[[deps.Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "afead5aba5aa507ad5a3bf01f58f82c8d1403495" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.8.6+0" + +[[deps.Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6035850dcc70518ca32f012e46015b9beeda49d8" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.11+0" + +[[deps.Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "34d526d318358a859d7de23da945578e8e8727b7" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.4+0" + +[[deps.Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "d2d1a5c49fae4ba39983f63de6afcbea47194e85" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.6+0" + +[[deps.Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "47e45cd78224c53109495b3e324df0c37bb61fbe" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.11+0" + +[[deps.Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8fdda4c692503d44d04a0603d9ac0982054635f9" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.1+0" + +[[deps.Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "bcd466676fef0878338c61e655629fa7bbc69d8e" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.17.0+0" + +[[deps.Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e92a1a012a10506618f10b7047e478403a046c77" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.5.0+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e678132f07ddb5bfa46857f0d7620fb9be675d3b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "d7015d2e18a5fd9a4f47de711837e980519781a4" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.43+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" + +[[deps.tectonic_jll]] +deps = ["Artifacts", "Fontconfig_jll", "FreeType2_jll", "Graphite2_jll", "HarfBuzz_ICU_jll", "HarfBuzz_jll", "ICU_jll", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "54867b00af20c70b52a1f9c00043864d8b926a21" +uuid = "d7dd28d6-a5e6-559c-9131-7eb760cdacc5" +version = "0.13.1+0" +""" + +# ╔═╡ Cell order: +# ╠═e0a7076c-9345-40ef-a26e-99e8bad31463 +# ╟─55719688-3940-11ef-1f29-f51dea064ff3 +# ╠═39eaa982-ba88-49b9-ad52-076a169d0439 +# ╟─2478d5a4-a11e-42aa-87dd-d97a3fa5d378 +# ╠═1e15898e-568c-4211-ba00-27de61806aeb +# ╟─d43a2094-b215-4d6c-b6d8-8d32fe8898d6 +# ╠═a77621ce-1749-4e16-b3dc-f5312cc5ee73 +# ╟─48997ff5-a492-4bfd-8c64-d935433228c0 +# ╠═3e4f7ed5-bfba-4aa0-9f43-2978a9082054 +# ╟─2a6f6456-2937-4520-8427-8d7595076ec5 +# ╠═c0dcdbc8-2355-4f4d-85c2-ec37bfeab226 +# ╟─b3ac6903-8513-40cc-91ee-8ae7beb08d1d +# ╟─f765b471-74f8-4e17-8e1e-e556d88eb84b +# ╠═1b251d0f-3a77-494f-a359-d8dc33ad5d44 +# ╟─aff77be9-365f-4672-bbd4-07f23528e32e +# ╠═c99860ae-644d-4b72-b176-413095cd9b2c +# ╠═0be36600-9528-46b6-96e6-515e2c338f57 +# ╠═7f960487-8268-4b45-8bed-4e073fbbbb4e +# ╠═4a733a80-e98b-4f87-adf4-f72a16afd0c1 +# ╠═b55fa52d-75cc-4311-8387-1e6c2682282f +# ╠═204e7a43-ebc7-4caf-af46-7502be14a314 +# ╠═767cfc7d-de5a-4e86-9198-b99db4760634 +# ╠═bd2b8c2d-c9a4-40b5-afcd-214daad9eccd +# ╠═f7202e23-56e8-4611-9cde-633156e7e4da +# ╠═9db3b242-a14b-4932-80cd-75c54a21946b +# ╠═ceb78d01-8e80-4356-95ad-bac889bc3fc1 +# ╠═1f700607-3cdf-43bf-91f2-72de3c9abc85 +# ╟─00000000-0000-0000-0000-000000000001 +# ╟─00000000-0000-0000-0000-000000000002 diff --git a/docs/src/tutorials/working_with_interpreters.md b/docs/src/tutorials/working_with_interpreters.md new file mode 100644 index 0000000..d1cc62e --- /dev/null +++ b/docs/src/tutorials/working_with_interpreters.md @@ -0,0 +1,141 @@ +# Using the Julia interpreter + +To know how good a candidate program is, program synthesisers execute them. The easiest way to execute a program is to rely on Julia itself. To leverage the Julia interpreter, you only have to ensure that your programs are valid Julia expressions. + +First, let's import the necessary packages + + +```julia +using HerbGrammar, HerbInterpret +``` + + +Now, assume the following grammar. + + +```julia +g = @cfgrammar begin + Number = |(1:2) + Number = x + Number = Number + Number + Number = Number * Number + end +``` + + + 1: Number = 1 + 2: Number = 2 + 3: Number = x + 4: Number = Number + Number + 5: Number = Number * Number + + + +Let's construct a program `x+3`, which would correspond to the following `RuleNode` representation + + +```julia +myprog = RuleNode(4,[RuleNode(3),RuleNode(1)]) +``` + + + 4{3,1} + + +To run this program, we have to convert it into a Julia expression, which we can do in the following way: + + +```julia +myprog_julia = rulenode2expr(myprog, g) +``` + + + :(x + 1) + + +Now we have a valid Julia expression, but we are still missing one key ingredient: we have to inform the interpreter about the special symbols. In our case, these are `:x` and `:+`. To do so, we need to create a symbol table, which is nothing more than a dictionary mapping symbols to their values: + + +```julia +symboltable = Dict{Symbol,Any}(:x => 2, :+ => +) +``` + + + Dict{Symbol, Any} with 2 entries: + :+ => + + :x => 2 + + +Now we can execute our program through the defaul interpreter available in `HerbInterpret`: + + + +```julia +interpret(symboltable, myprog_julia) +``` + + + 3 + + +And that's it! + +# Defining a custom interpreter + +A disadvantage of the default Julia interpreter is that it needs to traverse abstract syntax tree twice -- once to convert it into a Julia expression, and the second time to execute that expression. Program execution is regularly the most consuming part of the entire pipeline and, by eliminating one of these steps, we can cut the runtime in half. + +We can define an interpreter that works directly over `RuleNode`s. +Consider the scenario in which we want to write programs for robot navigation: imagine a 2D world in which the robot can move around and pick up a ball. The programs we could write direct the robot to go up, down, left, and right. For convenience, the programming language also offers conditionals and loops: + + +```julia +grammar_robots = @csgrammar begin + Start = Sequence #1 + + Sequence = Operation #2 + Sequence = (Operation; Sequence) #3 + Operation = Transformation #4 + Operation = ControlStatement #5 + + Transformation = moveRight() | moveDown() | moveLeft() | moveUp() | drop() | grab() #6 + ControlStatement = IF(Condition, Sequence, Sequence) #12 + ControlStatement = WHILE(Condition, Sequence) #13 + + Condition = atTop() | atBottom() | atLeft() | atRight() | notAtTop() | notAtBottom() | notAtLeft() | notAtRight() #14 +end +``` + +This grammar specifies a simple sequential program with instructions for the robot. A couple of example programs: + - `moveRight(); moveLeft(); drop()` + - WHILE(notAtTop(), moveUp()) + +The idea behind this programming language is that the program specifies a set of transformations over a state of the robot world. Thus, a program can only be executed over a particular state. In this case, the state represents the size of the 2D world, the current position of a robot, the current position of a ball, and whether the robot is currently holding a ball. The execution of a particular instruction acts as a state transformation: each instruction takes a state as an input, transforms it, and passes it to the subsequent instruction. For example, execution of the program `moveRight(); moveLeft(); drop()` would proceed as: + 1. take an input state, + 2. pass it to the `moveRight()` instruction, + 3. pass the output of `moveRight()` to `moveLeft()` instructions, + 4. pass the output of `moveLeft()` to `drop()`, + 5. return the output of `drop()`. + + + The following is only one possible way to implement a custom interpreter, but it demonstrates a general template that can always be followed. + + We want to implement the following function, which would take in a program in the form of a `RuleNode`, a grammar, and a starting state, and return the state obtained after executing the program: + + interpret(prog::AbstractRuleNode, grammar::ContextSensitiveGrammar, state::RobotState)::RobotState + + As `RuleNode`s only store indices of derivation rules from the grammar, not the functions themselves, we will first pull the function call associated with every derivation rule. In Julia, this is indicated by the top-level symbol of the rules. For example, the top-level symbol for the derivation rule 6 is `:moveRight`; for rule 12, that is `:IF`. + +The remaining functions follow a similar idea. (You can see the full implementation of this interpreter [here](https://github.com/Herb-AI/HerbBenchmarks.jl/blob/new-robots/src/data/Robots_2020/robots_primitives.jl)). + + +```julia +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7" +HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7" + +[compat] +HerbGrammar = "~0.3.0" +HerbInterpret = "~0.1.3" +""" +```