diff --git a/README.md b/README.md index 3bd39bd..3f84144 100644 --- a/README.md +++ b/README.md @@ -19,31 +19,42 @@ git clone --recurse-submodules https://github.com/binpash/dish.git ## Install docker using our script (tested on Ubuntu) ## Alternatively see https://docs.docker.com/engine/install/ to install docker. +cd dish ./scripts/setup-docker.sh cd docker-hadoop -./setup-compose.sh # Creates the virtual cluster on the host machine -## The cluster can be torn down using `docker-compose down` -docker exec -it nodemanager1 bash # We will use this node as a client -``` +## Create the virtual cluster on the host machine +./setup-compose.sh # currently takes several minutes due to rebuilding the images +## The cluster can be torn down using `docker compose down` -## Running DiSh +## Create a shell on the client +docker exec -it nodemanager1 bash +``` -__TODO__ +## Running DiSh Hello World -From docker, we need to add some files to hdfs: +Let's run a very simple example using DiSh: ```sh cd $DISH_TOP hdfs dfs -put README.md /README.md # Copies the readme to hdfs ``` -Now, you can create your own script or use `sample.sh`. Run both DiSh and Bash and compare the results! +Now, you can run [this sample script](./scripts/sample.sh) (or create a script of your own). Run both DiSh and Bash and compare the results! ``` ./di.sh sample.sh bash sample.sh ``` + + + + ## Repo Structure This repo hosts most of the components of the `dish` development. Some of them are incorporated in [PaSh](https://github.com/binpash/pash) The structure is as follows: diff --git a/evaluation/distr_benchmarks/oneliners/input/setup.sh b/evaluation/distr_benchmarks/oneliners/input/setup.sh index 81842f6..47b1134 100755 --- a/evaluation/distr_benchmarks/oneliners/input/setup.sh +++ b/evaluation/distr_benchmarks/oneliners/input/setup.sh @@ -15,14 +15,13 @@ if [[ "$1" == "-c" ]]; then exit fi -hdfs dfs -mkdir /oneliners +hdfs dfs -mkdir -p /oneliners if [ ! -f ./1M.txt ]; then curl -sf --connect-timeout 10 'http://ndr.md/data/dummy/1M.txt' > 1M.txt if [ $? -ne 0 ]; then curl -f 'https://zenodo.org/record/7650885/files/1M.txt' > 1M.txt [ $? -ne 0 ] && eexit 'cannot find 1M.txt' - fi fi append_nl_if_not ./1M.txt fi @@ -42,7 +41,7 @@ if [ ! -f ./100M.txt ]; then fi if [ ! -f ./1G.txt ]; then - curl -sf 'http://ndr.md/data/dummy/1G.txt' > 1G.txt + curl -sf --connect-timeout 10 'http://ndr.md/data/dummy/1G.txt' > 1G.txt if [ $? -ne 0 ]; then touch 1G.txt for (( i = 0; i < 10; i++ )); do diff --git a/scripts/sample.sh b/scripts/sample.sh new file mode 100644 index 0000000..78d3e0a --- /dev/null +++ b/scripts/sample.sh @@ -0,0 +1 @@ +hdfs dfs -cat /README.md | grep the | wc -l diff --git a/scripts/setup-docker.sh b/scripts/setup-docker.sh index ea285b4..1f3906c 100755 --- a/scripts/setup-docker.sh +++ b/scripts/setup-docker.sh @@ -9,7 +9,7 @@ # sudo usermod -aG docker "${USER}" sudo apt-get update -sudo apt-get install \ +sudo apt-get install -y \ ca-certificates \ curl \ gnupg @@ -21,7 +21,7 @@ echo \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update -sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo usermod -aG docker "${USER}"