-
Notifications
You must be signed in to change notification settings - Fork 3
/
docker-compose.yml
153 lines (142 loc) · 3.36 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
version: "3.3"
services:
namenode:
image: apachehudi/hudi-hadoop_2.8.4-namenode:latest
hostname: namenode
container_name: namenode
environment:
- CLUSTER_NAME=hudi_example
ports:
- "50070:50070"
- "8020:8020"
env_file:
- ./config/hadoop.env
volumes:
- ./data/wiki:/wiki
networks:
- spark-network
datanode:
image: apachehudi/hudi-hadoop_2.8.4-datanode:latest
container_name: datanode
hostname: datanode
environment:
- CLUSTER_NAME=hudi_example
env_file:
- ./config/hadoop.env
ports:
- "50075:50075"
- "50010:50010"
depends_on:
- namenode
networks:
- spark-network
historyserver:
image: apachehudi/hudi-hadoop_2.8.4-history:latest
hostname: historyserver
container_name: historyserver
environment:
- CLUSTER_NAME=hudi_example
depends_on:
- "namenode"
ports:
- "58188:8188"
env_file:
- ./config/hadoop.env
volumes:
- historyserver:/hadoop/yarn/timeline
networks:
- spark-network
hive-metastore-postgresql:
image: bde2020/hive-metastore-postgresql:2.3.0
volumes:
- hive-metastore-postgresql:/var/lib/postgresql
hostname: hive-metastore-postgresql
container_name: hive-metastore-postgresql
networks:
- spark-network
hivemetastore:
image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest
hostname: hivemetastore
container_name: hivemetastore
env_file:
- ./config/hadoop.env
command: /opt/hive/bin/hive --service metastore
environment:
SERVICE_PRECONDITION: "namenode:50070 hive-metastore-postgresql:5432"
ports:
- "9083:9083"
depends_on:
- "hive-metastore-postgresql"
- "namenode"
networks:
- spark-network
hiveserver:
image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest
hostname: hiveserver
container_name: hiveserver
env_file:
- ./config/hadoop.env
environment:
SERVICE_PRECONDITION: "hivemetastore:9083"
ports:
- "10000:10000"
depends_on:
- "hivemetastore"
volumes:
- ./shared:/var/hoodie/ws
networks:
- spark-network
sparkmaster:
image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:latest
hostname: sparkmaster
container_name: sparkmaster
env_file:
- ./config/hadoop.env
ports:
- "8080:8080"
- "7077:7077"
environment:
- INIT_DAEMON_STEP=setup_spark
networks:
- spark-network
spark-worker-1:
image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:latest
hostname: spark-worker-1
container_name: spark-worker-1
env_file:
- ./config/hadoop.env
depends_on:
- sparkmaster
ports:
- "8081:8081"
environment:
- "SPARK_MASTER=spark://sparkmaster:7077"
networks:
- spark-network
adhoc-1:
build:
context: ./docker/adhoc-1
hostname: adhoc-1
container_name: adhoc-1
env_file:
- ./config/hadoop.env
depends_on:
- sparkmaster
ports:
- "4040:4040"
environment:
- "SPARK_MASTER=spark://sparkmaster:7077"
volumes:
- ./shared:/var/hoodie/ws
networks:
- spark-network
volumes:
namenode:
datanode:
historyserver:
hive-metastore-postgresql:
networks:
spark-network:
driver: bridge
ipam:
driver: default