-
Notifications
You must be signed in to change notification settings - Fork 9
/
ceph
290 lines (246 loc) · 7.09 KB
/
ceph
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
dont use raid1 raid0 raid6 raid5 raid10 ...
one osd daemon = one disk
rbd
- rbd_info
- rbd_directory
- <image>.rbd
- id
- size of image, objects
- snapshots
- rbd.<id>.<n>
- images are sparse
rbd
snap
* freeze fs (xfs_freeze)
* rbd snap create
* unfreeze fs
http://storageconference.org/2012/Presentations/T02.Weil.pdf
cephfs
$ mount -t ceph 1.2.3.4:/ /mnt
$ cd /mnt
$ ls -alSh | head
total 0
drwxr-xr-x 1 root root 9.7T 2011-02-04 15:51 .
drwxr-xr-x 1 root root 9.7T 2010-12-16 15:06 ..
drwxr-xr-x 1 pomceph pg4194980 9.6T 2011-02-24 08:25 pomceph
drwxr-xr-x 1 mcg_test1 pg2419992 23G 2011-02-02 08:57 mcg_test1
drwx--x--- 1 luko adm 19G 2011-01-21 12:17 luko
drwx--x--- 1 eest adm 14G 2011-02-04 16:29 eest
drwxr-xr-x 1 mcg_test2 pg2419992 3.0G 2011-02-02 09:34 mcg_test2
drwx--x--- 1 fuzyceph adm 1.5G 2011-01-18 10:46 fuzyceph
drwxr-xr-x 1 dallasceph pg275 596M 2011-01-14 10:06 dallasceph
$ mkdir foo/.snap/one # create snapshot
$ ls foo/.snap
one
$ ls foo/bar/.snap
_one_1099511627776 # parent's snap name is mangled
$ rm foo/myfile
$ ls -F foo
bar/
$ ls -F foo/.snap/one
myfile bar/
$ rmdir foo/.snap/one # remove snapshot
INSTALL
echo deb http://ceph.newdream.net/debian precise main | sudo tee /etc/apt/sources.list.d/ceph.list
sudo apt-get install ceph
sudo apt-get install librbd1, librados2, libcephfs1
sudo apt-get install radosgw
sudo vi /etc/ceph/ceph.conf
> [global]
> auth supported = cephx
> [mon]
> mon data = /var/lib/ceph/ceph-mon.$id
> [mon.a]
> host = mymon-a
> mon addr = 1.2.3.4:6789
> [mon.b]
> host = mymon-b
> mon addr = 1.2.3.5:6789
> [mon.c]
> host = mymon-c
> mon addr = 1.2.3.4:6789
> [osd]
> osd data = /var/lib/ceph/ceph-osd.$id
> [osd.0]
> host = myosd0
# set up ssh keys
$ sudo mkcephfs -c conf -a -mkbtrfs
# distribute admin key
$ sudo service ceph start
$ sudo ceph health
$ sudo ceph -w
$ sudo ceph osd dump
$ sudo ceph osd tree
> dumped osdmap tree epoch 621
> # id weight type name up/down reweight
> -1 12 pool default
> -3 12 rack le-rack
> -2 3 host ceph-01
> 0 1 osd.0 up 1
> 1 1 osd.1 up 1
> -4 3 host ceph-02
> 2 1 osd.2 up 1
> 3 1 osd.3 up 1
== osd failure ==
$ sudo killall ceph-osd
$ sudo service stop osd.12
$ sudo ceph osd out 12
== add new osd ==
$ sudo ceph osd create
12
$ sudo vi ceph.conf
[osd.12]
host = plana12
btrfs devs = /dev/sdb
$ sudo mkfs.btrfs /dev/sdb
$ sudo mkdir -p /var/lib/ceph/osd-data/12
$ sudo mount /dev/sdb /var/lib/ceph/osd-data/12
$ sudo ceph-osd –mkfs -i 12 –mkkey
$ sudo ceph auth add osd.12 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd-data/12/keyring
$ sudo service ceph start osd.12
$ sudo ceph osd tree
$ sudo ceph osd crush add 12 osd.12 1.0 host=plana12 rack=unknownrack pool=default
$ sudo ceph osd tree
$ sudo ceph osd crush reweight osd.12 .7
== modifying crush map ==
$ sudo ceph osd getcrushmap -o cm
$ sudo crushtool -d cm -o cm.txt
$ sudo vi cm.txt
> # begin crush map
>
> # devices
> device 0 osd.0
> device 1 osd.1
> device 2 osd.2
> device 3 osd.3
>
> # types
> type 0 osd
> type 1 host
> type 2 rack
> type 3 row
> type 4 room
> type 5 datacenter
> type 6 pool
>
> # buckets
> host ceph-01 {
> id -2 # do not change unnecessarily
> # weight 3.000
> alg straw
> hash 0 # rjenkins1
> item osd.0 weight 1.000
> item osd.1 weight 1.000
> }
> host ceph-02 {
> id -4 # do not change unnecessarily
> # weight 3.000
> alg straw
> hash 0 # rjenkins1
> item osd.2 weight 1.000
> item osd.3 weight 1.000
> }
> rack le-rack {
> id -3 # do not change unnecessarily
> # weight 12.000
> alg straw
> hash 0 # rjenkins1
> item ceph-01 weight 2.000
> item ceph-02 weight 2.000
> }
> pool default {
> id -1 # do not change unnecessarily
> # weight 12.000
> alg straw
> hash 0 # rjenkins1
> item le-rack weight 4.000
> }
>
> # rules
> rule data {
> ruleset 0
> type replicated
> min_size 1
> max_size 10
> step take default
> step chooseleaf firstn 0 type host
> step emit
> }
> rule metadata {
> ruleset 1
> type replicated
> min_size 1
> max_size 10
> step take default
> step chooseleaf firstn 0 type host
> step emit
> }
> rule rbd {
> ruleset 2
> type replicated
> min_size 1
> max_size 10
> step take default
> step chooseleaf firstn 0 type host
> step emit
> }
>
> # end crush map
$ sudo crushtool -c cm.txt -o cm.new
$ sudo ceph osd setcrushmap -i cm.new
== adjust replication ==
$ sudo ceph osd dump | grep ^pool # learn
$ sudo ceph osd pool <poolname> set data size 3
== rbd ==
# create an rbd user
sudo ceph-authtool --create-keyring -n client.rbd –gen- key rbd.keyring
sudo ceph auth add client.rbd osd "allow *" mon "allow *" -i rbd.keyring
# import an image
$ sudo rbd import precise-server.img foo
# take an initial snapshot
$ sudo rbd snap create –snap=orig foo
# resize
$ sudo rbd resize –size 20000 foo
$ sudo rbd info foo
$ sudo rbd resize –size 10000 foo
# rollback snapshot
$ sudo rbd snap rollback –snap=orig foo
== juju ==
http://static.usenix.org/events/osdi06/tech/full_papers/weil/weil_html/index.html
sudo apt-add-repository ppa:juju/pkgs
sudo apt-get update && sudo apt-get install juju
juju bootstrap
vi ~/.juju/environments.yaml
juju bootstrap --constraints instance-type=t1.micro
juju status
juju deploy -n 3 --config ceph.yaml ceph
juju deploy -n 3 --config ceph.yaml ceph-osd
juju status
== scenarios ==
http://www.sebastien-han.fr/blog/2012/12/07/ceph-2-speed-storage-with-crush/
storage nodes full of SSDs disks
storage nodes full of SAS disks
storage nodes full of SATA disks
how dreamhosts using ceph
http://ceph.com/presentations/20121102-ceph-day/20121102-dreamobjects.pdf
== benchmarks ==
http://learnitwithme.com/?p=303
== geo replication ==
http://www.sebastien-han.fr/blog/2013/01/28/ceph-geo-replication-sort-of/
== ceph openstack ==
http://www.sebastien-han.fr/blog/2012/06/10/introducing-ceph-to-openstack/
== webcasts ==
http://en.community.dell.com/techcenter/cloud/w/wiki/4477.inktank-ceph-webinar-series-jan-feb-2013.aspx
== enable guest fs trim ==
https://lwn.net/Articles/415889/
== publication/paper ==
http://static.usenix.org/events/osdi06/tech/full_papers/weil/weil_html/index.html
== some other links ==
http://ceph.com/presentations/20121102-ceph-day/20121102-cluster-design-deployment.pdf
http://ceph.com/presentations/20121102-ceph-day/20121102-ceph-in-the-cloud.pdf
http://it.toolbox.com/blogs/database-soup/testing-disk-speed-the-dd-test-31069
http://ceph.com/docs/master/rados/operations/crush-map/
http://ceph.com/docs/master/rados/configuration/filesystem-recommendations/
http://ceph.com/docs/master/faq/
http://ceph.com/community/ceph-bobtail-jbod-performance-tuning/
http://storageioblog.com/?p=588