Skip to content

Commit

Permalink
Merge pull request #94 from seraphin/memtuning
Browse files Browse the repository at this point in the history 
reapplying memory tuning resolves #93
  • Loading branch information
@ahoybender
ahoybender authored Dec 6, 2016
2 parents a04f495 + 9ffdc6e commit ba6edca
Showing 1 changed file with 317 additions and 1 deletion.
318 changes: 317 additions & 1 deletion playbooks/roles/ambari-server/templates/blueprint-multi-node-3-masters.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,114 @@
"hadoop.proxyuser.hue.hosts" : "*"
}
},
{% if hostvars[groups['master-nodes'][0]]['ansible_memtotal_mb'] > 89000 -%}
{
"zoo.cfg" : {
"properties_attributes" : { },
"properties" : {
"dataDir" : "{% for disk in hostvars[groups['master-nodes'][0]]['ansible_mounts'] if disk.mount | match("/hadoop/ms/*") %}/hadoop/ms/zookeeper{% if not loop.last %},{% endif %}{% else %}/hadoop/zookeeper{%- endfor %}"
}
}
},
{
"ams-hbase-env" : {
"regionserver_xmn_size" : "512m",
"hbase_regionserver_heapsize" : "4096m",
"hbase_master_heapsize" : "4096m",
"hbase_master_xmn_size" : "512m"
}
},
{
"ams-env" : {
"metrics_collector_heapsize" : "4096m"
}
},
{
"hive-env" : {
"hive.heapsize" : "8192",
"hive.metastore.heapsize" : "8192",
"hive.client.heapsize" : "2048"
}
},
{
"hbase-env" : {
"hbase_master_heapsize" : "8192m"
}
},
{
"hadoop-env" : {
"namenode_heapsize" : "8192m",
"namenode_opt_maxnewsize" : "512m",
"namenode_opt_newsize" : "512m"
}
},
{% elif hostvars[groups['master-nodes'][0]]['ansible_memtotal_mb'] > 24000 -%}
{
"ams-hbase-env" : {
"regionserver_xmn_size" : "512m",
"hbase_regionserver_heapsize" : "2048m",
"hbase_master_heapsize" : "2048m",
"hbase_master_xmn_size" : "512m"
}
},
{
"ams-env" : {
"metrics_collector_heapsize" : "2048m"
}
},
{
"hive-env" : {
"hive.heapsize" : "4096",
"hive.metastore.heapsize" : "1024",
"hive.client.heapsize" : "1024"
}
},
{
"hbase-env" : {
"hbase_master_heapsize" : "4096m"
}
},
{
"hadoop-env" : {
"namenode_heapsize" : "4096m",
"namenode_opt_maxnewsize" : "512m",
"namenode_opt_newsize" : "512m"
}
},
{% elif hostvars[groups['master-nodes'][0]]['ansible_memtotal_mb'] > 10000 -%}
{
"ams-hbase-env" : {
"regionserver_xmn_size" : "384m",
"hbase_regionserver_heapsize" : "1024m",
"hbase_master_heapsize" : "1024m",
"hbase_master_xmn_size" : "384m"
}
},
{
"ams-env" : {
"metrics_collector_heapsize" : "1024m"
}
},
{
"hive-env" : {
"hive.heapsize" : "1024",
"hive.metastore.heapsize" : "1024",
"hive.client.heapsize" : "1024"
}
},
{
"hbase-env" : {
"hbase_master_heapsize" : "1024m"
}
},
{
"hadoop-env" : {
"namenode_heapsize" : "2048m",
"namenode_opt_maxnewsize" : "384m",
"namenode_opt_newsize" : "384m"
}
},
{% endif -%}
{
"hdfs-site" : {
{% if hdfs.ha_namenode -%}
Expand All @@ -31,6 +139,13 @@
"dfs.nameservices" : "{{ cluster_name }}",
{% endif -%}
"dfs.datanode.data.dir" : "{% for disk in hostvars[groups['slave-nodes'][0]]['ansible_mounts'] if disk.mount | match("/grid/*") %}{{ disk.mount }}/hadoop/hdfs/data{% if not loop.last %},{% endif %}{% else %}/hadoop/hdfs/data{%- endfor %}",
{% if hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 59000 -%}
"dfs.datanode.balance.bandwidthPerSec" : "12500000",
"dfs.datanode.max.transfer.threads": "4096",
{% endif -%}
{% if arcadia == true -%}
"dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
{% endif -%}
"dfs.journalnode.edits.dir": "{% for disk in hostvars[groups['master-nodes'][0]]['ansible_mounts'] if disk.mount | match("/hadoop/nn/*") %}/hadoop/nn/hdfs/journalnode{% if not loop.last %},{% endif %}{% else %}/hadoop/hdfs/journalnode{%- endfor %}",
"dfs.namenode.checkpoint.dir": "{% for disk in hostvars[groups['master-nodes'][0]]['ansible_mounts'] if disk.mount | match("/hadoop/nn/*") %}/hadoop/nn/hdfs/namenodesecondary{% if not loop.last %},{% endif %}{% else %}/hadoop/hdfs/namenodesecondary{%- endfor %}",
"dfs.namenode.name.dir": "{% for disk in hostvars[groups['master-nodes'][0]]['ansible_mounts'] if disk.mount | match("/hadoop/nn/*") %}/hadoop/nn/hdfs/namenode{% if not loop.last %},{% endif %}{% else %}/hadoop/hdfs/namenode{%- endfor %}",
Expand All @@ -56,11 +171,205 @@
}
},
{% endif -%}
{% if hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 119000 -%}
{
"hadoop-env" : {
"dtnode_heapsize" : "4096m"
}
},
{
"yarn-site" : {
{% if install_hbase == true -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 16384 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//6)//4096)*4096 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 16384 - 8192 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//6)//4096)*4096 }}",
{% else -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//6)//4096)*4096 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 8192 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//6)//4096)*4096 }}",
{% endif -%}
"yarn.scheduler.minimum-allocation-mb" : "4096"
}
},
{
"mapred-site" : {
"mapreduce.map.java.opts" : "-Xmx6553m",
"mapreduce.map.memory.mb" : "8192",
"mapreduce.reduce.java.opts" : "-Xmx6553m",
"mapreduce.reduce.memory.mb" : "8192",
"mapreduce.task.io.sort.mb" : "2047",
"yarn.app.mapreduce.am.command-opts" : "-Xmx3276m -Dhdp.version=${hdp.version}",
"yarn.app.mapreduce.am.resource.mb" : "4096"
}
},
{
"tez-site" : {
"tez.am.resource.memory.mb" : "8192",
"tez.task.resource.memory.mb" : "8192"
}
},
{
"spark-defaults" : {
"spark.executor.instances" : "{{ groups['slave-nodes']|length }}",
"spark.executor.memory" : "7808m",
"spark.driver.memory" : "7808m",
"spark.yarn.am.memory" : "7808m",
"spark.yarn.executor.memoryOverhead" : "384",
"spark.yarn.driver.memoryOverhead" : "384",
"spark.yarn.am.memoryOverhead" : "384"
}
},
{
"zeppelin-config" : {
"properties_attributes" : { },
"properties" : {
"zeppelin.notebook.dir" : "notebook",
"zeppelin.ssl.keystore.type" : "JKS",
"zeppelin.ssl.truststore.path" : "conf/truststore",
"zeppelin.websocket.max.text.message.size" : "1024000",
"zeppelin.notebook.homescreen.hide" : "false",
"zeppelin.server.port" : "9995",
"zeppelin.ssl" : "false",
"zeppelin.notebook.homescreen" : " ",
"zeppelin.notebook.storage" : "org.apache.zeppelin.notebook.repo.VFSNotebookRepo",
"zeppelin.interpreter.connect.timeout" : "30000",
"zeppelin.anonymous.allowed" : "true",
"zeppelin.interpreter.dir" : "interpreter",
"zeppelin.ssl.truststore.type" : "JKS",
"zeppelin.ssl.keystore.path" : "conf/keystore",
"zeppelin.notebook.s3.bucket" : "zeppelin",
"zeppelin.server.allowed.origins" : "*",
"zeppelin.server.addr" : "0.0.0.0",
"zeppelin.ssl.client.auth" : "false"
}
}
},

{
"zeppelin-env" : {
"properties_attributes" : { },
"properties" : {
"zeppelin.spark.jar.dir" : "/apps/zeppelin",
"zeppelin.executor.mem" : "512m",
"zeppelin.server.kerberos.keytab" : "",
"zeppelin_log_dir" : "/var/log/zeppelin",
"zeppelin.executor.instances" : "2",
"zeppelin.server.kerberos.principal" : "",
"zeppelin_user" : "zeppelin",
"zeppelin_pid_dir" : "/var/run/zeppelin",
"zeppelin_group" : "zeppelin"
}
}
},
{
"hbase-env" : {
"hbase_regionserver_heapsize" : "16384m",
"hbase_regionserver_xmn_max" : "2048"
}
},
{% elif hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 59000 -%}
{
"hadoop-env" : {
"dtnode_heapsize" : "2048m"
}
},
{
"yarn-site" : {
{% if install_hbase == true -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 8192 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//4096)*4096 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 8192 - 4096 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//4096)*4096 }}",
{% else -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//4096)*4096 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 4096 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//4096)*4096 }}",
{% endif -%}
"yarn.scheduler.minimum-allocation-mb" : "4096"
}
},
{
"mapred-site" : {
"mapreduce.map.java.opts" : "-Xmx3276m",
"mapreduce.map.memory.mb" : "4096",
"mapreduce.reduce.java.opts" : "-Xmx3276m",
"mapreduce.reduce.memory.mb" : "4096",
"mapreduce.task.io.sort.mb" : "1536",
"yarn.app.mapreduce.am.command-opts" : "-Xmx3276m -Dhdp.version=${hdp.version}",
"yarn.app.mapreduce.am.resource.mb" : "4096"
}
},
{
"tez-site" : {
"tez.am.resource.memory.mb" : "4096",
"tez.task.resource.memory.mb" : "4096"
}
},
{
"spark-defaults" : {
"spark.executor.instances" : "{{ groups['slave-nodes']|length }}"
"spark.executor.instances" : "{{ groups['slave-nodes']|length }}",
"spark.executor.memory" : "7808m",
"spark.driver.memory" : "3712m",
"spark.yarn.am.memory" : "3712m",
"spark.yarn.executor.memoryOverhead" : "384",
"spark.yarn.driver.memoryOverhead" : "384",
"spark.yarn.am.memoryOverhead" : "384"
}
},
{
"hbase-env" : {
"hbase_regionserver_heapsize" : "8192m",
"hbase_regionserver_xmn_max" : "1536"
}
},
{% elif hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 24000 -%}
{
"hadoop-env" : {
"dtnode_heapsize" : "1024m"
}
},
{
"yarn-site" : {
{% if install_hbase == true -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 4096 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//2048)*2048 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 4096 - 2048 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//2048)*2048 }}",
{% else -%}
"yarn.nodemanager.resource.memory-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//2048)*2048 }}",
"yarn.scheduler.maximum-allocation-mb" : "{{ ((hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] - 2048 - hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb']//8)//2048)*2048 }}",
{% endif -%}
"yarn.scheduler.minimum-allocation-mb" : "2048"
}
},
{
"mapred-site" : {
"mapreduce.map.java.opts" : "-Xmx1638m",
"mapreduce.map.memory.mb" : "2048",
"mapreduce.reduce.java.opts" : "-Xmx1638m",
"mapreduce.reduce.memory.mb" : "2048",
"mapreduce.task.io.sort.mb" : "768",
"yarn.app.mapreduce.am.command-opts" : "-Xmx1638m -Dhdp.version=${hdp.version}",
"yarn.app.mapreduce.am.resource.mb" : "2048"
}
},
{
"tez-site" : {
"tez.am.resource.memory.mb" : "2048",
"tez.task.resource.memory.mb" : "2048"
}
},
{
"spark-defaults" : {
"spark.executor.instances" : "{{ groups['slave-nodes']|length }}",
"spark.executor.memory" : "3712m",
"spark.driver.memory" : "1664m",
"spark.yarn.am.memory" : "1664m",
"spark.yarn.executor.memoryOverhead" : "384",
"spark.yarn.driver.memoryOverhead" : "384",
"spark.yarn.am.memoryOverhead" : "384"
}
},
{
"hbase-env" : {
"hbase_regionserver_heapsize" : "4096m",
"hbase_regionserver_xmn_max" : "768"
}
},
{% endif -%}
{
"yarn-site" : {
"yarn.nodemanager.local-dirs" : "{% for disk in hostvars[groups['slave-nodes'][0]]['ansible_mounts'] if disk.mount | match("/grid/*") %}{{ disk.mount }}/hadoop/yarn/local{% if not loop.last %},{% endif %}{% else %}/hadoop/yarn/local{%- endfor %}",
Expand Down Expand Up @@ -91,6 +400,13 @@
},
{
"hive-site" : {
{% if hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 119000 -%}
"hive.tez.container.size" : "8192",
{% elif hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 59000 -%}
"hive.tez.container.size" : "4096",
{% elif hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] > 24000 -%}
"hive.tez.container.size" : "2048",
{% endif -%}
"fs.file.impl.disable.cache" : "true",
"fs.hdfs.impl.disable.cache" : "true",
"javax.jdo.option.ConnectionPassword" : "{{ services_password }}"
Expand Down

0 comments on commit ba6edca

Please sign in to comment.