集群模式cluster-config.json配置优化

如果存储引擎或RPC组件是使用自定义组件，忽略以下hbase/hadoop/ice相关配置即可！

config.json包括XL-LightHouse自身配置参数和其依赖组件的配置参数。为了方便起见，将各组件常用配置参数统一放在config.json文件中。

如需进行相关优化，请修改操作节点的config.json文件，执行重启操作(restart-all.sh)即可！

各组件内存初始占用比例

整体优化建议

• 1、各组件最大内存总共占用节点内存的70%以内，集群运行时内存使用率在60~70%之间为宜；
• 2、一般来说集群的内存资源优先分配原则为：HBase > LightHouse-Tasks 、 Reids > 其他组件进程，所指的HBase进程是指RegionServer进程。
• 3、LightHouse-Tasks单进程内存上限配置3000M即可（除非Spark任务页面出现GC耗时标红则可适当增加少许内存），对应配置参数为：tasks_executor_memory。通过调整executors数量调整任务并行度，对应参数：tasks_num_executors，该数量配置取决于集群有多少内存资源可以分配以及当前并行度是否满足使用，初始配置时可参考以下公式： Executors数量配置为：((单节点内存 10%)/单进程内存) (集群节点数量 - 1)。 比如以三台128G内存的节点配置为例，executor数量为： ((128000 10%)/单进程内存) (集群节点数量 - 1) = 8或9
• 4、一般来说除了HBase和Redis外，其他进程的内存都有配置上限，进程只要GC正常即可，配置过多不会对整体性能有提升，反而会浪费资源。
• 5、redis的max_memory参数将会影响bitcount相关计算，如果bitcount计算较多建议适当增加该参数值！

参数说明

cluster-config.json快速配置参考

如果您对于各组件优化缺乏相关经验，可以按照自身服务器配置情况，直接拷贝以下配置即可！

单节点16G内存配置参考

{
  "lighthouse": {
    "timezone": "Asia/Shanghai",
    "ice_xmx_memory": "600M",
    "ice_xms_memory": "600M",
    "tasks_driver_memory": "500M",
    "tasks_executor_memory": "800M",
    "tasks_direct_memory": "256M",
    "tasks_num_executors": 6,
    "tasks_executor_cores": 2,
    "insights_xmx_memory": "300M",
    "insights_xms_memory": "300M"
  },
  "zookeeper": {
    "xmx_memory": "260M",
    "xms_memory": "260M"
  },
  "hadoop": {
    "namenode_xmx_memory": "480M",
    "namenode_xms_memory": "480M",
    "datanode_xmx_memory": "600M",
    "datanode_xms_memory": "600M",
    "secondarynamenode_xmx_memory": "480M",
    "secondarynamenode_xms_memory": "480M",
    "resourcemanager_xmx_memory": "480M",
    "resourcemanager_xms_memory": "480M",
    "nodemanager_xmx_memory": "480M",
    "nodemanager_xms_memory": "480M",
    "yarn.scheduler.maximum-allocation-mb": "160000",
    "yarn.nodemanager.resource.memory-mb": "168000",
    "yarn.scheduler.maximum-allocation-vcores": "32",
    "yarn.nodemanager.resource.cpu-vcores": "32"
  },
  "hbase": {
    "master_xmx_memory": "480M",
    "master_xms_memory": "480M",
    "master_direct_memory": "400M",
    "regionserver_xmx_memory": "1300M",
    "regionserver_xms_memory": "1300M",
    "regionserver_direct_memory": "400M"
  },
  "kafka": {
    "xmx_memory": "300M",
    "xms_memory": "300M"
  },
  "redis": {
    "max_memory": "200M"
  },
  "mysql": {
    "max_connections": "151",
    "innodb_buffer_pool_size": "500M",
    "tmp_table_size": "8M",
    "max_heap_table_size": "8M"
  }
}

单节点32G内存配置参考

• cluster-config.json配置

  {
    "lighthouse": {
        "timezone": "Asia/Shanghai",
        "ice_xmx_memory": "1300M",
        "ice_xms_memory": "1300M",
        "tasks_driver_memory": "800M",
        "tasks_executor_memory": "1500M",
        "tasks_direct_memory": "256M",
        "tasks_num_executors": 6,
        "tasks_executor_cores": 2,
        "insights_xmx_memory": "512M",
        "insights_xms_memory": "512M"
    },
    "zookeeper": {
        "xmx_memory": "480M",
        "xms_memory": "480M"
    },
    "hadoop": {
        "namenode_xmx_memory": "1280M",
        "namenode_xms_memory": "1280M",
        "datanode_xmx_memory": "1280M",
        "datanode_xms_memory": "1280M",
        "secondarynamenode_xmx_memory": "1280M",
        "secondarynamenode_xms_memory": "1280M",
        "resourcemanager_xmx_memory": "1280M",
        "resourcemanager_xms_memory": "1280M",
        "nodemanager_xmx_memory": "1280M",
        "nodemanager_xms_memory": "1280M",
        "yarn.scheduler.maximum-allocation-mb": "160000",
        "yarn.nodemanager.resource.memory-mb": "168000",
        "yarn.scheduler.maximum-allocation-vcores": "32",
        "yarn.nodemanager.resource.cpu-vcores": "32"
    },
    "hbase": {
        "master_xmx_memory": "1280M",
        "master_xms_memory": "1280M",
        "master_direct_memory": "512M",
        "regionserver_xmx_memory": "5600M",
        "regionserver_xms_memory": "5600M",
        "regionserver_direct_memory": "800M"
    },
    "kafka": {
        "xmx_memory": "800M",
        "xms_memory": "800M"
    },
    "redis": {
        "max_memory": "1024M"
    },
    "mysql": {
        "max_connections": "151",
        "innodb_buffer_pool_size": "500M",
        "tmp_table_size": "16M",
        "max_heap_table_size": "16M"
    }
  }
  

单节点64G内存配置参考

• cluster-config.json配置

  {
    "lighthouse": {
        "timezone": "Asia/Shanghai",
        "ice_xmx_memory": "1500M",
        "ice_xms_memory": "1500M",
        "tasks_driver_memory": "1024M",
        "tasks_executor_memory": "1800M",
        "tasks_direct_memory": "256M",
        "tasks_num_executors": 7,
        "tasks_executor_cores": 2,
        "insights_xmx_memory": "1024M",
        "insights_xms_memory": "1024M"
    },
    "zookeeper": {
        "xmx_memory": "512M",
        "xms_memory": "512M"
    },
    "hadoop": {
        "namenode_xmx_memory": "2560M",
        "namenode_xms_memory": "2560M",
        "datanode_xmx_memory": "2560M",
        "datanode_xms_memory": "2560M",
        "secondarynamenode_xmx_memory": "2560M",
        "secondarynamenode_xms_memory": "2560M",
        "resourcemanager_xmx_memory": "2560M",
        "resourcemanager_xms_memory": "2560M",
        "nodemanager_xmx_memory": "2560M",
        "nodemanager_xms_memory": "2560M",
        "yarn.scheduler.maximum-allocation-mb": "160000",
        "yarn.nodemanager.resource.memory-mb": "168000",
        "yarn.scheduler.maximum-allocation-vcores": "32",
        "yarn.nodemanager.resource.cpu-vcores": "32"
    },
    "hbase": {
        "master_xmx_memory": "2560M",
        "master_xms_memory": "2560M",
        "master_direct_memory": "1024M",
        "regionserver_xmx_memory": "11200M",
        "regionserver_xms_memory": "11200M",
        "regionserver_direct_memory": "1600M"
    },
    "kafka": {
        "nodes_size": 3,
        "xmx_memory": "1600M",
        "xms_memory": "1600M"
    },
    "redis": {
        "max_memory": "1600M"
    },
    "mysql": {
        "max_connections": "151",
        "innodb_buffer_pool_size": "500M",
        "tmp_table_size": "16M",
        "max_heap_table_size": "16M"
    }
  }
  

  单节点128G内存配置参考

• cluster-config.json配置

  {
    "lighthouse": {
        "timezone": "Asia/Shanghai",
        "ice_xmx_memory": "2048M",
        "ice_xms_memory": "2048M",
        "tasks_driver_memory": "1500M",
        "tasks_executor_memory": "2000M",
        "tasks_direct_memory": "256M",
        "tasks_num_executors": 8,
        "tasks_executor_cores": 2,
        "insights_xmx_memory": "4096M",
        "insights_xms_memory": "4096M"
    },
    "zookeeper": {
        "xmx_memory": "1920M",
        "xms_memory": "1920M"
    },
    "hadoop": {
        "namenode_xmx_memory": "2560M",
        "namenode_xms_memory": "2560M",
        "datanode_xmx_memory": "5120M",
        "datanode_xms_memory": "5120M",
        "secondarynamenode_xmx_memory": "2560M",
        "secondarynamenode_xms_memory": "2560M",
        "resourcemanager_xmx_memory": "2560M",
        "resourcemanager_xms_memory": "2560M",
        "nodemanager_xmx_memory": "5120M",
        "nodemanager_xms_memory": "5120M",
        "yarn.scheduler.maximum-allocation-mb": "160000",
        "yarn.nodemanager.resource.memory-mb": "168000",
        "yarn.scheduler.maximum-allocation-vcores": "32",
        "yarn.nodemanager.resource.cpu-vcores": "32"
    },
    "hbase": {
        "master_xmx_memory": "2560M",
        "master_xms_memory": "2560M",
        "master_direct_memory": "1024M",
        "regionserver_xmx_memory": "22400M",
        "regionserver_xms_memory": "22400M",
        "regionserver_direct_memory": "3200M"
    },
    "kafka": {
        "xmx_memory": "3200M",
        "xms_memory": "3200M"
    },
    "redis": {
        "max_memory": "3200M"
    },
    "mysql": {
        "max_connections": "200",
        "innodb_buffer_pool_size": "1024M",
        "tmp_table_size": "32M",
        "max_heap_table_size": "32M"
    }
  }
  

单节点256G内存配置参考

• cluster-config.json配置

  {
    "lighthouse": {
        "timezone": "Asia/Shanghai",
        "ice_xmx_memory": "4096M",
        "ice_xms_memory": "4096M",
        "tasks_driver_memory": "2048M",
        "tasks_executor_memory": "2000M",
        "tasks_direct_memory": "256M",
        "tasks_num_executors": 12,
        "tasks_executor_cores": 2,
        "insights_xmx_memory": "4096M",
        "insights_xms_memory": "4096M"
    },
    "zookeeper": {
        "xmx_memory": "3840M",
        "xms_memory": "3840M"
    },
    "hadoop": {
        "namenode_xmx_memory": "5120M",
        "namenode_xms_memory": "5120M",
        "datanode_xmx_memory": "5120M",
        "datanode_xms_memory": "5120M",
        "secondarynamenode_xmx_memory": "5120M",
        "secondarynamenode_xms_memory": "5120M",
        "resourcemanager_xmx_memory": "5120M",
        "resourcemanager_xms_memory": "5120M",
        "nodemanager_xmx_memory": "5120M",
        "nodemanager_xms_memory": "5120M",
        "yarn.scheduler.maximum-allocation-mb": "160000",
        "yarn.nodemanager.resource.memory-mb": "168000",
        "yarn.scheduler.maximum-allocation-vcores": "32",
        "yarn.nodemanager.resource.cpu-vcores": "32"
    },
    "hbase": {
        "master_xmx_memory": "2560M",
        "master_xms_memory": "2560M",
        "master_direct_memory": "1024M",
        "regionserver_xmx_memory": "44800M",
        "regionserver_xms_memory": "44800M",
        "regionserver_direct_memory": "6400M"
    },
    "kafka": {
        "xmx_memory": "5120M",
        "xms_memory": "5120M"
    },
    "redis": {
        "max_memory": "6400M"
    },
    "mysql": {
        "max_connections": "200",
        "innodb_buffer_pool_size": "1024M",
        "tmp_table_size": "32M",
        "max_heap_table_size": "32M"
    }
  }