# tar zcvf - cloudera | split -b 1024m - cloudera.tgz cat cloudera.tgz* | tar zxvf - -C /data ## Filesystem 참고 lvcreate -n data -L 100GB datavg mkfs.xfs /dev/datavg/data mkdir /data mount -t xfs /dev/datavg/data /data lvcreate -n mysqldata -L 20GB datavg mkfs.xfs /dev/datavg/mysqldata mkdir /mysqldata mount -t xfs /dev/datavg/mysqldata /mysqldata vi /etc/hosts xxx.xx.xxx.10 cdpmast01.hadoop.com cdpmast01 xxx.xx.xxx.11 cdpwork01.hadoop.com cdpwork01 xxx.xx.xxx.12 cdpwork02.hadoop.com cdpwork02 xxx.xx.xxx.13 cdpwork03.hadoop.com cdpwork03 # Passwordless-SSH ssh-keygen chmod 700 .ssh cd .ssh cat id_rsa.pub >> authorized_keys chmod 600 authorized_keys cd rsync -av .ssh cdpwork01:/root rsync -av .ssh cdpwork02:/root rsync -av .ssh cdpwork03:/root ssh cdpwork01 vi ssh.sh ------------------------------ #!/bin/bash hosts=( cdpmast01 cdpwork01 cdpwork02 cdpwork03 ) for host in "${hosts[@]}" do echo "===== $host =====" ssh $host "$@" echo "" done ------------------------------ vi rsync.sh ------------------------------ #!/bin/bash hosts=( #cdpmast01 cdpwork01 cdpwork02 cdpwork03 ) for host in "${hosts[@]}" do echo "===== $host =====" rsync -av $1 $host:$2 echo "" done ------------------------------ chmod +x *.sh /root/rsync.sh /etc/hosts /etc ## Disable THP cat /sys/kernel/mm/transparent_hugepage/enabled cat /sys/kernel/mm/transparent_hugepage/defrag vi /etc/rc.local echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag chmod +x /etc/rc.d/rc.local systemctl enable rc-local systemctl start rc-local vi /etc/default/grub grub2-mkconfig -o /boot/grub2/grub.cfg grub2-mkconfig -o /boot/efi/EFI/redhat/grub.cfg grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg cat /proc/cmdline tuned-adm active mkdir /etc/tuned/nothp_profile vi /etc/tuned/nothp_profile/tuned.conf ------------------------------- [main] include=throughput-performance [vm] transparent_hugepages=never ------------------------------- chmod +x /etc/tuned/nothp_profile/tuned.conf tuned-adm profile nothp_profile cat /proc/meminfo | grep Hug # 시스템 점검 getenforce cat /etc/selinux/config systemctl status firewalld systemctl status packagekit systemctl status chronyd hostname -f hostnamectl status hostnamectl set-hostname cdpmast01.hadoop.com vi /etc/sysctl.conf ---------------------------------------- vm.swappiness=1 vm.dirty_ratio=50 vm.dirty_background_ratio=20 vm.overcommit_memory=1 vm.overcommit_ratio=50 #kernel.threads-max=1029157 net.core.somaxconn=8192 net.core.netdev_max_backlog=8192 net.core.rmem_max=134217728 net.core.wmem_max=134217728 net.core.rmem_default=524288 net.core.wmem_default=524288 net.ipv4.tcp_sack=0 net.ipv4.tcp_dsack=0 net.ipv4.tcp_keepalive_time=600 net.ipv4.tcp_keepalive_probes=5 net.ipv4.tcp_keepalive_intvl=15 net.ipv4.tcp_fin_timeout=30 net.ipv4.tcp_rmem=4096 65536 134217728 net.ipv4.tcp_wmem=4096 65536 134217728 net.ipv4.tcp_retries2=10 net.ipv4.tcp_synack_retries=3 net.ipv4.tcp_max_tw_buckets=1440000 net.ipv4.tcp_tw_reuse=1 net.ipv6.conf.all.disable_ipv6=1 net.ipv6.conf.default.disable_ipv6=1 ---------------------------------------- sysctl -p sysctl -w net.ipv4.tcp_tw_recycle=0 sysctl net.core.somaxconn /root/rsync.sh /etc/sysctl.conf /etc vi /etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv4.conf.default.forwarding=1 vi /etc/security/limits.conf ---------------------------------------- * - nofile 65536 * - nproc 65536 ---------------------------------------- /root/rsync.sh /etc/security/limits.conf /etc/security vi /etc/security/limits.d/20-nproc.conf ---------------------------------------- * soft nproc 65536 ---------------------------------------- /root/rsync.sh /etc/security/limits.d/20-nproc.conf /etc/security/limits.d systemctl status bluetooth systemctl disable bluetooth systemctl stop bluetooth systemctl status cups systemctl disable cups systemctl stop cups systemctl status libvirtd systemctl disable libvirtd systemctl stop libvirtd ps -ef|grep -v grep|grep dnsmasq brctl show ip link set virbr0 down ip link set virbr0-nic down ip link delete virbr0 ip link delete virbr0-nic brctl delbr virbr0 brctl delbr virbr0-nic yum install -y ntp yum install -y nscd postfix vi /etc/postfix/main.cf #inet_interfaces = localhost inet_interfaces = 127.0.0.1 inet_protocols = ipv4 cat /etc/chrony.conf systemctl stop chronyd systemctl disable chronyd vi /etc/ntp.conf systemctl status ntpd systemctl start ntpd systemctl enable ntpd /root/ssh.sh yum install -y mysql-connector-java /root/ssh.sh yum install -y java-1.8.0-openjdk-devel /root/ssh.sh yum remove -y java-1.7.0-openjdk-headless echo umask 022 >> /etc/profile /root/rsync.sh /etc/profile /etc mkdir /data/repo ln -s /data/repo /var/www/html/repo -------------------------------------------------------------------------- ## Repository 작업 # httpd 설치 yum install -y httpd systemctl enable httpd cd /etc/httpd/conf.d mv nss.conf nss.conf.bak cd /etc/httpd/conf vi httpd.conf systemctl start httpd # 파일 작업 cd /var/www/html/ ln -s /hadoop/repo repo tar -zxvf cm7.2.4-redhat7.tar.gz -C /hadoop/repo tar -zxvf mariadb-10.2.36.tar.gz -C /hadoop/repo chown -R apache:apache /hadoop/repo mariadb-10.2.36.tar.gz # yum repo 설정 cd /etc/yum.repos.d/ vi cloudera-manager.repo -------------------------------------------------- [cloudera-manager] name=Cloudera Manager 7.2.4 baseurl=http://cdpmast01.hadoop.com/repo/cm7.2.4 enabled=1 gpgcheck=0 -------------------------------------------------- vi mariadb.repo -------------------------------------------------- [mariadb] name=MariaDB 10.2.36 baseurl=http://cdpmast01.hadoop.com/repo/mariadb enabled=1 gpgcheck=0 -------------------------------------------------- # yum 확인 yum clean all yum repolist ## MariaDB yum install MariaDB-server cd /etc/my.cnf.d/ vi server.cnf -------------------------------------------------- [server] port=3306 datadir=/hadoop/mariadb socket=/var/lib/mysql/mysql.sock transaction-isolation=READ-COMMITTED symbolic-links=0 key_buffer_size=32M max_allowed_packet=32M thread_stack=256K thread_cache_size=64 query_cache_limit=8M query_cache_size=64M query_cache_type=1 max_connections=1024 read_buffer_size=2M read_rnd_buffer_size=16M sort_buffer_size=8M join_buffer_size=8M character-set-server=utf8 collation-server=utf8_general_ci # for RMS log_bin_trust_function_creators=1 # InnoDB settings innodb_file_per_table=1 innodb_flush_log_at_trx_commit=2 innodb_log_buffer_size=64M innodb_buffer_pool_size=4G innodb_thread_concurrency=8 innodb_flush_method=O_DIRECT innodb_log_file_size=512M log-error=/var/log/mariadb/mariadb.log pid-file=/var/run/mariadb/mariadb.pid -------------------------------------------------- cd /usr/lib/tmpfiles.d/ cat mariadb.conf cat /usr/lib/systemd/system/mariadb.service LimitNOFILE = 65536 systemctl daemon-reload mysql_install_db --user=mysql --datadir=/hadoop/mariadb systemctl start mariadb mysql_secure_installation mysql -uroot -p create database scm default character set utf8 collate utf8_general_ci; create database rman default character set utf8 collate utf8_general_ci; create database hue default character set utf8 collate utf8_general_ci; create database metastore default character set utf8 collate utf8_general_ci; create database oozie default character set utf8 collate utf8_general_ci; create database ranger default character set utf8 collate utf8_general_ci; create database rangerkms default character set utf8 collate utf8_general_ci; create user 'scm'@'%' identified by 'Hadoop1!'; create user 'rman'@'%' identified by 'Hadoop1!'; create user 'hue'@'%' identified by 'Hadoop1!'; create user 'hive'@'%' identified by 'Hadoop1!'; create user 'oozie'@'%' identified by 'Hadoop1!'; create user 'rangeradmin'@'%' identified by 'Hadoop1!'; create user 'rangerkms'@'%' identified by 'Hadoop1!'; grant all privileges on scm.* to 'scm'@'%'; grant all privileges on rman.* to 'rman'@'%'; grant all privileges on hue.* to 'hue'@'%'; grant all privileges on metastore.* to 'hive'@'%'; grant all privileges on oozie.* to 'oozie'@'%'; grant all privileges on ranger.* to 'rangeradmin'@'%'; grant all privileges on rangerkms.* to 'rangerkms'@'%'; flush privileges; select host,user from mysql.user; show databases; exit; -------------------------------------------------------------------------- -------------------------------------------------------------------------- ## Cloudera Manager 설치 # Cloudera Manager 설치 yum install -y cloudera-manager-server cloudera-manager-agent cloudera-manager-daemons # Enable auto-TLS with an embedded Cloudera Manager CA, run the following command: /opt/cloudera/cm-agent/bin/certmanager setup --configure-services tail -100f /var/log/cloudera-scm-agent/certmanager.log ls -l /var/lib/cloudera-scm-server/certmanager/ # Cloudera Manager Server DB 설정 #scm_prepare_database.sh [options] [password] /opt/cloudera/cm/schema/scm_prepare_database.sh -h cdpmast01.hadoop.com:3306 mysql scm scm ls -l /etc/cloudera-scm-server cat /etc/cloudera-scm-server/db.properties # Cloudera Manager Server 시작 systemctl start cloudera-scm-server -------------------------------------------------------------------------- -------------------------------------------------------------------------- ## 웹페이지에서 작업 # Cloudera Manager 접속 http://cdpmast01.hadoop.com:7180/ admin / admin aiplatform -------------------------------------------------------------------------- jdbc:mysql://cdpmast01.hadoop.com:3306/hive?useUnicode=true&characterEncoding=utf8 jdbc:mysql://cdpmast01.hadoop.com:3306/oozie?useUnicode=true&characterEncoding=utf8 alter table COLUMNS_V2 modify COMMENT mediumtext character set utf8 collate utf8_general_ci; alter table TABLE_PARAMS modify PARAM_VALUE mediumtext character set utf8 collate utf8_general_ci; alter table SERDE_PARAMS modify PARAM_VALUE mediumtext character set utf8 collate utf8_general_ci; alter table SD_PARAMS modify PARAM_VALUE mediumtext character set utf8 collate utf8_general_ci; alter table PARTITION_PARAMS modify PARAM_VALUE varchar(4000) character set utf8 collate utf8_general_ci; alter table PARTITION_KEYS modify PKEY_COMMENT varchar(4000) character set utf8 collate utf8_general_ci; alter table INDEX_PARAMS modify PARAM_VALUE varchar(4000) character set utf8 collate utf8_general_ci; alter table DATABASE_PARAMS modify PARAM_VALUE varchar(4000) character set utf8 collate utf8_general_ci; alter table DBS modify `DESC` varchar(4000) character set utf8 collate utf8_general_ci; vi /etc/python/cert-verification.cfg verify=disable vi /usr/lib64/python2.7/site.py encoding = "ascii" <- utf8 /root/rsync.sh /usr/lib64/python2.7/site.py /usr/lib64/python2.7 # 컬럼 comment cd /usr/local/hue/build/static/desktop/js/bundles/hue vi hue-bundle-65fa61ac2cc2d5853de4.js #self.comment = self.catalogEntry().getCommentObservable(); self.comment = self.catalogEntry().definition.comment; # 자동완성 안됨 vi desktop/core/src/desktop/views.py @login_notrequired <- 추가 def dynamic_bundle(request, config, bundle_name): vi desktop/core/src/desktop/templates/global_js_constants.mako ##window.HAS_READ_ONLY_CATALOG window.HAS_READ_ONLY_CATALOG = true; # Hive + 버튼 404에러 vi desktop/core/src/desktop/templates/global_js_constants.mako window.HUE_URLS = { ##IMPORTER_CREATE_TABLE: '${ 'indexer' in apps and url('indexer:importer_prefill', source_type = 'all', target_type = 'table')}', ##IMPORTER_CREATE_DATABASE: '${ 'indexer' in apps and url('indexer:importer_prefill', source_type = 'manual', target_type = 'database')}', IMPORTER_CREATE_TABLE: '${url('indexer:importer_prefill', source_type = 'all', target_type = 'table')}', IMPORTER_CREATE_DATABASE: '${url('indexer:importer_prefill', source_type = 'manual', target_type = 'database')}', ## 파일작업 /var/www/html/tez-ui tar -zxvf tez-ui.tar.gz -C /var/www/html cd /var/www/html chown -R apache:apache tez-ui cd config vi configs.env -- configs.env timeline 주석 해제, 값 수정 http://localhost:8188 → http://cdpmast01.hadoop.com:8188 rm 주석 해제, 값 수정 http://localhost:8088 → http://cdpmast01.hadoop.com:8088 timeZone 주석 해제, 값 수정 UTC → Asia/Seoul -- vendor.js 파일 8632 라인 마지막에 추가 cacheURL + ( rquery.test( cacheURL ) ? "&" : "?" ) + "_=" + nonce++ + "&user.name=hive"; -- tez-ui.js 파일 7122 라인 demand → lazy로 수정 loadType: "lazy", ### Tez yarn.timeline-service.enabled 수정 false → true tez.am.tez-ui.history-url.template 수정 __HISTORY_URL_BASE__#/app/__APPLICATION_ID__/dags 원래값 --> __HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__ tez.history.logging.service.class 수정 org.apache.tez.dag.history.logging.proto.ProtoHistoryLoggingService → org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService tez.tez-ui.history-url.base=http://cdpmast01.hadoop.com:8686/tez-ui/ 추가 ### Hive General hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook hive.exec.pre.hooks=org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook,org.apache.hadoop.hive.ql.security.authorization.plugin.DisallowTransformHook #### YARN # 2곳에 동일하게 설정 넣어줌 - YARN Service Advanced Configuration Snippet (Safety Valve) for yarn-site.xml - YARN Client Advanced Configuration Snippet (Safety Valve) for yarn-site.xml yarn.timeline-service.hostnamecdpmast01.hadoop.com yarn.timeline-service.bind-host0.0.0.0 yarn.timeline-service.client.fd-flush-interval-secs5Default: 10 yarn.timeline-service.enabledtrueDefault: false yarn.timeline-service.entity-group-fs-store.active-dir/ats/active/Default: /tmp/entity-file-history/active/ yarn.timeline-service.entity-group-fs-store.done-dir/ats/done/Default: /tmp/entity-file-history/done/ yarn.timeline-service.entity-group-fs-store.group-id-plugin-classesorg.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin yarn.timeline-service.entity-group-fs-store.scan-interval-seconds15Default: 60 yarn.timeline-service.entity-group-fs-store.summary-storeorg.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStoreDefault: org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore yarn.timeline-service.generic-application-history.store-classorg.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore yarn.timeline-service.http-authentication.proxyuser.root.groups* yarn.timeline-service.http-authentication.proxyuser.root.hosts* yarn.timeline-service.http-cross-origin.enabledtrueDefault: false yarn.timeline-service.leveldb-state-store.path/hadoop/yarn/timelineDefault: ${hadoop.tmp.dir}/yarn/timeline yarn.timeline-service.leveldb-timeline-store.path/hadoop/yarn/timelineDefault: ${hadoop.tmp.dir}/yarn/timeline yarn.timeline-service.reader.webapp.addresscdpmast01.hadoop.com:8198Default: ${yarn.timeline-service.webapp.address} yarn.timeline-service.reader.webapp.https.addresscdpmast01.hadoop.com:8199${yarn.timeline-service.webapp.https.address} yarn.timeline-service.recovery.enabledtrueDefault: false yarn.timeline-service.store-classorg.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStoreDefault: org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore yarn.timeline-service.ttl-ms2678400000Default: 604800000 yarn.timeline-service.version1.5fDefault: 1.0f yarn.timeline-service.versions1.5fDefault: 1.5f,2.0f yarn.timeline-service.webapp.addresscdpmast01.hadoop.com:8188Default: ${yarn.timeline-service.hostname}:8188 yarn.timeline-service.webapp.https.addresscdpmast01.hadoop.com:8190Default: ${yarn.timeline-service.hostname}:8190 yarn.resourcemanager.webapp.cross-origin.enabledtrueDefault: false #### Timeline Server 실행(yarn 계정에서) # Foreground 실행(에러 확인) yarn timelineserver # Background 실행 yarn --daemon start timelineserver yarn --daemon stop timelineserver # https://docs.cloudera.com/cdp-private-cloud-base/7.1.5/installation/topics/cdpdc-additional-steps-ranger.html solrctl instancedir --get ranger_audits /tmp/ranger_audits vi /tmp/ranger_audits/conf/solrconfig.xml ttl +90DAYS solrctl instancedir --update ranger_audits /tmp/ranger_audits solrctl collection --reload ranger_audits /* 1. SERDES */ update SERDES set SLIB = 'org.apache.hadoop.hive.kudu.KuduSerDe' where SERDE_ID in (select * from ( select c.SERDE_ID from TABLE_PARAMS a join TBLS b on b.TBL_ID = a.TBL_ID join SDS c on c.SD_ID = b.SD_ID where a.PARAM_KEY = 'storage_handler' and a.PARAM_VALUE = 'org.apache.hadoop.hive.kudu.KuduStorageHandler' and c.INPUT_FORMAT = '' ) t); /* 2. SDS */ update SDS set INPUT_FORMAT = 'org.apache.hadoop.hive.kudu.KuduInputFormat', OUTPUT_FORMAT = 'org.apache.hadoop.hive.kudu.KuduOutputFormat' where SD_ID in (select * from ( select c.SD_ID from TABLE_PARAMS a join TBLS b on b.TBL_ID = a.TBL_ID join SDS c on c.SD_ID = b.SD_ID where a.PARAM_KEY = 'storage_handler' and a.PARAM_VALUE = 'org.apache.hadoop.hive.kudu.KuduStorageHandler' and c.INPUT_FORMAT = '' ) t); # libmysqlclient.so.21 AppStream rpm -qlp mysql-libs-8.0.21-1.module+el8.2.0+7855+47abd494.x86_64.rpm mkdir files cd files rpm2cpio ../mysql-libs-8.0.21-1.module+el8.2.0+7855+47abd494.x86_64.rpm | cpio -idmv /bin/mv usr/lib64/mysql/libmysqlclient.so.21* /usr/lib64/ cd ../ /bin/rm -r files